Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+15 -14
View File
@@ -67,20 +67,20 @@ class BlockTable:
cp_kv_cache_interleave_size: int,
slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT,
):
"""
Args:
block_size: Block size used for KV cache memory allocation
max_num_reqs: Maximum number of concurrent requests supported.
max_num_blocks_per_req: Maximum number of blocks per request.
max_num_batched_tokens: Maximum number of tokens in a batch.
pin_memory: Whether to pin memory for faster GPU transfers.
device: Target device for the block table.
kernel_block_size: The block_size of underlying attention kernel.
Will be the same as `block_size` if `block_size` is supported
by the attention kernel.
slot_mapping_mode: How this cache group maps scheduled tokens to
cache slots. Mamba-like state caches do not use token slot
mappings and should use SlotMappingMode.NONE.
"""Args:
block_size: Block size used for KV cache memory allocation
max_num_reqs: Maximum number of concurrent requests supported.
max_num_blocks_per_req: Maximum number of blocks per request.
max_num_batched_tokens: Maximum number of tokens in a batch.
pin_memory: Whether to pin memory for faster GPU transfers.
device: Target device for the block table.
kernel_block_size: The block_size of underlying attention kernel.
Will be the same as `block_size` if `block_size` is supported
by the attention kernel.
slot_mapping_mode: How this cache group maps scheduled tokens to
cache slots. Mamba-like state caches do not use token slot
mappings and should use SlotMappingMode.NONE.
"""
self.max_num_reqs = max_num_reqs
self.max_num_batched_tokens = max_num_batched_tokens
@@ -254,6 +254,7 @@ class BlockTable:
# kv_manager_block_id 0 → kernel block id [0, 1]
# kv_manager_block_id 1 → kernel block id [2, 3]
# kv_manager_block_id 2 → kernel block id [4, 5]
"""
if blocks_per_kv_block == 1:
return kv_manager_block_ids
+3 -6
View File
@@ -112,8 +112,7 @@ def _post_process_dp_padding(tensor: torch.Tensor, should_dp_pad: bool) -> torch
def _post_process_cudagraph_mode(tensor: torch.Tensor) -> int:
"""
Synchronize cudagraph_mode across DP ranks by taking the minimum.
"""Synchronize cudagraph_mode across DP ranks by taking the minimum.
If any rank has NONE (0), all ranks use NONE.
This ensures all ranks send consistent values (all padded or all unpadded).
"""
@@ -127,8 +126,7 @@ def _synchronize_dp_ranks(
cudagraph_mode: int,
parallel_config: ParallelConfig,
) -> tuple[bool, torch.Tensor | None, int]:
"""
1. Decides if each DP rank is going to microbatch. Either all ranks
"""1. Decides if each DP rank is going to microbatch. Either all ranks
run with microbatching or none of them do.
2. Determines the total number of tokens that each rank will run.
@@ -197,8 +195,7 @@ def coordinate_batch_across_dp(
uniform_decode: bool | None = None,
cudagraph_mode: int = 0,
) -> tuple[bool, torch.Tensor | None, int]:
"""
Coordinates amongst all DP ranks to determine if and how the full batch
"""Coordinates amongst all DP ranks to determine if and how the full batch
should be split into microbatches.
Args:
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Define EC connector functionality mixin for model runners.
"""
"""Define EC connector functionality mixin for model runners."""
from collections.abc import Generator
from contextlib import AbstractContextManager, contextmanager, nullcontext
+8 -1
View File
@@ -76,7 +76,8 @@ class EncoderCudaGraphManager:
model: SupportsEncoderCudaGraph,
):
"""Initialize CUDA graph manager with provided token budgets
and max batch size."""
and max batch size.
"""
self.vllm_config = vllm_config
self.device = device
self.dtype = dtype
@@ -351,6 +352,7 @@ class EncoderCudaGraphManager:
Returns:
Token budget if found, None if no fitting budget.
"""
budgets = budgets if budgets is not None else self.token_budgets
for budget in budgets:
@@ -411,8 +413,11 @@ class EncoderCudaGraphManager:
mm_kwargs: Multimodal inputs for the batch.
token_budget: Token budget to use.
path: Configured encoder path.
axis_keys: Resolved capture-axis keys for this batch.
Returns:
Encoder outputs, or None if graph not captured.
"""
graph_set = self._get_graph_set(path)
num_items = len(self._get_item_specs(mm_kwargs))
@@ -562,6 +567,7 @@ class EncoderCudaGraphManager:
images_per_rank: Number of items per rank.
max_output_tokens_per_rank: Max output tokens across all ranks
(for padding during all_gather).
"""
tp_size = get_tensor_model_parallel_world_size()
current_rank = get_tensor_model_parallel_rank()
@@ -687,6 +693,7 @@ class EncoderCudaGraphManager:
Returns:
List of encoder outputs (one per item).
"""
if self.use_dp:
per_item_out_tokens = self._get_per_item_out_tokens(mm_kwargs)
+10 -5
View File
@@ -62,7 +62,8 @@ class AttentionState(NamedTuple):
@dataclass(frozen=True)
class BatchExecutionDescriptor:
"""Describes the shape of the batch and CG mode to run; this is used to make shape
matches between the capture and runtime."""
matches between the capture and runtime.
"""
cg_mode: CUDAGraphMode
num_tokens: int
@@ -90,7 +91,8 @@ def make_cudagraph_stats(
class CreateForwardFn(Protocol):
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
forward_fn. Called with warmup=True for the warmup pass and warmup=False
for the captured pass."""
for the captured pass.
"""
def __call__(
self,
@@ -408,6 +410,8 @@ class CudaGraphManager:
it is invoked once with warmup=True and again with warmup=False
because attention backends may mutate or lazily initialize
metadata during warmup.
progress_bar_desc: Description shown on the capture progress bar.
"""
with graph_capture(device=self.device), ExitStack() as stack:
if self.ubatch_runner is not None:
@@ -503,7 +507,6 @@ class CudaGraphManager:
num_ubatches: int = 1,
) -> BatchExecutionDescriptor:
"""Find matching cudagraph descriptor from priority-ordered candidates."""
effective_loras = self._resolve_effective_loras(num_active_loras)
key = (num_tokens, effective_loras)
if self._graphs_captured and num_tokens > 0 and key in self._candidates:
@@ -952,7 +955,8 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) -> int:
"""Extrapolate the total FULL capture cost from samples of the largest
graphs. The first capture allocates the pool baseline; later graphs mostly
reuse it, so the second sample is taken as the per-graph cost."""
reuse it, so the second sample is taken as the per-graph cost.
"""
if not mem_samples:
return 0
first_capture = mem_samples[0]
@@ -989,7 +993,8 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
"""Release the profiling KV cache and captured graphs while keeping model
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
weights, so the real ``initialize_kv_cache`` starts from a clean slate.
"""
torch.accelerator.synchronize()
if hasattr(runner.model_state, "_mamba_ctx"):
runner.model_state._mamba_ctx = None
+8 -2
View File
@@ -55,8 +55,7 @@ def sync_cudagraph_and_dp_padding(
allow_ubatching: bool = False,
uniform_decode: bool = False,
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
"""
Coordinates the batch descriptor and DP padding across all ranks.
"""Coordinates the batch descriptor and DP padding across all ranks.
`parallel_config` is only needed to decide whether to microbatch, so callers
that never do (`allow_ubatching=False`) can leave it out.
@@ -248,6 +247,12 @@ def dispatch_cg_and_sync_dp(
need_eager: Force `CUDAGraphMode.NONE` instead of dispatching.
num_active_loras: Active LoRA count for this rank. Does not need
cross-rank agreement; it never changes a bucket's token count.
parallel_config: Only needed to decide whether to microbatch, so
callers that never do (`allow_ubatching=False`) can leave it out.
allow_ubatching: Whether this rank's batch may be split into
microbatches. Agreed across ranks before it takes effect.
uniform_decode: Whether this rank's batch is a uniform decode, used
to pick the microbatching split.
dp_sync: Agreement from a prior dispatch over this same batch, to reuse.
Must come from a batch with this same padded `num_tokens` and the
same `uniform_token_count`; `num_reqs` may differ, as neither
@@ -257,6 +262,7 @@ def dispatch_cg_and_sync_dp(
Returns:
(batch_desc, sync), where `sync` is this batch's agreement for a later
dispatch to reuse. It is None when `dp_size` is 1 or no rank has work.
"""
reuse_eager = dp_sync is not None and dp_sync.eager
+1 -2
View File
@@ -21,8 +21,7 @@ def get_lora_capture_cases(
lora_config: "LoRAConfig | None",
compilation_config: "CompilationConfig",
) -> list[int]:
"""
Return num_active_loras values for cudagraph capture.
"""Return num_active_loras values for cudagraph capture.
When cudagraph_specialize_lora=True: powers of 2 up to max_loras, plus
max_loras+1. When False: [0, max_loras+1]. When LoRA disabled: [0].
+1 -2
View File
@@ -24,8 +24,7 @@ class EncoderCache:
self.mm_features.pop(req_id, None)
def reset_mm_cache(self) -> None:
"""
Clear the multi-modal cache that was used during profiling,
"""Clear the multi-modal cache that was used during profiling,
but no longer needed during inference.
"""
# NOTE: v2 encoder cache profiling skips the multi-modal cache
+3 -3
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
NOTE: Coding style guide for this file:
"""NOTE: Coding style guide for this file:
This model runner is shared by all models: text and multimodal, generative
and embedding, public and private. As a result, this file must only contain
code that is common to every model. Model-specific behavior belongs in the
@@ -2229,7 +2228,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
def shutdown(self) -> None:
"""Release GPU tensors (model weights, KV caches, workspace) so that
memory is reclaimable when running in the same process."""
memory is reclaimable when running in the same process.
"""
torch.accelerator.synchronize()
self.cudagraph_manager = None
self.fast_prefill = None
+2 -1
View File
@@ -144,7 +144,8 @@ class ModelState(ABC):
) -> None:
"""Hook run on real batches before the forward pass (after block tables
are gathered). Used by mamba "align" prefix caching to pre-copy state
across block boundaries. No-op by default."""
across block boundaries. No-op by default.
"""
return None
def postprocess_state(
+4 -3
View File
@@ -38,8 +38,8 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
that produce a sampled token this step, and therefore must have that token
(and the draft block proposed from it) propagated to the earlier PP stages.
Returns None if no request in the batch produces a sample."""
Returns None if no request in the batch produces a sample.
"""
old_computed = input_batch.num_computed_tokens_np
prefill_len = input_batch.prefill_len_np
# Exclude non-final prefill chunks (they don't produce a sample).
@@ -179,7 +179,8 @@ class PPHandler:
def receive(self, input_batch: InputBatch) -> bool:
"""Returns True iff sampled tokens need to be gathered from *all*
requests in the batch."""
requests in the batch.
"""
assert not self.is_last_rank
need_sampled_mask = compute_need_sampled_mask(input_batch)
if need_sampled_mask is None:
+2 -1
View File
@@ -76,7 +76,8 @@ MAX_COMPACT_SUPPORT = 2048
class SamplingMaskTensors(NamedTuple):
"""Device-side masks pending async D2H: compact ids, plus the bitmask as
the exact fallback for rows wider than ``max_num_kept``."""
the exact fallback for rows wider than ``max_num_kept``.
"""
# [num_requests, max_num_kept]
token_ids: torch.Tensor
+2 -1
View File
@@ -107,7 +107,8 @@ class Sampler:
self, idx_mapping_np: np.ndarray, include_token_ids: bool = True
) -> tuple[int, int] | None:
"""(num_logprobs, max_per_req_token_ids) for the given requests, or
None when none of them want logprobs."""
None when none of them want logprobs.
"""
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
max_token_ids = (
self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np)
+1
View File
@@ -163,6 +163,7 @@ def repeated_context_mask(
include_prompt: Search the prompt as well as the generated tokens.
skip_partial_context: Mark contexts containing start padding so they use
ordinary sampling.
"""
if max_history is not None and max_history < 1:
raise ValueError("max_history must be positive or None")
@@ -88,9 +88,14 @@ def build_cost_tables_from_curves(
"""Build cost tables: graph-padded below the capture limit, smooth above.
Args:
draft_curve: (size, cost) samples for the draft model.
verify_curve: (size, cost) samples for the verify model.
max_num_reqs: Largest request count to build a table for.
max_batch_tokens: Largest token count to build a table for.
cudagraph_limit: Largest cudagraph-captured size. At or below it,
execution pads up to the next captured size, so cost is a step
function. Above it there is no padding, so cost is continuous.
"""
def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray:
@@ -185,7 +190,8 @@ class AdaptiveVerificationManager:
"""Dummy-run kwargs whose step timings seed the cost tables.
Run these inside StepTimingCollector.collect(), then hand the block's
timings to set_initial_cost_curves."""
timings to set_initial_cost_curves.
"""
max_num_tokens = self.req_states.max_num_batched_tokens
size = self._cudagraph_limit = capture_sizes[-1] if capture_sizes else 0
# Also profile beyond the capture limit: real steps run there
@@ -255,7 +261,8 @@ class AdaptiveVerificationManager:
input_batch: "InputBatch",
) -> None:
"""Publish this step's raw confidences for the ranking kernel and start
copying them to the CPU, where a later step's budget reads them."""
copying them to the CPU, where a later step's budget reads them.
"""
num_reqs = input_batch.num_reqs
ready_idx = self._stale_idx ^ 1
with gpu_sync_allowed():
@@ -73,8 +73,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
@property
def advance_draft_positions(self) -> bool:
"""
Whether to increment positions and seq_lens between draft steps.
"""Whether to increment positions and seq_lens between draft steps.
True for Eagle/standard MTP (each step produces new KV).
False for Gemma4 MTP (Q-only, shares target KV, constant positions).
@@ -72,7 +72,8 @@ def _prepare_dflash_inputs_to_capture(
class DFlashCudaGraphManager(CudaGraphManager):
"""DFlash CudaGraphManager for the parallel-drafting query forward,
building its own attention metadata from scratch."""
building its own attention metadata from scratch.
"""
def capture(
self,
@@ -13,7 +13,6 @@ from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
"""Share when the draft has no own copy, or its copy matches the target."""
if not getattr(eagle, flag, False) or draft is None:
return True
if target is None:
@@ -29,7 +28,8 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module):
"""The target's lm_head — from get_language_model() for
*ForConditionalGeneration targets, else the top-level module."""
*ForConditionalGeneration targets, else the top-level module.
"""
return getattr(target_language_model, "lm_head", None) or getattr(
target_model, "lm_head", None
)
+2 -1
View File
@@ -268,7 +268,8 @@ class DraftModelSpeculator(BaseSpeculator):
@property
def attn_vllm_config(self) -> VllmConfig:
"""Config for the draft's attention metadata builders. Overridden by
speculators whose attention mode differs from the target's."""
speculators whose attention mode differs from the target's.
"""
return self.vllm_config
def set_attn(
+12 -15
View File
@@ -323,7 +323,6 @@ class InputBatch:
"""Track add-request operations for logits processors.
Not applicable to pooling models.
"""
# Fill the next empty index if there is one.
if (new_req_index := self.batch_update_builder.pop_removed()) is None:
# Append to end otherwise.
@@ -531,8 +530,8 @@ class InputBatch:
Returns:
Removed request index, or `None` if `req_id` not recognized
"""
"""
req_index = self.req_id_to_index.pop(req_id, None)
if req_index is None:
return None
@@ -710,6 +709,7 @@ class InputBatch:
Returns:
swaps: list of (from,to) swap tuples for moved requests
empty_req_indices: indices not filled by condensation
"""
num_reqs = self.num_reqs
@@ -835,7 +835,6 @@ class InputBatch:
def refresh_metadata(self):
"""Apply any batch updates to sampling metadata."""
if self.is_pooling_model:
batch_changed = self.batch_update_builder.reset()
if batch_changed:
@@ -1001,18 +1000,19 @@ class InputBatch:
def make_lora_inputs(
self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray
) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]:
"""
Given the num_scheduled_tokens for each request in the batch, return
"""Given the num_scheduled_tokens for each request in the batch, return
datastructures used to activate the current LoRAs.
Returns:
1. prompt_lora_mapping: A tuple of size np.sum(num_sampled_tokens)
where, prompt_lora_mapping[i] is the LoRA id to use for the ith
sampled token.
where, prompt_lora_mapping[i] is the LoRA id to use for the ith
sampled token.
2. token_lora_mapping: A tuple of size np.sum(num_scheduled_tokens)
where, token_lora_mapping[i] is the LoRA id to use for ith token.
where, token_lora_mapping[i] is the LoRA id to use for ith
token.
3. lora_requests: Set of relevant LoRA requests.
"""
"""
req_lora_mapping = self.request_lora_mapping[: self.num_reqs]
prompt_lora_mapping = tuple(req_lora_mapping.repeat(num_sampled_tokens))
token_lora_mapping = tuple(req_lora_mapping.repeat(num_scheduled_tokens))
@@ -1028,8 +1028,7 @@ class InputBatch:
sampled_token_ids_cpu: torch.Tensor,
async_copy_ready_event: torch.Event,
) -> None:
"""
In async scheduling case, store ref to sampled_token_ids_cpu
"""In async scheduling case, store ref to sampled_token_ids_cpu
tensor and corresponding copy-ready event. Used to repair
output_token_ids prior to sampling, if needed by logits processors.
"""
@@ -1041,8 +1040,7 @@ class InputBatch:
self.async_copy_ready_event = None
def update_async_output_token_ids(self) -> None:
"""
In async scheduling case, update output_token_ids in sampling metadata
"""In async scheduling case, update output_token_ids in sampling metadata
from prior steps sampled token ids once they've finished copying to CPU.
This is called right before they are needed by the logits processors.
"""
@@ -1088,8 +1086,7 @@ class InputBatch:
# ^ Implicitly resizes to (first_placeholder + num_to_replace)
def update_async_spec_token_ids(self, draft_token_ids: list[list[int]]) -> None:
"""
In async scheduling case, update spec_token_ids in sampling metadata with
"""In async scheduling case, update spec_token_ids in sampling metadata with
real draft token ids from prior step. This is called right before they are
needed by the rejection sampler for penalty/bad_words computation.
"""
+77 -74
View File
@@ -481,7 +481,8 @@ class AsyncGPUPoolingModelRunnerOutput(AsyncModelRunnerOutput):
class ExecuteModelState(NamedTuple):
"""Ephemeral cached state transferred between execute_model() and
sample_tokens(), after execute_model() returns None."""
sample_tokens(), after execute_model() returns None.
"""
scheduler_output: "SchedulerOutput"
logits: torch.Tensor
@@ -1003,8 +1004,7 @@ class GPUModelRunner(
self.effective_drafter_max_model_len = self.max_model_len
def reset_mm_cache(self) -> None:
"""
Clear the multi-modal cache that was used during profiling,
"""Clear the multi-modal cache that was used during profiling,
but no longer needed during inference.
"""
if self.mm_budget:
@@ -1072,7 +1072,8 @@ class GPUModelRunner(
first scheduled token (column j is position start - 1 - j); -1 where
the position is before the prompt or already past it. Generated
positions are left to the model: under async scheduling the CPU token
table holds placeholders for them."""
table holds placeholders for them.
"""
buf = self.lookback_token_ids
assert buf is not None
buf.np.fill(-1)
@@ -1136,14 +1137,14 @@ class GPUModelRunner(
return model_kwargs
def _may_reorder_batch(self, scheduler_output: "SchedulerOutput") -> None:
"""
Update the order of requests in the batch based on the attention
"""Update the order of requests in the batch based on the attention
backend's needs. For example, some attention backends (namely MLA) may
want to separate requests based on if the attention computation will be
compute-bound or memory-bound.
Args:
scheduler_output: The scheduler output.
"""
# Attention free models have zero kv_cache_groups, however models
# like Mamba are also attention free but use the kv_cache for
@@ -1182,7 +1183,6 @@ class GPUModelRunner(
# Note: used for model runner override.
def _init_device_properties(self) -> None:
"""Initialize attributes from torch.cuda.get_device_properties"""
self.num_sms = num_compute_units(self.device.index)
# Note: used for model runner override.
@@ -1803,7 +1803,6 @@ class GPUModelRunner(
Uses self.prev_positions[:num_reqs] which maps current pos -> prev pos
(-1 for new requests).
"""
# Sync the async PP broadcast before reading sampled tokens.
if self._pp_recv_work is not None:
self._pp_recv_work.wait()
@@ -1977,9 +1976,9 @@ class GPUModelRunner(
SpecDecodeMetadata | None,
int,
]:
"""
Returns:
tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens]
"""Returns:
tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens]
"""
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
assert total_num_scheduled_tokens > 0
@@ -2310,9 +2309,9 @@ class GPUModelRunner(
cascade_attn_prefix_lens: list[list[int]] | None = None,
slot_mappings: dict[int, torch.Tensor] | None = None,
) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]:
"""
Returns:
tuple[attn_metadata, spec_decode_common_attn_metadata]
"""Returns:
tuple[attn_metadata, spec_decode_common_attn_metadata]
"""
# Attention metadata is not needed for attention free models
if len(self.kv_cache_config.kv_cache_groups) == 0:
@@ -2651,14 +2650,13 @@ class GPUModelRunner(
num_computed_tokens: np.ndarray,
num_common_prefix_blocks: list[int],
) -> list[list[int]] | None:
"""
Returns:
Optional[cascade_attn_prefix_lens]
cascade_attn_prefix_lens is 2D:
``[kv_cache_group_id][attn_group_idx]``,
None if we should not use cascade attention
"""
"""Returns:
Optional[cascade_attn_prefix_lens]
cascade_attn_prefix_lens is 2D:
``[kv_cache_group_id][attn_group_idx]``,
None if we should not use cascade attention
"""
use_cascade_attn = False
num_kv_cache_groups = len(self.kv_cache_config.kv_cache_groups)
cascade_attn_prefix_lens: list[list[int]] = [
@@ -2703,12 +2701,16 @@ class GPUModelRunner(
Args:
num_scheduled_tokens: Number of tokens scheduled per request.
num_computed_tokens: Number of tokens already computed per request.
num_common_prefix_blocks: Number of shared KV cache blocks.
kv_cache_spec: The KV cache spec of the group being considered.
attn_metadata_builder: Builder used to decide whether cascade
attention can be used.
Returns:
int: Length of common prefix in tokens.
"""
"""
common_prefix_len = num_common_prefix_blocks * kv_cache_spec.block_size
if common_prefix_len == 0:
# Common case.
@@ -2947,6 +2949,7 @@ class GPUModelRunner(
- mm_hashes: List of multimodal hashes for each item
- mm_kwargs: List of multimodal kwargs for each item
- mm_lora_refs: List of (req_id, placeholder_range) for each item
"""
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
if not scheduled_encoder_inputs:
@@ -3203,7 +3206,8 @@ class GPUModelRunner(
def _get_encoder_output_from_cache(self, mm_hash: str) -> torch.Tensor | None:
"""Return a cached encoder output for multimodal
embedding gather."""
embedding gather.
"""
return self.encoder_cache.get(mm_hash, None)
def _gather_mm_embeddings(
@@ -3404,9 +3408,7 @@ class GPUModelRunner(
)
def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None:
"""
Step for the EPLB (Expert Parallelism Load Balancing) state.
"""
"""Step for the EPLB (Expert Parallelism Load Balancing) state."""
if not self.parallel_config.enable_eplb or self.eep_eplb_suppressed:
return
@@ -3875,6 +3877,7 @@ class GPUModelRunner(
Returns:
Model output tensor
"""
return self.model(
input_ids=input_ids,
@@ -3892,8 +3895,7 @@ class GPUModelRunner(
num_reqs: int,
force_uniform_decode: bool | None = None,
) -> bool:
"""
Checks if it's a decode batch with same amount scheduled tokens
"""Checks if it's a decode batch with same amount scheduled tokens
across all requests.
"""
return (
@@ -4065,11 +4067,9 @@ class GPUModelRunner(
)
def _register_layerwise_nvtx_hooks(self) -> None:
"""
Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled
"""Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled
to trace detailed information of each layer or module in the model.
"""
if (
self.vllm_config.observability_config.enable_layerwise_nvtx_tracing
and not self.layerwise_nvtx_hooks_registered
@@ -4110,8 +4110,7 @@ class GPUModelRunner(
dict[int, torch.Tensor] | None,
dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
]:
"""
Build slot mappings in both formats needed by the system.
"""Build slot mappings in both formats needed by the system.
Args:
num_tokens_padded: Total number of tokens (padded)
@@ -4123,6 +4122,7 @@ class GPUModelRunner(
A tuple of:
- slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata
- slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext
"""
if not (
hasattr(self, "kv_cache_config")
@@ -4179,7 +4179,8 @@ class GPUModelRunner(
This is true when `discard_request_mask` is set for every scheduled
request (e.g., for chunked prefill requests that are not the last
prefill chunk)."""
prefill chunk).
"""
num_reqs = self.input_batch.num_reqs
return bool(self.discard_request_mask.np[:num_reqs].all())
@@ -5310,9 +5311,9 @@ class GPUModelRunner(
@instrument(span_name="Loading (GPU)")
def load_model(self, load_dummy_weights: bool = False) -> None:
"""
Args:
load_dummy_weights: load dummy weights instead of real weights.
"""Args:
load_dummy_weights: load dummy weights instead of real weights.
"""
logger.info_once(
"Starting to load model %s...",
@@ -5525,6 +5526,7 @@ class GPUModelRunner(
Returns:
Tuple of layer indices if found in draft model config,
None otherwise.
"""
if not (self.speculative_config and self.speculative_config.draft_model_config):
return None
@@ -5556,8 +5558,7 @@ class GPUModelRunner(
weights_path: str | None = None,
is_checkpoint_format: bool = True,
) -> None:
"""
Reload weights from a weights iterator or from disk
"""Reload weights from a weights iterator or from disk
Args:
weights_iterator: weights to load into model
@@ -5565,6 +5566,7 @@ class GPUModelRunner(
provided. Use path of original model if neither is provided.
is_checkpoint_format: set to False if weights have already been
processed into kernel format (repacking, renaming, etc.)
"""
# TODO(@kylesayrs): generalize to all runners and loaders
# argument validation
@@ -5774,13 +5776,11 @@ class GPUModelRunner(
inputs_embeds: torch.Tensor | None,
randomize_inputs: bool = False,
):
"""
Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set.
"""Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set.
This is to help balance expert-selection
- during profile_run
- during DP rank dummy run
"""
dp_size = self.vllm_config.parallel_config.data_parallel_size
randomize_inputs = randomize_inputs or (
envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
@@ -5863,8 +5863,7 @@ class GPUModelRunner(
profile_seq_lens: int | None = None,
randomize_inputs: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Run a dummy forward pass to warm up/profile run or capture the
"""Run a dummy forward pass to warm up/profile run or capture the
CUDA graph for the model.
Args:
@@ -5889,6 +5888,12 @@ class GPUModelRunner(
profile_seq_lens: If provided, use this value for seq_lens instead
of max_query_len. Used to profile attention workspace that
scales with context length.
allow_microbatching: Whether the dummy batch may be split into
microbatches.
is_graph_capturing: Whether this run is a cudagraph capture.
randomize_inputs: Whether to fill the dummy inputs with random
values rather than zeros.
"""
mm_config = self.vllm_config.model_config.multimodal_config
if mm_config and mm_config.mm_encoder_only:
@@ -6568,7 +6573,8 @@ class GPUModelRunner(
def shutdown(self) -> None:
"""Release GPU tensors (model weights, KV caches, workspace) so that
memory is reclaimable when running in the same process."""
memory is reclaimable when running in the same process.
"""
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
from vllm.v1.worker.workspace import reset_workspace_manager
@@ -7038,9 +7044,7 @@ class GPUModelRunner(
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
) -> None:
"""
Initialize the attention backends and attention metadata builders.
"""
"""Initialize the attention backends and attention metadata builders."""
assert len(self.attn_groups) == 0, "Attention backends are already initialized"
class AttentionGroupKey(NamedTuple):
@@ -7143,9 +7147,7 @@ class GPUModelRunner(
def initialize_metadata_builders(
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
) -> None:
"""
Create the metadata builders for all KV cache groups and attn groups.
"""
"""Create the metadata builders for all KV cache groups and attn groups."""
for kv_cache_group_id in range(len(kv_cache_config.kv_cache_groups)):
for attn_group in self.attn_groups[kv_cache_group_id]:
attn_group.create_metadata_builders(
@@ -7180,8 +7182,7 @@ class GPUModelRunner(
kv_cache_groups: list[KVCacheGroupSpec],
is_profiling: bool = False,
) -> None:
"""
Resolve the cudagraph_mode when there are multiple attention
"""Resolve the cudagraph_mode when there are multiple attention
groups with potential conflicting CUDA graph support.
Then initialize the cudagraph_dispatcher based on the resolved
cudagraph_mode.
@@ -7234,8 +7235,7 @@ class GPUModelRunner(
self.drafter.initialize_cudagraph_keys(cudagraph_mode)
def calculate_reorder_batch_threshold(self) -> None:
"""
Choose the minimum reorder batch threshold from all attention groups.
"""Choose the minimum reorder batch threshold from all attention groups.
Backends should be able to support lower threshold then what they request
just may have a performance penalty due to that backend treating decodes
as prefills.
@@ -7256,8 +7256,7 @@ class GPUModelRunner(
def may_reinitialize_input_batch(
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
) -> None:
"""
Re-initialize the input batch if the block sizes are different from
"""Re-initialize the input batch if the block sizes are different from
what it was originally created with. This happens when the final
block size (determined after model loading) differs from the
placeholder used during __init__, or when there are multiple
@@ -7266,6 +7265,7 @@ class GPUModelRunner(
Args:
kv_cache_config: The KV cache configuration.
kernel_block_sizes: The kernel block sizes for each KV cache group.
"""
block_sizes = []
max_num_blocks = []
@@ -7347,18 +7347,19 @@ class GPUModelRunner(
kernel_block_sizes: list[int],
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> dict[str, torch.Tensor]:
"""
Initialize the memory buffer for KV cache.
"""Initialize the memory buffer for KV cache.
Args:
kv_cache_config: The KV cache config
kernel_block_sizes: The kernel block sizes for each KV cache group.
kv_cache_allocation_context: Context manager the allocation runs
inside, e.g. to place the KV cache in a specific memory pool.
Returns:
Dict[str, torch.Tensor]: A map between layer names to their
corresponding memory buffer for KV cache.
"""
"""
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
kv_caches = allocate_kv_cache(
@@ -7388,8 +7389,7 @@ class GPUModelRunner(
def maybe_add_kv_sharing_layers_to_kv_cache_groups(
self, kv_cache_config: KVCacheConfig
) -> None:
"""
Add layers that re-use KV cache to KV cache group of its target layer.
"""Add layers that re-use KV cache to KV cache group of its target layer.
Mapping of KV cache tensors happens in `initialize_kv_cache_tensors()`
"""
if not self.shared_kv_cache_layers:
@@ -7419,11 +7419,15 @@ class GPUModelRunner(
is_profiling: bool = False,
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> None:
"""
Initialize KV cache based on `kv_cache_config`.
"""Initialize KV cache based on `kv_cache_config`.
Args:
kv_cache_config: Configuration for the KV cache, including the KV
cache size of each layer
cache size of each layer
is_profiling: Whether this call is part of a profiling run.
kv_cache_allocation_context: Context manager the allocation runs
inside, e.g. to place the KV cache in a specific memory pool.
"""
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
@@ -7530,9 +7534,7 @@ class GPUModelRunner(
self.routed_experts_initialized = True
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
"""
Add encoder-only layers to the KV cache config.
"""
"""Add encoder-only layers to the KV cache config."""
block_size = self.vllm_config.cache_config.block_size
encoder_only_attn_specs: dict[AttentionSpec, list[str]] = defaultdict(list)
attn_layers = get_layers_from_vllm_config(self.vllm_config, Attention)
@@ -7556,12 +7558,13 @@ class GPUModelRunner(
)
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
"""
Generates the KVCacheSpec by parsing the kv cache format from each
"""Generates the KVCacheSpec by parsing the kv cache format from each
Attention module in the static forward context.
Returns:
KVCacheSpec: A dictionary mapping layer names to their KV cache
format. Layers that do not need KV cache are not included.
"""
if has_ec_transfer() and not get_ec_transfer().is_consumer:
return {}
@@ -7605,11 +7608,11 @@ class GPUModelRunner(
return pinned.tolist()
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
"""
Get encoder timing stats for all requests and clear the registry.
"""Get encoder timing stats for all requests and clear the registry.
Returns:
Dictionary mapping request_id to stats dict.
"""
with self._encoder_timing_lock:
stats = {
@@ -7627,14 +7630,14 @@ class GPUModelRunner(
current_item_idx: int,
num_items: int,
):
"""
Context manager to time encoder forward operations.
"""Context manager to time encoder forward operations.
Args:
should_time: Whether timing is enabled
group_lora_refs: Full list of (request_id, pos_info) tuples
current_item_idx: Starting index for this group
num_items: Number of items in this group
"""
if not should_time:
yield
+1 -2
View File
@@ -119,8 +119,7 @@ class UBatchWrapper:
return self.runnable
def _capture_ubatches(self, ubatch_metadata, model) -> torch.Tensor:
"""
Capture a cudagraph for a microbatched run.
"""Capture a cudagraph for a microbatched run.
The logic here is somewhat complicated because we need to make sure that
each of the ubatch threads initialize the cuda context before we start
+9 -11
View File
@@ -336,7 +336,8 @@ class Worker(WorkerBase):
def _scoped_allocator_max_split(self, max_split_size_mb: int):
"""Temporarily set max_split_size_mb to reduce allocator fragmentation at the
cost of more cudaMalloc calls (negligible in practice). Restores the original
value on exit."""
value on exit.
"""
if not current_platform.is_cuda():
yield
return
@@ -536,6 +537,7 @@ class Worker(WorkerBase):
Tip:
You may limit the usage of GPU memory
by adjusting the `gpu_memory_utilization` parameter.
"""
maybe_apply_startup_plan(self)
@@ -694,7 +696,6 @@ class Worker(WorkerBase):
Returned dict is keyed by `(pp_rank, tp_rank)`.
"""
if not has_kv_transfer_group():
return None
@@ -734,7 +735,6 @@ class Worker(WorkerBase):
@instrument(span_name="Allocate KV cache")
def initialize_from_config(self, kv_cache_config: KVCacheConfig) -> None:
"""Allocate GPU KV cache with the specified kv_cache_config."""
# Update local config with adjusted num blocks after profiling,
# so that it's available to the warmup stage.
self.cache_config.num_gpu_blocks = kv_cache_config.num_blocks
@@ -1364,12 +1364,12 @@ class Worker(WorkerBase):
)
def init_weight_transfer_engine(self, init_info: dict) -> None:
"""
Initialize weight transfer mechanism.
"""Initialize weight transfer mechanism.
For NCCL backend, this creates a process group with the trainer.
Args:
init_info: Dictionary containing backend-specific initialization info
"""
self._check_weight_transfer_engine()
assert self.weight_transfer_engine is not None
@@ -1378,8 +1378,7 @@ class Worker(WorkerBase):
self.weight_transfer_engine.init_transfer_engine(typed_init_info)
def start_weight_update(self) -> None:
"""
Start a new weight update session.
"""Start a new weight update session.
Delegates engine-specific preparation (e.g. layerwise reload setup) to
the configured weight transfer engine. The worker only tracks that a
@@ -1389,8 +1388,7 @@ class Worker(WorkerBase):
self._start_weight_update()
def start_draft_weight_update(self) -> None:
"""
Like start_weight_update, but retargets the engine at the speculative
"""Like start_weight_update, but retargets the engine at the speculative
draft model for this session.
"""
with set_current_vllm_config(self.vllm_config):
@@ -1423,8 +1421,7 @@ class Worker(WorkerBase):
self._weight_update_is_draft = is_draft
def update_weights(self, update_info: dict | list[dict]) -> None:
"""
Receive one weight update chunk from the trainer.
"""Receive one weight update chunk from the trainer.
start_weight_update must be called before update_weights and
finish_weight_update must be called after all chunks have been sent.
@@ -1434,6 +1431,7 @@ class Worker(WorkerBase):
Args:
update_info: Backend-specific update info, or a list indexed by
global worker rank across data parallel replicas.
"""
self._check_weight_transfer_engine()
assert self.weight_transfer_engine is not None
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Define KV connector functionality mixin for model runners.
"""
"""Define KV connector functionality mixin for model runners."""
from collections.abc import Generator
from contextlib import AbstractContextManager, contextmanager, nullcontext
+7 -7
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Define LoRA functionality mixin for model runners.
"""
"""Define LoRA functionality mixin for model runners."""
from collections.abc import Callable
from contextlib import contextmanager
@@ -150,16 +148,17 @@ class LoRAModelRunnerMixin:
num_sampled_tokens: np.ndarray | None = None,
num_active_loras: int = 0,
):
"""
Context manager to select dummy LoRAs for capture/warmup.
"""Context manager to select dummy LoRAs for capture/warmup.
Args:
lora_config: LoRA configuration, or None if LoRA is disabled.
num_scheduled_tokens: Array of scheduled token counts per request.
mapping_type: Which LoRA mapping to build (language or encoder).
num_sampled_tokens: Array of sampled token counts per request.
num_active_loras: Number of distinct active LoRAs to use.
- 0: No LoRA active (set up zero mappings).
- >0: Use exactly this many distinct LoRAs.
"""
if num_sampled_tokens is None:
num_sampled_tokens = np.ones_like(num_scheduled_tokens, dtype=np.int32)
@@ -255,8 +254,7 @@ class LoRAModelRunnerMixin:
num_active_loras: int = 0,
mapping_type: LoRAMappingType = LoRAMappingType.LANGUAGE,
):
"""
Context manager for dummy runs with LoRA.
"""Context manager for dummy runs with LoRA.
Args:
lora_config: LoRA configuration.
@@ -265,6 +263,8 @@ class LoRAModelRunnerMixin:
remove_lora: Whether to remove LoRAs after the context exits.
num_active_loras: Number of distinct active LoRAs to use.
LoRA is activated when num_active_loras > 0.
mapping_type: Which LoRA mapping to build (language or encoder).
"""
with (
self.maybe_setup_dummy_loras(lora_config, remove_lora),
+9 -11
View File
@@ -415,8 +415,7 @@ def postprocess_mamba_fused_kernel(
# the existing 2D-grid contract.
TEMPORAL_TILES: tl.constexpr = 1,
):
"""
Fused GPU kernel for postprocess_mamba that computes decisions AND performs
"""Fused GPU kernel for postprocess_mamba that computes decisions AND performs
mamba state copies without any CPU-GPU synchronization.
Grid: (num_reqs, num_states [, TEMPORAL_TILES])
@@ -769,8 +768,7 @@ class MambaCopyBuffers:
@dataclasses.dataclass
class MambaSpecDecodeGPUContext:
"""
Context for GPU-side Mamba state copy operations during the
"""Context for GPU-side Mamba state copy operations during the
fused postprocess path.
Only used when speculative decoding is enabled on a hybrid model
@@ -927,8 +925,7 @@ class MambaSpecDecodeGPUContext:
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
block_tables: list[torch.Tensor],
) -> None:
"""
Extract and cache memory layout metadata from Mamba state tensors.
"""Extract and cache memory layout metadata from Mamba state tensors.
This method populates the pre-allocated metadata tensors with information
needed by `postprocess_mamba_fused_kernel` to perform state copies entirely
@@ -960,6 +957,7 @@ class MambaSpecDecodeGPUContext:
block_tables: per-mamba-group persistent block-table tensors, in
the same order as `mamba_group_ids`. Their `data_ptr()` /
`stride(0)` are captured once for the kernel to index into.
"""
if self.is_initialized:
return
@@ -1096,7 +1094,7 @@ class MambaSpecDecodeGPUContext:
seq_lens: torch.Tensor,
num_reqs: int,
) -> torch.Tensor:
"""compute every Mamba group's aligned physical state IDs in one launch."""
"""Compute every Mamba group's aligned physical state IDs in one launch."""
assert self.is_initialized
assert seq_lens.is_cuda
assert 0 <= num_reqs <= seq_lens.shape[0]
@@ -1137,8 +1135,7 @@ class MambaSpecDecodeGPUContext:
num_computed_tokens_gpu: torch.Tensor,
num_draft_tokens_gpu: torch.Tensor,
) -> None:
"""
Run the fused postprocess_mamba kernel on GPU.
"""Run the fused postprocess_mamba kernel on GPU.
This computes decisions and performs mamba state copies entirely on GPU,
eliminating the CPU-GPU sync that was previously needed.
@@ -1150,6 +1147,7 @@ class MambaSpecDecodeGPUContext:
num_scheduled_tokens_gpu: [num_reqs] scheduled token counts
num_computed_tokens_gpu: [num_reqs] computed token counts
num_draft_tokens_gpu: [num_reqs] draft token counts
"""
if num_reqs == 0 or not self.is_initialized:
return
@@ -1205,6 +1203,7 @@ class MambaSpecDecodeGPUContext:
token_bias_gpu: [max_reqs] accepted-token bias (num_accepted - 1).
idx_mapping: optional [num_reqs] batch_idx -> req_state_idx.
None means V1 batch order already equals request state order.
"""
if num_reqs == 0 or not self.is_initialized:
return
@@ -1442,8 +1441,7 @@ def preprocess_mamba(
copy_bufs: MambaCopyBuffers,
align_ctx: MambaSpecDecodeGPUContext | None = None,
):
"""
Copy the mamba state of previous step to the last
"""Copy the mamba state of previous step to the last
(1 + num_speculative_blocks) block.
"""
fused = _resolve_fused_precopy(align_ctx)
+6 -3
View File
@@ -77,7 +77,8 @@ def compute_plan_fingerprint(
def _plan_path(fingerprint: str) -> str:
"""Plans are regenerable derived state, so they live under the standard
vLLM cache root (like the torch.compile cache) and relocate with
``VLLM_CACHE_ROOT`` instead of needing a location knob of their own."""
``VLLM_CACHE_ROOT`` instead of needing a location knob of their own.
"""
# VLLM_CACHE_ROOT is already user-expanded by envs.py.
return os.path.join(
envs.VLLM_CACHE_ROOT, "startup_plan", f"startup_plan_{fingerprint}.json"
@@ -134,7 +135,8 @@ def _applicable_kv_cache_memory_bytes(
def maybe_apply_startup_plan(worker: "Worker") -> None:
"""If enabled and ``--kv-cache-memory`` was not set explicitly, apply a
persisted plan by setting ``worker.cache_config.kv_cache_memory_bytes``.
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``."""
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``.
"""
if (
not envs.VLLM_ENABLE_STARTUP_PLAN
or worker.cache_config.kv_cache_memory_bytes is not None
@@ -167,7 +169,8 @@ def maybe_apply_startup_plan(worker: "Worker") -> None:
def maybe_save_startup_plan(worker: "Worker", kv_cache_memory_bytes: int) -> None:
"""Atomically persist this boot's profiling result for future boots.
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``; failures are logged,
never raised."""
never raised.
"""
if not envs.VLLM_ENABLE_STARTUP_PLAN:
return
fingerprint = compute_plan_fingerprint(
+5 -10
View File
@@ -43,8 +43,7 @@ class SMControlContextManager:
set_comm_sms: Callable[[int], None],
set_compute_sms: Callable[[int], None],
):
"""
Context manager for controlling SM (Streaming Multiprocessor)
"""Context manager for controlling SM (Streaming Multiprocessor)
allocation. Upon entering the context, it sets the number of SMs
allocated for communication and computation to comm_sms and
total_sms - comm_sms respectively. Upon exiting, it restores the
@@ -57,8 +56,8 @@ class SMControlContextManager:
A function that sets the number of SMs for communication.
set_compute_sms (Callable[[int], None]):
A function that sets the number of SMs for computation.
"""
"""
assert current_platform.is_cuda() or current_platform.is_rocm(), (
"SM/CU control is supported on CUDA and ROCm platforms"
)
@@ -221,8 +220,7 @@ def slice_query_start_locs(
query_start_loc: torch.Tensor,
request_slice: slice,
) -> torch.Tensor:
"""
Creates a new query_start_loc that corresponds to the requests in
"""Creates a new query_start_loc that corresponds to the requests in
request_slice.
Note: This function creates a new tensor to hold the new query_start_locs.
@@ -237,11 +235,9 @@ def slice_query_start_locs(
def _make_metadata_with_slice(
ubatch_slice: UBatchSlice, attn_metadata: CommonAttentionMetadata
) -> CommonAttentionMetadata:
"""
This function creates a new CommonAttentionMetadata that corresponds to
"""This function creates a new CommonAttentionMetadata that corresponds to
the requests included in ubatch_slice
"""
assert not ubatch_slice.is_empty(), f"Ubatch slice {ubatch_slice} is empty"
request_slice = ubatch_slice.request_slice
@@ -338,8 +334,7 @@ def split_attn_metadata(
ubatch_slices: list[UBatchSlice],
common_attn_metadata: CommonAttentionMetadata,
) -> list[CommonAttentionMetadata]:
"""
Creates a new CommonAttentionMetadata instance that corresponds to the
"""Creates a new CommonAttentionMetadata instance that corresponds to the
requests for each UBatchSlice in ubatch_slices.
Note: This function does not modify common_attn_metadata
+1 -3
View File
@@ -18,9 +18,7 @@ _CURRENT_CONTEXTS: list["UBatchContext | None"] = []
class UBatchContext:
"""
Context manager for micro-batching synchronization using threading events.
"""
"""Context manager for micro-batching synchronization using threading events."""
def __init__(
self,
+15 -12
View File
@@ -331,8 +331,7 @@ def select_common_block_size(
kv_manager_block_size: int,
backends: list[type[AttentionBackend]],
) -> int:
"""
Select a block size that is supported by all backends and is a factor of
"""Select a block size that is supported by all backends and is a factor of
kv_manager_block_size.
If kv_manager_block_size is supported by all backends, return it directly.
@@ -347,6 +346,7 @@ def select_common_block_size(
Raises:
ValueError: If no valid block size found.
"""
def block_size_is_supported(
@@ -458,8 +458,7 @@ def allocate_kv_cache(
def prepare_kernel_block_sizes(
kv_cache_config: KVCacheConfig, attn_groups: list[list[AttentionGroup]]
) -> list[int]:
"""
Generate kernel_block_sizes that matches each block_size.
"""Generate kernel_block_sizes that matches each block_size.
For attention backends that support virtual block splitting,
use the supported block sizes from the backend.
@@ -471,6 +470,7 @@ def prepare_kernel_block_sizes(
Returns:
List of kernel block sizes for each cache group.
"""
kernel_block_sizes = []
for kv_cache_gid, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups):
@@ -505,8 +505,7 @@ def sanity_check_mm_encoder_outputs(
mm_embeddings: MultiModalEmbeddings,
expected_num_items: int,
) -> None:
"""
Perform sanity checks for the result of
"""Perform sanity checks for the result of
[`vllm.model_executor.models.SupportsMultiModal.embed_multimodal`][].
"""
assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), (
@@ -532,8 +531,7 @@ def sanity_check_mm_encoder_outputs(
def request_memory(init_snapshot: MemorySnapshot, cache_config: CacheConfig) -> int:
"""
Calculate the amount of memory required by vLLM, then validate
"""Calculate the amount of memory required by vLLM, then validate
that the current amount of free memory is sufficient for that.
"""
requested_memory = math.ceil(
@@ -559,8 +557,7 @@ def add_kv_sharing_layers_to_kv_cache_groups(
kv_cache_groups: list[KVCacheGroupSpec],
runner_only_attn_layers: set[str] | None = None,
) -> None:
"""
Sets up KV cache sharing by reusing the allocated KV caches in `kv_caches`
"""Sets up KV cache sharing by reusing the allocated KV caches in `kv_caches`
for layers that do not allocate its own KV cache, based on the mapping in
`shared_kv_cache_layers`. Adds these layers to the corresponding KV cache
group, which is needed to ensure that attention metadata is assigned later.
@@ -571,6 +568,9 @@ def add_kv_sharing_layers_to_kv_cache_groups(
means this layer will perform attention using the keys and values
from the KV cache of `shared_kv_cache_layers[layer_name]`.
kv_cache_groups: The KV cache groups of the model.
runner_only_attn_layers: Attention layers handled by the runner only,
which are excluded from the KV cache groups.
"""
if not shared_kv_cache_layers:
return
@@ -595,8 +595,7 @@ def bind_kv_cache(
num_attn_module: int = 1,
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
) -> None:
"""
Bind the allocated KV cache to both ModelRunner and forward context so
"""Bind the allocated KV cache to both ModelRunner and forward context so
that the KV cache can be used in the forward pass.
This function:
@@ -607,9 +606,13 @@ def bind_kv_cache(
Args:
kv_caches: The allocated kv_caches with layer names as keys.
num_attn_module: Number of attention modules per layer entry.
forward_context: The global forward context containing all Attention
layers with layer names as keys.
runner_kv_caches: The kv_cache declared by ModelRunner.
kv_cache_groups: The KV cache groups of the model, used to resolve
layers that share a KV cache.
"""
# Bind kv_caches to ModelRunner
assert len(runner_kv_caches) == 0
+8 -9
View File
@@ -55,8 +55,7 @@ class WorkerBase:
distributed_init_method: str,
is_driver_worker: bool = False,
) -> None:
"""
Initialize common worker components.
"""Initialize common worker components.
Args:
vllm_config: Complete vLLM configuration
@@ -65,6 +64,7 @@ class WorkerBase:
distributed_init_method: Distributed initialization method
is_driver_worker: Whether this worker handles driver
responsibilities
"""
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
@@ -118,6 +118,7 @@ class WorkerBase:
Returns:
Compilation times (language_model, encoder) in seconds.
"""
raise NotImplementedError
@@ -127,7 +128,8 @@ class WorkerBase:
def synchronize_device(self) -> None:
"""Block until in-flight device work completes; backends outside
``torch.accelerator`` must override with their own wait."""
``torch.accelerator`` must override with their own wait.
"""
if torch.accelerator.is_available():
torch.accelerator.synchronize()
@@ -209,8 +211,7 @@ class WorkerBase:
class WorkerWrapperBase:
"""
This class represents one process in an executor/engine. It is responsible
"""This class represents one process in an executor/engine. It is responsible
for lazily initializing the worker and handling the worker's lifecycle.
We first instantiate the WorkerWrapper, which remembers the worker module
and class name. Then, when we call `update_environment_variables`, and the
@@ -222,8 +223,7 @@ class WorkerWrapperBase:
rpc_rank: int = 0,
global_rank: int | None = None,
) -> None:
"""
Initialize the worker wrapper with the given vllm_config and rpc_rank.
"""Initialize the worker wrapper with the given vllm_config and rpc_rank.
Note: rpc_rank is the rank of the worker in the executor. In most cases,
it is also the rank of the worker in the distributed group. However,
when multiple executors work together, they can be different.
@@ -252,8 +252,7 @@ class WorkerWrapperBase:
@instrument(span_name="Worker init")
def init_worker(self, all_kwargs: list[dict[str, Any]]) -> None:
"""
Here we inject some common logic before initializing the worker.
"""Here we inject some common logic before initializing the worker.
Arguments are passed to the worker class constructor.
"""
kwargs = all_kwargs[self.rpc_rank]
+6
View File
@@ -123,6 +123,7 @@ class WorkspaceManager:
Returns:
List of tensor views into the workspace buffer, one per shape/dtype pair.
"""
actual_bytes = [_compute_bytes(s, d) for s, d in shapes_and_dtypes]
aligned_bytes = [round_up(actual, 256) for actual in actual_bytes]
@@ -148,6 +149,7 @@ class WorkspaceManager:
Returns:
The current workspace tensor.
"""
ubatch_id = dbo_current_ubatch_id()
lane = _workspace_lane.get()
@@ -228,6 +230,7 @@ def is_workspace_manager_initialized() -> bool:
Returns:
True if workspace manager is initialized, False otherwise.
"""
return _manager is not None
@@ -237,6 +240,7 @@ def current_workspace_manager() -> "WorkspaceManager":
Raises:
AssertionError: If workspace manager has not been initialized.
"""
assert _manager is not None, (
"WorkspaceManager not initialized. Call init_workspace_manager() "
@@ -259,6 +263,7 @@ def init_workspace_manager(
device: The device to allocate workspace on.
num_ubatches: Number of workspace ubatch slots. Defaults to 1.
num_lanes: Number of independent execution lanes per ubatch. Defaults to 1.
"""
global _manager
if _manager is not None:
@@ -289,6 +294,7 @@ def lock_workspace() -> None:
lock_workspace()
# Now all get_workspace calls must fit in pre-allocated size
"""
current_workspace_manager().lock()