Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -67,20 +67,20 @@ class BlockTable:
|
||||
cp_kv_cache_interleave_size: int,
|
||||
slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
block_size: Block size used for KV cache memory allocation
|
||||
max_num_reqs: Maximum number of concurrent requests supported.
|
||||
max_num_blocks_per_req: Maximum number of blocks per request.
|
||||
max_num_batched_tokens: Maximum number of tokens in a batch.
|
||||
pin_memory: Whether to pin memory for faster GPU transfers.
|
||||
device: Target device for the block table.
|
||||
kernel_block_size: The block_size of underlying attention kernel.
|
||||
Will be the same as `block_size` if `block_size` is supported
|
||||
by the attention kernel.
|
||||
slot_mapping_mode: How this cache group maps scheduled tokens to
|
||||
cache slots. Mamba-like state caches do not use token slot
|
||||
mappings and should use SlotMappingMode.NONE.
|
||||
"""Args:
|
||||
block_size: Block size used for KV cache memory allocation
|
||||
max_num_reqs: Maximum number of concurrent requests supported.
|
||||
max_num_blocks_per_req: Maximum number of blocks per request.
|
||||
max_num_batched_tokens: Maximum number of tokens in a batch.
|
||||
pin_memory: Whether to pin memory for faster GPU transfers.
|
||||
device: Target device for the block table.
|
||||
kernel_block_size: The block_size of underlying attention kernel.
|
||||
Will be the same as `block_size` if `block_size` is supported
|
||||
by the attention kernel.
|
||||
slot_mapping_mode: How this cache group maps scheduled tokens to
|
||||
cache slots. Mamba-like state caches do not use token slot
|
||||
mappings and should use SlotMappingMode.NONE.
|
||||
|
||||
"""
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
@@ -254,6 +254,7 @@ class BlockTable:
|
||||
# kv_manager_block_id 0 → kernel block id [0, 1]
|
||||
# kv_manager_block_id 1 → kernel block id [2, 3]
|
||||
# kv_manager_block_id 2 → kernel block id [4, 5]
|
||||
|
||||
"""
|
||||
if blocks_per_kv_block == 1:
|
||||
return kv_manager_block_ids
|
||||
|
||||
@@ -112,8 +112,7 @@ def _post_process_dp_padding(tensor: torch.Tensor, should_dp_pad: bool) -> torch
|
||||
|
||||
|
||||
def _post_process_cudagraph_mode(tensor: torch.Tensor) -> int:
|
||||
"""
|
||||
Synchronize cudagraph_mode across DP ranks by taking the minimum.
|
||||
"""Synchronize cudagraph_mode across DP ranks by taking the minimum.
|
||||
If any rank has NONE (0), all ranks use NONE.
|
||||
This ensures all ranks send consistent values (all padded or all unpadded).
|
||||
"""
|
||||
@@ -127,8 +126,7 @@ def _synchronize_dp_ranks(
|
||||
cudagraph_mode: int,
|
||||
parallel_config: ParallelConfig,
|
||||
) -> tuple[bool, torch.Tensor | None, int]:
|
||||
"""
|
||||
1. Decides if each DP rank is going to microbatch. Either all ranks
|
||||
"""1. Decides if each DP rank is going to microbatch. Either all ranks
|
||||
run with microbatching or none of them do.
|
||||
|
||||
2. Determines the total number of tokens that each rank will run.
|
||||
@@ -197,8 +195,7 @@ def coordinate_batch_across_dp(
|
||||
uniform_decode: bool | None = None,
|
||||
cudagraph_mode: int = 0,
|
||||
) -> tuple[bool, torch.Tensor | None, int]:
|
||||
"""
|
||||
Coordinates amongst all DP ranks to determine if and how the full batch
|
||||
"""Coordinates amongst all DP ranks to determine if and how the full batch
|
||||
should be split into microbatches.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Define EC connector functionality mixin for model runners.
|
||||
"""
|
||||
"""Define EC connector functionality mixin for model runners."""
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
|
||||
@@ -76,7 +76,8 @@ class EncoderCudaGraphManager:
|
||||
model: SupportsEncoderCudaGraph,
|
||||
):
|
||||
"""Initialize CUDA graph manager with provided token budgets
|
||||
and max batch size."""
|
||||
and max batch size.
|
||||
"""
|
||||
self.vllm_config = vllm_config
|
||||
self.device = device
|
||||
self.dtype = dtype
|
||||
@@ -351,6 +352,7 @@ class EncoderCudaGraphManager:
|
||||
|
||||
Returns:
|
||||
Token budget if found, None if no fitting budget.
|
||||
|
||||
"""
|
||||
budgets = budgets if budgets is not None else self.token_budgets
|
||||
for budget in budgets:
|
||||
@@ -411,8 +413,11 @@ class EncoderCudaGraphManager:
|
||||
mm_kwargs: Multimodal inputs for the batch.
|
||||
token_budget: Token budget to use.
|
||||
path: Configured encoder path.
|
||||
axis_keys: Resolved capture-axis keys for this batch.
|
||||
|
||||
Returns:
|
||||
Encoder outputs, or None if graph not captured.
|
||||
|
||||
"""
|
||||
graph_set = self._get_graph_set(path)
|
||||
num_items = len(self._get_item_specs(mm_kwargs))
|
||||
@@ -562,6 +567,7 @@ class EncoderCudaGraphManager:
|
||||
images_per_rank: Number of items per rank.
|
||||
max_output_tokens_per_rank: Max output tokens across all ranks
|
||||
(for padding during all_gather).
|
||||
|
||||
"""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
current_rank = get_tensor_model_parallel_rank()
|
||||
@@ -687,6 +693,7 @@ class EncoderCudaGraphManager:
|
||||
|
||||
Returns:
|
||||
List of encoder outputs (one per item).
|
||||
|
||||
"""
|
||||
if self.use_dp:
|
||||
per_item_out_tokens = self._get_per_item_out_tokens(mm_kwargs)
|
||||
|
||||
@@ -62,7 +62,8 @@ class AttentionState(NamedTuple):
|
||||
@dataclass(frozen=True)
|
||||
class BatchExecutionDescriptor:
|
||||
"""Describes the shape of the batch and CG mode to run; this is used to make shape
|
||||
matches between the capture and runtime."""
|
||||
matches between the capture and runtime.
|
||||
"""
|
||||
|
||||
cg_mode: CUDAGraphMode
|
||||
num_tokens: int
|
||||
@@ -90,7 +91,8 @@ def make_cudagraph_stats(
|
||||
class CreateForwardFn(Protocol):
|
||||
"""Factory that prepares inputs (OUTSIDE the graph) and returns a
|
||||
forward_fn. Called with warmup=True for the warmup pass and warmup=False
|
||||
for the captured pass."""
|
||||
for the captured pass.
|
||||
"""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
@@ -408,6 +410,8 @@ class CudaGraphManager:
|
||||
it is invoked once with warmup=True and again with warmup=False
|
||||
because attention backends may mutate or lazily initialize
|
||||
metadata during warmup.
|
||||
progress_bar_desc: Description shown on the capture progress bar.
|
||||
|
||||
"""
|
||||
with graph_capture(device=self.device), ExitStack() as stack:
|
||||
if self.ubatch_runner is not None:
|
||||
@@ -503,7 +507,6 @@ class CudaGraphManager:
|
||||
num_ubatches: int = 1,
|
||||
) -> BatchExecutionDescriptor:
|
||||
"""Find matching cudagraph descriptor from priority-ordered candidates."""
|
||||
|
||||
effective_loras = self._resolve_effective_loras(num_active_loras)
|
||||
key = (num_tokens, effective_loras)
|
||||
if self._graphs_captured and num_tokens > 0 and key in self._candidates:
|
||||
@@ -952,7 +955,8 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
||||
def _extrapolate_full_graph_memory(mem_samples: list[int], total_graphs: int) -> int:
|
||||
"""Extrapolate the total FULL capture cost from samples of the largest
|
||||
graphs. The first capture allocates the pool baseline; later graphs mostly
|
||||
reuse it, so the second sample is taken as the per-graph cost."""
|
||||
reuse it, so the second sample is taken as the per-graph cost.
|
||||
"""
|
||||
if not mem_samples:
|
||||
return 0
|
||||
first_capture = mem_samples[0]
|
||||
@@ -989,7 +993,8 @@ def _init_minimal_kv_cache_for_profiling(runner: "GPUModelRunner") -> None:
|
||||
|
||||
def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
"""Release the profiling KV cache and captured graphs while keeping model
|
||||
weights, so the real ``initialize_kv_cache`` starts from a clean slate."""
|
||||
weights, so the real ``initialize_kv_cache`` starts from a clean slate.
|
||||
"""
|
||||
torch.accelerator.synchronize()
|
||||
if hasattr(runner.model_state, "_mamba_ctx"):
|
||||
runner.model_state._mamba_ctx = None
|
||||
|
||||
@@ -55,8 +55,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
allow_ubatching: bool = False,
|
||||
uniform_decode: bool = False,
|
||||
) -> tuple[BatchExecutionDescriptor, DPSyncState | None]:
|
||||
"""
|
||||
Coordinates the batch descriptor and DP padding across all ranks.
|
||||
"""Coordinates the batch descriptor and DP padding across all ranks.
|
||||
|
||||
`parallel_config` is only needed to decide whether to microbatch, so callers
|
||||
that never do (`allow_ubatching=False`) can leave it out.
|
||||
@@ -248,6 +247,12 @@ def dispatch_cg_and_sync_dp(
|
||||
need_eager: Force `CUDAGraphMode.NONE` instead of dispatching.
|
||||
num_active_loras: Active LoRA count for this rank. Does not need
|
||||
cross-rank agreement; it never changes a bucket's token count.
|
||||
parallel_config: Only needed to decide whether to microbatch, so
|
||||
callers that never do (`allow_ubatching=False`) can leave it out.
|
||||
allow_ubatching: Whether this rank's batch may be split into
|
||||
microbatches. Agreed across ranks before it takes effect.
|
||||
uniform_decode: Whether this rank's batch is a uniform decode, used
|
||||
to pick the microbatching split.
|
||||
dp_sync: Agreement from a prior dispatch over this same batch, to reuse.
|
||||
Must come from a batch with this same padded `num_tokens` and the
|
||||
same `uniform_token_count`; `num_reqs` may differ, as neither
|
||||
@@ -257,6 +262,7 @@ def dispatch_cg_and_sync_dp(
|
||||
Returns:
|
||||
(batch_desc, sync), where `sync` is this batch's agreement for a later
|
||||
dispatch to reuse. It is None when `dp_size` is 1 or no rank has work.
|
||||
|
||||
"""
|
||||
reuse_eager = dp_sync is not None and dp_sync.eager
|
||||
|
||||
|
||||
@@ -21,8 +21,7 @@ def get_lora_capture_cases(
|
||||
lora_config: "LoRAConfig | None",
|
||||
compilation_config: "CompilationConfig",
|
||||
) -> list[int]:
|
||||
"""
|
||||
Return num_active_loras values for cudagraph capture.
|
||||
"""Return num_active_loras values for cudagraph capture.
|
||||
|
||||
When cudagraph_specialize_lora=True: powers of 2 up to max_loras, plus
|
||||
max_loras+1. When False: [0, max_loras+1]. When LoRA disabled: [0].
|
||||
|
||||
@@ -24,8 +24,7 @@ class EncoderCache:
|
||||
self.mm_features.pop(req_id, None)
|
||||
|
||||
def reset_mm_cache(self) -> None:
|
||||
"""
|
||||
Clear the multi-modal cache that was used during profiling,
|
||||
"""Clear the multi-modal cache that was used during profiling,
|
||||
but no longer needed during inference.
|
||||
"""
|
||||
# NOTE: v2 encoder cache profiling skips the multi-modal cache
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
NOTE: Coding style guide for this file:
|
||||
"""NOTE: Coding style guide for this file:
|
||||
This model runner is shared by all models: text and multimodal, generative
|
||||
and embedding, public and private. As a result, this file must only contain
|
||||
code that is common to every model. Model-specific behavior belongs in the
|
||||
@@ -2229,7 +2228,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
memory is reclaimable when running in the same process."""
|
||||
memory is reclaimable when running in the same process.
|
||||
"""
|
||||
torch.accelerator.synchronize()
|
||||
self.cudagraph_manager = None
|
||||
self.fast_prefill = None
|
||||
|
||||
@@ -144,7 +144,8 @@ class ModelState(ABC):
|
||||
) -> None:
|
||||
"""Hook run on real batches before the forward pass (after block tables
|
||||
are gathered). Used by mamba "align" prefix caching to pre-copy state
|
||||
across block boundaries. No-op by default."""
|
||||
across block boundaries. No-op by default.
|
||||
"""
|
||||
return None
|
||||
|
||||
def postprocess_state(
|
||||
|
||||
@@ -38,8 +38,8 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
"""Return a bool array of shape `[input_batch.num_reqs]` marking requests
|
||||
that produce a sampled token this step, and therefore must have that token
|
||||
(and the draft block proposed from it) propagated to the earlier PP stages.
|
||||
Returns None if no request in the batch produces a sample."""
|
||||
|
||||
Returns None if no request in the batch produces a sample.
|
||||
"""
|
||||
old_computed = input_batch.num_computed_tokens_np
|
||||
prefill_len = input_batch.prefill_len_np
|
||||
# Exclude non-final prefill chunks (they don't produce a sample).
|
||||
@@ -179,7 +179,8 @@ class PPHandler:
|
||||
|
||||
def receive(self, input_batch: InputBatch) -> bool:
|
||||
"""Returns True iff sampled tokens need to be gathered from *all*
|
||||
requests in the batch."""
|
||||
requests in the batch.
|
||||
"""
|
||||
assert not self.is_last_rank
|
||||
need_sampled_mask = compute_need_sampled_mask(input_batch)
|
||||
if need_sampled_mask is None:
|
||||
|
||||
@@ -76,7 +76,8 @@ MAX_COMPACT_SUPPORT = 2048
|
||||
|
||||
class SamplingMaskTensors(NamedTuple):
|
||||
"""Device-side masks pending async D2H: compact ids, plus the bitmask as
|
||||
the exact fallback for rows wider than ``max_num_kept``."""
|
||||
the exact fallback for rows wider than ``max_num_kept``.
|
||||
"""
|
||||
|
||||
# [num_requests, max_num_kept]
|
||||
token_ids: torch.Tensor
|
||||
|
||||
@@ -107,7 +107,8 @@ class Sampler:
|
||||
self, idx_mapping_np: np.ndarray, include_token_ids: bool = True
|
||||
) -> tuple[int, int] | None:
|
||||
"""(num_logprobs, max_per_req_token_ids) for the given requests, or
|
||||
None when none of them want logprobs."""
|
||||
None when none of them want logprobs.
|
||||
"""
|
||||
max_num_logprobs = self.sampling_states.max_num_logprobs(idx_mapping_np)
|
||||
max_token_ids = (
|
||||
self.logprob_token_ids_state.max_num_token_ids(idx_mapping_np)
|
||||
|
||||
@@ -163,6 +163,7 @@ def repeated_context_mask(
|
||||
include_prompt: Search the prompt as well as the generated tokens.
|
||||
skip_partial_context: Mark contexts containing start padding so they use
|
||||
ordinary sampling.
|
||||
|
||||
"""
|
||||
if max_history is not None and max_history < 1:
|
||||
raise ValueError("max_history must be positive or None")
|
||||
|
||||
@@ -88,9 +88,14 @@ def build_cost_tables_from_curves(
|
||||
"""Build cost tables: graph-padded below the capture limit, smooth above.
|
||||
|
||||
Args:
|
||||
draft_curve: (size, cost) samples for the draft model.
|
||||
verify_curve: (size, cost) samples for the verify model.
|
||||
max_num_reqs: Largest request count to build a table for.
|
||||
max_batch_tokens: Largest token count to build a table for.
|
||||
cudagraph_limit: Largest cudagraph-captured size. At or below it,
|
||||
execution pads up to the next captured size, so cost is a step
|
||||
function. Above it there is no padding, so cost is continuous.
|
||||
|
||||
"""
|
||||
|
||||
def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray:
|
||||
@@ -185,7 +190,8 @@ class AdaptiveVerificationManager:
|
||||
"""Dummy-run kwargs whose step timings seed the cost tables.
|
||||
|
||||
Run these inside StepTimingCollector.collect(), then hand the block's
|
||||
timings to set_initial_cost_curves."""
|
||||
timings to set_initial_cost_curves.
|
||||
"""
|
||||
max_num_tokens = self.req_states.max_num_batched_tokens
|
||||
size = self._cudagraph_limit = capture_sizes[-1] if capture_sizes else 0
|
||||
# Also profile beyond the capture limit: real steps run there
|
||||
@@ -255,7 +261,8 @@ class AdaptiveVerificationManager:
|
||||
input_batch: "InputBatch",
|
||||
) -> None:
|
||||
"""Publish this step's raw confidences for the ranking kernel and start
|
||||
copying them to the CPU, where a later step's budget reads them."""
|
||||
copying them to the CPU, where a later step's budget reads them.
|
||||
"""
|
||||
num_reqs = input_batch.num_reqs
|
||||
ready_idx = self._stale_idx ^ 1
|
||||
with gpu_sync_allowed():
|
||||
|
||||
@@ -73,8 +73,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
||||
|
||||
@property
|
||||
def advance_draft_positions(self) -> bool:
|
||||
"""
|
||||
Whether to increment positions and seq_lens between draft steps.
|
||||
"""Whether to increment positions and seq_lens between draft steps.
|
||||
|
||||
True for Eagle/standard MTP (each step produces new KV).
|
||||
False for Gemma4 MTP (Q-only, shares target KV, constant positions).
|
||||
|
||||
@@ -72,7 +72,8 @@ def _prepare_dflash_inputs_to_capture(
|
||||
|
||||
class DFlashCudaGraphManager(CudaGraphManager):
|
||||
"""DFlash CudaGraphManager for the parallel-drafting query forward,
|
||||
building its own attention metadata from scratch."""
|
||||
building its own attention metadata from scratch.
|
||||
"""
|
||||
|
||||
def capture(
|
||||
self,
|
||||
|
||||
@@ -13,7 +13,6 @@ from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config
|
||||
|
||||
def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
|
||||
"""Share when the draft has no own copy, or its copy matches the target."""
|
||||
|
||||
if not getattr(eagle, flag, False) or draft is None:
|
||||
return True
|
||||
if target is None:
|
||||
@@ -29,7 +28,8 @@ def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool:
|
||||
|
||||
def get_target_lm_head(target_model: nn.Module, target_language_model: nn.Module):
|
||||
"""The target's lm_head — from get_language_model() for
|
||||
*ForConditionalGeneration targets, else the top-level module."""
|
||||
*ForConditionalGeneration targets, else the top-level module.
|
||||
"""
|
||||
return getattr(target_language_model, "lm_head", None) or getattr(
|
||||
target_model, "lm_head", None
|
||||
)
|
||||
|
||||
@@ -268,7 +268,8 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
@property
|
||||
def attn_vllm_config(self) -> VllmConfig:
|
||||
"""Config for the draft's attention metadata builders. Overridden by
|
||||
speculators whose attention mode differs from the target's."""
|
||||
speculators whose attention mode differs from the target's.
|
||||
"""
|
||||
return self.vllm_config
|
||||
|
||||
def set_attn(
|
||||
|
||||
@@ -323,7 +323,6 @@ class InputBatch:
|
||||
"""Track add-request operations for logits processors.
|
||||
Not applicable to pooling models.
|
||||
"""
|
||||
|
||||
# Fill the next empty index if there is one.
|
||||
if (new_req_index := self.batch_update_builder.pop_removed()) is None:
|
||||
# Append to end otherwise.
|
||||
@@ -531,8 +530,8 @@ class InputBatch:
|
||||
|
||||
Returns:
|
||||
Removed request index, or `None` if `req_id` not recognized
|
||||
"""
|
||||
|
||||
"""
|
||||
req_index = self.req_id_to_index.pop(req_id, None)
|
||||
if req_index is None:
|
||||
return None
|
||||
@@ -710,6 +709,7 @@ class InputBatch:
|
||||
Returns:
|
||||
swaps: list of (from,to) swap tuples for moved requests
|
||||
empty_req_indices: indices not filled by condensation
|
||||
|
||||
"""
|
||||
num_reqs = self.num_reqs
|
||||
|
||||
@@ -835,7 +835,6 @@ class InputBatch:
|
||||
|
||||
def refresh_metadata(self):
|
||||
"""Apply any batch updates to sampling metadata."""
|
||||
|
||||
if self.is_pooling_model:
|
||||
batch_changed = self.batch_update_builder.reset()
|
||||
if batch_changed:
|
||||
@@ -1001,18 +1000,19 @@ class InputBatch:
|
||||
def make_lora_inputs(
|
||||
self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]:
|
||||
"""
|
||||
Given the num_scheduled_tokens for each request in the batch, return
|
||||
"""Given the num_scheduled_tokens for each request in the batch, return
|
||||
datastructures used to activate the current LoRAs.
|
||||
|
||||
Returns:
|
||||
1. prompt_lora_mapping: A tuple of size np.sum(num_sampled_tokens)
|
||||
where, prompt_lora_mapping[i] is the LoRA id to use for the ith
|
||||
sampled token.
|
||||
where, prompt_lora_mapping[i] is the LoRA id to use for the ith
|
||||
sampled token.
|
||||
2. token_lora_mapping: A tuple of size np.sum(num_scheduled_tokens)
|
||||
where, token_lora_mapping[i] is the LoRA id to use for ith token.
|
||||
where, token_lora_mapping[i] is the LoRA id to use for ith
|
||||
token.
|
||||
3. lora_requests: Set of relevant LoRA requests.
|
||||
"""
|
||||
|
||||
"""
|
||||
req_lora_mapping = self.request_lora_mapping[: self.num_reqs]
|
||||
prompt_lora_mapping = tuple(req_lora_mapping.repeat(num_sampled_tokens))
|
||||
token_lora_mapping = tuple(req_lora_mapping.repeat(num_scheduled_tokens))
|
||||
@@ -1028,8 +1028,7 @@ class InputBatch:
|
||||
sampled_token_ids_cpu: torch.Tensor,
|
||||
async_copy_ready_event: torch.Event,
|
||||
) -> None:
|
||||
"""
|
||||
In async scheduling case, store ref to sampled_token_ids_cpu
|
||||
"""In async scheduling case, store ref to sampled_token_ids_cpu
|
||||
tensor and corresponding copy-ready event. Used to repair
|
||||
output_token_ids prior to sampling, if needed by logits processors.
|
||||
"""
|
||||
@@ -1041,8 +1040,7 @@ class InputBatch:
|
||||
self.async_copy_ready_event = None
|
||||
|
||||
def update_async_output_token_ids(self) -> None:
|
||||
"""
|
||||
In async scheduling case, update output_token_ids in sampling metadata
|
||||
"""In async scheduling case, update output_token_ids in sampling metadata
|
||||
from prior steps sampled token ids once they've finished copying to CPU.
|
||||
This is called right before they are needed by the logits processors.
|
||||
"""
|
||||
@@ -1088,8 +1086,7 @@ class InputBatch:
|
||||
# ^ Implicitly resizes to (first_placeholder + num_to_replace)
|
||||
|
||||
def update_async_spec_token_ids(self, draft_token_ids: list[list[int]]) -> None:
|
||||
"""
|
||||
In async scheduling case, update spec_token_ids in sampling metadata with
|
||||
"""In async scheduling case, update spec_token_ids in sampling metadata with
|
||||
real draft token ids from prior step. This is called right before they are
|
||||
needed by the rejection sampler for penalty/bad_words computation.
|
||||
"""
|
||||
|
||||
@@ -481,7 +481,8 @@ class AsyncGPUPoolingModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
|
||||
class ExecuteModelState(NamedTuple):
|
||||
"""Ephemeral cached state transferred between execute_model() and
|
||||
sample_tokens(), after execute_model() returns None."""
|
||||
sample_tokens(), after execute_model() returns None.
|
||||
"""
|
||||
|
||||
scheduler_output: "SchedulerOutput"
|
||||
logits: torch.Tensor
|
||||
@@ -1003,8 +1004,7 @@ class GPUModelRunner(
|
||||
self.effective_drafter_max_model_len = self.max_model_len
|
||||
|
||||
def reset_mm_cache(self) -> None:
|
||||
"""
|
||||
Clear the multi-modal cache that was used during profiling,
|
||||
"""Clear the multi-modal cache that was used during profiling,
|
||||
but no longer needed during inference.
|
||||
"""
|
||||
if self.mm_budget:
|
||||
@@ -1072,7 +1072,8 @@ class GPUModelRunner(
|
||||
first scheduled token (column j is position start - 1 - j); -1 where
|
||||
the position is before the prompt or already past it. Generated
|
||||
positions are left to the model: under async scheduling the CPU token
|
||||
table holds placeholders for them."""
|
||||
table holds placeholders for them.
|
||||
"""
|
||||
buf = self.lookback_token_ids
|
||||
assert buf is not None
|
||||
buf.np.fill(-1)
|
||||
@@ -1136,14 +1137,14 @@ class GPUModelRunner(
|
||||
return model_kwargs
|
||||
|
||||
def _may_reorder_batch(self, scheduler_output: "SchedulerOutput") -> None:
|
||||
"""
|
||||
Update the order of requests in the batch based on the attention
|
||||
"""Update the order of requests in the batch based on the attention
|
||||
backend's needs. For example, some attention backends (namely MLA) may
|
||||
want to separate requests based on if the attention computation will be
|
||||
compute-bound or memory-bound.
|
||||
|
||||
Args:
|
||||
scheduler_output: The scheduler output.
|
||||
|
||||
"""
|
||||
# Attention free models have zero kv_cache_groups, however models
|
||||
# like Mamba are also attention free but use the kv_cache for
|
||||
@@ -1182,7 +1183,6 @@ class GPUModelRunner(
|
||||
# Note: used for model runner override.
|
||||
def _init_device_properties(self) -> None:
|
||||
"""Initialize attributes from torch.cuda.get_device_properties"""
|
||||
|
||||
self.num_sms = num_compute_units(self.device.index)
|
||||
|
||||
# Note: used for model runner override.
|
||||
@@ -1803,7 +1803,6 @@ class GPUModelRunner(
|
||||
Uses self.prev_positions[:num_reqs] which maps current pos -> prev pos
|
||||
(-1 for new requests).
|
||||
"""
|
||||
|
||||
# Sync the async PP broadcast before reading sampled tokens.
|
||||
if self._pp_recv_work is not None:
|
||||
self._pp_recv_work.wait()
|
||||
@@ -1977,9 +1976,9 @@ class GPUModelRunner(
|
||||
SpecDecodeMetadata | None,
|
||||
int,
|
||||
]:
|
||||
"""
|
||||
Returns:
|
||||
tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens]
|
||||
"""Returns:
|
||||
tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens]
|
||||
|
||||
"""
|
||||
total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
|
||||
assert total_num_scheduled_tokens > 0
|
||||
@@ -2310,9 +2309,9 @@ class GPUModelRunner(
|
||||
cascade_attn_prefix_lens: list[list[int]] | None = None,
|
||||
slot_mappings: dict[int, torch.Tensor] | None = None,
|
||||
) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]:
|
||||
"""
|
||||
Returns:
|
||||
tuple[attn_metadata, spec_decode_common_attn_metadata]
|
||||
"""Returns:
|
||||
tuple[attn_metadata, spec_decode_common_attn_metadata]
|
||||
|
||||
"""
|
||||
# Attention metadata is not needed for attention free models
|
||||
if len(self.kv_cache_config.kv_cache_groups) == 0:
|
||||
@@ -2651,14 +2650,13 @@ class GPUModelRunner(
|
||||
num_computed_tokens: np.ndarray,
|
||||
num_common_prefix_blocks: list[int],
|
||||
) -> list[list[int]] | None:
|
||||
"""
|
||||
Returns:
|
||||
Optional[cascade_attn_prefix_lens]
|
||||
cascade_attn_prefix_lens is 2D:
|
||||
``[kv_cache_group_id][attn_group_idx]``,
|
||||
None if we should not use cascade attention
|
||||
"""
|
||||
"""Returns:
|
||||
Optional[cascade_attn_prefix_lens]
|
||||
cascade_attn_prefix_lens is 2D:
|
||||
``[kv_cache_group_id][attn_group_idx]``,
|
||||
None if we should not use cascade attention
|
||||
|
||||
"""
|
||||
use_cascade_attn = False
|
||||
num_kv_cache_groups = len(self.kv_cache_config.kv_cache_groups)
|
||||
cascade_attn_prefix_lens: list[list[int]] = [
|
||||
@@ -2703,12 +2701,16 @@ class GPUModelRunner(
|
||||
|
||||
Args:
|
||||
num_scheduled_tokens: Number of tokens scheduled per request.
|
||||
num_computed_tokens: Number of tokens already computed per request.
|
||||
num_common_prefix_blocks: Number of shared KV cache blocks.
|
||||
kv_cache_spec: The KV cache spec of the group being considered.
|
||||
attn_metadata_builder: Builder used to decide whether cascade
|
||||
attention can be used.
|
||||
|
||||
Returns:
|
||||
int: Length of common prefix in tokens.
|
||||
"""
|
||||
|
||||
"""
|
||||
common_prefix_len = num_common_prefix_blocks * kv_cache_spec.block_size
|
||||
if common_prefix_len == 0:
|
||||
# Common case.
|
||||
@@ -2947,6 +2949,7 @@ class GPUModelRunner(
|
||||
- mm_hashes: List of multimodal hashes for each item
|
||||
- mm_kwargs: List of multimodal kwargs for each item
|
||||
- mm_lora_refs: List of (req_id, placeholder_range) for each item
|
||||
|
||||
"""
|
||||
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
|
||||
if not scheduled_encoder_inputs:
|
||||
@@ -3203,7 +3206,8 @@ class GPUModelRunner(
|
||||
|
||||
def _get_encoder_output_from_cache(self, mm_hash: str) -> torch.Tensor | None:
|
||||
"""Return a cached encoder output for multimodal
|
||||
embedding gather."""
|
||||
embedding gather.
|
||||
"""
|
||||
return self.encoder_cache.get(mm_hash, None)
|
||||
|
||||
def _gather_mm_embeddings(
|
||||
@@ -3404,9 +3408,7 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None:
|
||||
"""
|
||||
Step for the EPLB (Expert Parallelism Load Balancing) state.
|
||||
"""
|
||||
"""Step for the EPLB (Expert Parallelism Load Balancing) state."""
|
||||
if not self.parallel_config.enable_eplb or self.eep_eplb_suppressed:
|
||||
return
|
||||
|
||||
@@ -3875,6 +3877,7 @@ class GPUModelRunner(
|
||||
|
||||
Returns:
|
||||
Model output tensor
|
||||
|
||||
"""
|
||||
return self.model(
|
||||
input_ids=input_ids,
|
||||
@@ -3892,8 +3895,7 @@ class GPUModelRunner(
|
||||
num_reqs: int,
|
||||
force_uniform_decode: bool | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
Checks if it's a decode batch with same amount scheduled tokens
|
||||
"""Checks if it's a decode batch with same amount scheduled tokens
|
||||
across all requests.
|
||||
"""
|
||||
return (
|
||||
@@ -4065,11 +4067,9 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
def _register_layerwise_nvtx_hooks(self) -> None:
|
||||
"""
|
||||
Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled
|
||||
"""Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled
|
||||
to trace detailed information of each layer or module in the model.
|
||||
"""
|
||||
|
||||
if (
|
||||
self.vllm_config.observability_config.enable_layerwise_nvtx_tracing
|
||||
and not self.layerwise_nvtx_hooks_registered
|
||||
@@ -4110,8 +4110,7 @@ class GPUModelRunner(
|
||||
dict[int, torch.Tensor] | None,
|
||||
dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None,
|
||||
]:
|
||||
"""
|
||||
Build slot mappings in both formats needed by the system.
|
||||
"""Build slot mappings in both formats needed by the system.
|
||||
|
||||
Args:
|
||||
num_tokens_padded: Total number of tokens (padded)
|
||||
@@ -4123,6 +4122,7 @@ class GPUModelRunner(
|
||||
A tuple of:
|
||||
- slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata
|
||||
- slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext
|
||||
|
||||
"""
|
||||
if not (
|
||||
hasattr(self, "kv_cache_config")
|
||||
@@ -4179,7 +4179,8 @@ class GPUModelRunner(
|
||||
|
||||
This is true when `discard_request_mask` is set for every scheduled
|
||||
request (e.g., for chunked prefill requests that are not the last
|
||||
prefill chunk)."""
|
||||
prefill chunk).
|
||||
"""
|
||||
num_reqs = self.input_batch.num_reqs
|
||||
return bool(self.discard_request_mask.np[:num_reqs].all())
|
||||
|
||||
@@ -5310,9 +5311,9 @@ class GPUModelRunner(
|
||||
|
||||
@instrument(span_name="Loading (GPU)")
|
||||
def load_model(self, load_dummy_weights: bool = False) -> None:
|
||||
"""
|
||||
Args:
|
||||
load_dummy_weights: load dummy weights instead of real weights.
|
||||
"""Args:
|
||||
load_dummy_weights: load dummy weights instead of real weights.
|
||||
|
||||
"""
|
||||
logger.info_once(
|
||||
"Starting to load model %s...",
|
||||
@@ -5525,6 +5526,7 @@ class GPUModelRunner(
|
||||
Returns:
|
||||
Tuple of layer indices if found in draft model config,
|
||||
None otherwise.
|
||||
|
||||
"""
|
||||
if not (self.speculative_config and self.speculative_config.draft_model_config):
|
||||
return None
|
||||
@@ -5556,8 +5558,7 @@ class GPUModelRunner(
|
||||
weights_path: str | None = None,
|
||||
is_checkpoint_format: bool = True,
|
||||
) -> None:
|
||||
"""
|
||||
Reload weights from a weights iterator or from disk
|
||||
"""Reload weights from a weights iterator or from disk
|
||||
|
||||
Args:
|
||||
weights_iterator: weights to load into model
|
||||
@@ -5565,6 +5566,7 @@ class GPUModelRunner(
|
||||
provided. Use path of original model if neither is provided.
|
||||
is_checkpoint_format: set to False if weights have already been
|
||||
processed into kernel format (repacking, renaming, etc.)
|
||||
|
||||
"""
|
||||
# TODO(@kylesayrs): generalize to all runners and loaders
|
||||
# argument validation
|
||||
@@ -5774,13 +5776,11 @@ class GPUModelRunner(
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
randomize_inputs: bool = False,
|
||||
):
|
||||
"""
|
||||
Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set.
|
||||
"""Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set.
|
||||
This is to help balance expert-selection
|
||||
- during profile_run
|
||||
- during DP rank dummy run
|
||||
"""
|
||||
|
||||
dp_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
randomize_inputs = randomize_inputs or (
|
||||
envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1
|
||||
@@ -5863,8 +5863,7 @@ class GPUModelRunner(
|
||||
profile_seq_lens: int | None = None,
|
||||
randomize_inputs: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Run a dummy forward pass to warm up/profile run or capture the
|
||||
"""Run a dummy forward pass to warm up/profile run or capture the
|
||||
CUDA graph for the model.
|
||||
|
||||
Args:
|
||||
@@ -5889,6 +5888,12 @@ class GPUModelRunner(
|
||||
profile_seq_lens: If provided, use this value for seq_lens instead
|
||||
of max_query_len. Used to profile attention workspace that
|
||||
scales with context length.
|
||||
allow_microbatching: Whether the dummy batch may be split into
|
||||
microbatches.
|
||||
is_graph_capturing: Whether this run is a cudagraph capture.
|
||||
randomize_inputs: Whether to fill the dummy inputs with random
|
||||
values rather than zeros.
|
||||
|
||||
"""
|
||||
mm_config = self.vllm_config.model_config.multimodal_config
|
||||
if mm_config and mm_config.mm_encoder_only:
|
||||
@@ -6568,7 +6573,8 @@ class GPUModelRunner(
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
memory is reclaimable when running in the same process."""
|
||||
memory is reclaimable when running in the same process.
|
||||
"""
|
||||
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
|
||||
from vllm.v1.worker.workspace import reset_workspace_manager
|
||||
|
||||
@@ -7038,9 +7044,7 @@ class GPUModelRunner(
|
||||
kv_cache_config: KVCacheConfig,
|
||||
is_profiling: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the attention backends and attention metadata builders.
|
||||
"""
|
||||
"""Initialize the attention backends and attention metadata builders."""
|
||||
assert len(self.attn_groups) == 0, "Attention backends are already initialized"
|
||||
|
||||
class AttentionGroupKey(NamedTuple):
|
||||
@@ -7143,9 +7147,7 @@ class GPUModelRunner(
|
||||
def initialize_metadata_builders(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
) -> None:
|
||||
"""
|
||||
Create the metadata builders for all KV cache groups and attn groups.
|
||||
"""
|
||||
"""Create the metadata builders for all KV cache groups and attn groups."""
|
||||
for kv_cache_group_id in range(len(kv_cache_config.kv_cache_groups)):
|
||||
for attn_group in self.attn_groups[kv_cache_group_id]:
|
||||
attn_group.create_metadata_builders(
|
||||
@@ -7180,8 +7182,7 @@ class GPUModelRunner(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
is_profiling: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Resolve the cudagraph_mode when there are multiple attention
|
||||
"""Resolve the cudagraph_mode when there are multiple attention
|
||||
groups with potential conflicting CUDA graph support.
|
||||
Then initialize the cudagraph_dispatcher based on the resolved
|
||||
cudagraph_mode.
|
||||
@@ -7234,8 +7235,7 @@ class GPUModelRunner(
|
||||
self.drafter.initialize_cudagraph_keys(cudagraph_mode)
|
||||
|
||||
def calculate_reorder_batch_threshold(self) -> None:
|
||||
"""
|
||||
Choose the minimum reorder batch threshold from all attention groups.
|
||||
"""Choose the minimum reorder batch threshold from all attention groups.
|
||||
Backends should be able to support lower threshold then what they request
|
||||
just may have a performance penalty due to that backend treating decodes
|
||||
as prefills.
|
||||
@@ -7256,8 +7256,7 @@ class GPUModelRunner(
|
||||
def may_reinitialize_input_batch(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
) -> None:
|
||||
"""
|
||||
Re-initialize the input batch if the block sizes are different from
|
||||
"""Re-initialize the input batch if the block sizes are different from
|
||||
what it was originally created with. This happens when the final
|
||||
block size (determined after model loading) differs from the
|
||||
placeholder used during __init__, or when there are multiple
|
||||
@@ -7266,6 +7265,7 @@ class GPUModelRunner(
|
||||
Args:
|
||||
kv_cache_config: The KV cache configuration.
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
|
||||
"""
|
||||
block_sizes = []
|
||||
max_num_blocks = []
|
||||
@@ -7347,18 +7347,19 @@ class GPUModelRunner(
|
||||
kernel_block_sizes: list[int],
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Initialize the memory buffer for KV cache.
|
||||
"""Initialize the memory buffer for KV cache.
|
||||
|
||||
Args:
|
||||
kv_cache_config: The KV cache config
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
kv_cache_allocation_context: Context manager the allocation runs
|
||||
inside, e.g. to place the KV cache in a specific memory pool.
|
||||
|
||||
Returns:
|
||||
Dict[str, torch.Tensor]: A map between layer names to their
|
||||
corresponding memory buffer for KV cache.
|
||||
"""
|
||||
|
||||
"""
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
kv_caches = allocate_kv_cache(
|
||||
@@ -7388,8 +7389,7 @@ class GPUModelRunner(
|
||||
def maybe_add_kv_sharing_layers_to_kv_cache_groups(
|
||||
self, kv_cache_config: KVCacheConfig
|
||||
) -> None:
|
||||
"""
|
||||
Add layers that re-use KV cache to KV cache group of its target layer.
|
||||
"""Add layers that re-use KV cache to KV cache group of its target layer.
|
||||
Mapping of KV cache tensors happens in `initialize_kv_cache_tensors()`
|
||||
"""
|
||||
if not self.shared_kv_cache_layers:
|
||||
@@ -7419,11 +7419,15 @@ class GPUModelRunner(
|
||||
is_profiling: bool = False,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize KV cache based on `kv_cache_config`.
|
||||
"""Initialize KV cache based on `kv_cache_config`.
|
||||
|
||||
Args:
|
||||
kv_cache_config: Configuration for the KV cache, including the KV
|
||||
cache size of each layer
|
||||
cache size of each layer
|
||||
is_profiling: Whether this call is part of a profiling run.
|
||||
kv_cache_allocation_context: Context manager the allocation runs
|
||||
inside, e.g. to place the KV cache in a specific memory pool.
|
||||
|
||||
"""
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -7530,9 +7534,7 @@ class GPUModelRunner(
|
||||
self.routed_experts_initialized = True
|
||||
|
||||
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
|
||||
"""
|
||||
Add encoder-only layers to the KV cache config.
|
||||
"""
|
||||
"""Add encoder-only layers to the KV cache config."""
|
||||
block_size = self.vllm_config.cache_config.block_size
|
||||
encoder_only_attn_specs: dict[AttentionSpec, list[str]] = defaultdict(list)
|
||||
attn_layers = get_layers_from_vllm_config(self.vllm_config, Attention)
|
||||
@@ -7556,12 +7558,13 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
|
||||
"""
|
||||
Generates the KVCacheSpec by parsing the kv cache format from each
|
||||
"""Generates the KVCacheSpec by parsing the kv cache format from each
|
||||
Attention module in the static forward context.
|
||||
|
||||
Returns:
|
||||
KVCacheSpec: A dictionary mapping layer names to their KV cache
|
||||
format. Layers that do not need KV cache are not included.
|
||||
|
||||
"""
|
||||
if has_ec_transfer() and not get_ec_transfer().is_consumer:
|
||||
return {}
|
||||
@@ -7605,11 +7608,11 @@ class GPUModelRunner(
|
||||
return pinned.tolist()
|
||||
|
||||
def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]:
|
||||
"""
|
||||
Get encoder timing stats for all requests and clear the registry.
|
||||
"""Get encoder timing stats for all requests and clear the registry.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping request_id to stats dict.
|
||||
|
||||
"""
|
||||
with self._encoder_timing_lock:
|
||||
stats = {
|
||||
@@ -7627,14 +7630,14 @@ class GPUModelRunner(
|
||||
current_item_idx: int,
|
||||
num_items: int,
|
||||
):
|
||||
"""
|
||||
Context manager to time encoder forward operations.
|
||||
"""Context manager to time encoder forward operations.
|
||||
|
||||
Args:
|
||||
should_time: Whether timing is enabled
|
||||
group_lora_refs: Full list of (request_id, pos_info) tuples
|
||||
current_item_idx: Starting index for this group
|
||||
num_items: Number of items in this group
|
||||
|
||||
"""
|
||||
if not should_time:
|
||||
yield
|
||||
|
||||
@@ -119,8 +119,7 @@ class UBatchWrapper:
|
||||
return self.runnable
|
||||
|
||||
def _capture_ubatches(self, ubatch_metadata, model) -> torch.Tensor:
|
||||
"""
|
||||
Capture a cudagraph for a microbatched run.
|
||||
"""Capture a cudagraph for a microbatched run.
|
||||
|
||||
The logic here is somewhat complicated because we need to make sure that
|
||||
each of the ubatch threads initialize the cuda context before we start
|
||||
|
||||
@@ -336,7 +336,8 @@ class Worker(WorkerBase):
|
||||
def _scoped_allocator_max_split(self, max_split_size_mb: int):
|
||||
"""Temporarily set max_split_size_mb to reduce allocator fragmentation at the
|
||||
cost of more cudaMalloc calls (negligible in practice). Restores the original
|
||||
value on exit."""
|
||||
value on exit.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
yield
|
||||
return
|
||||
@@ -536,6 +537,7 @@ class Worker(WorkerBase):
|
||||
Tip:
|
||||
You may limit the usage of GPU memory
|
||||
by adjusting the `gpu_memory_utilization` parameter.
|
||||
|
||||
"""
|
||||
maybe_apply_startup_plan(self)
|
||||
|
||||
@@ -694,7 +696,6 @@ class Worker(WorkerBase):
|
||||
|
||||
Returned dict is keyed by `(pp_rank, tp_rank)`.
|
||||
"""
|
||||
|
||||
if not has_kv_transfer_group():
|
||||
return None
|
||||
|
||||
@@ -734,7 +735,6 @@ class Worker(WorkerBase):
|
||||
@instrument(span_name="Allocate KV cache")
|
||||
def initialize_from_config(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Allocate GPU KV cache with the specified kv_cache_config."""
|
||||
|
||||
# Update local config with adjusted num blocks after profiling,
|
||||
# so that it's available to the warmup stage.
|
||||
self.cache_config.num_gpu_blocks = kv_cache_config.num_blocks
|
||||
@@ -1364,12 +1364,12 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
def init_weight_transfer_engine(self, init_info: dict) -> None:
|
||||
"""
|
||||
Initialize weight transfer mechanism.
|
||||
"""Initialize weight transfer mechanism.
|
||||
For NCCL backend, this creates a process group with the trainer.
|
||||
|
||||
Args:
|
||||
init_info: Dictionary containing backend-specific initialization info
|
||||
|
||||
"""
|
||||
self._check_weight_transfer_engine()
|
||||
assert self.weight_transfer_engine is not None
|
||||
@@ -1378,8 +1378,7 @@ class Worker(WorkerBase):
|
||||
self.weight_transfer_engine.init_transfer_engine(typed_init_info)
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""
|
||||
Start a new weight update session.
|
||||
"""Start a new weight update session.
|
||||
|
||||
Delegates engine-specific preparation (e.g. layerwise reload setup) to
|
||||
the configured weight transfer engine. The worker only tracks that a
|
||||
@@ -1389,8 +1388,7 @@ class Worker(WorkerBase):
|
||||
self._start_weight_update()
|
||||
|
||||
def start_draft_weight_update(self) -> None:
|
||||
"""
|
||||
Like start_weight_update, but retargets the engine at the speculative
|
||||
"""Like start_weight_update, but retargets the engine at the speculative
|
||||
draft model for this session.
|
||||
"""
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
@@ -1423,8 +1421,7 @@ class Worker(WorkerBase):
|
||||
self._weight_update_is_draft = is_draft
|
||||
|
||||
def update_weights(self, update_info: dict | list[dict]) -> None:
|
||||
"""
|
||||
Receive one weight update chunk from the trainer.
|
||||
"""Receive one weight update chunk from the trainer.
|
||||
|
||||
start_weight_update must be called before update_weights and
|
||||
finish_weight_update must be called after all chunks have been sent.
|
||||
@@ -1434,6 +1431,7 @@ class Worker(WorkerBase):
|
||||
Args:
|
||||
update_info: Backend-specific update info, or a list indexed by
|
||||
global worker rank across data parallel replicas.
|
||||
|
||||
"""
|
||||
self._check_weight_transfer_engine()
|
||||
assert self.weight_transfer_engine is not None
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Define KV connector functionality mixin for model runners.
|
||||
"""
|
||||
"""Define KV connector functionality mixin for model runners."""
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Define LoRA functionality mixin for model runners.
|
||||
"""
|
||||
"""Define LoRA functionality mixin for model runners."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
@@ -150,16 +148,17 @@ class LoRAModelRunnerMixin:
|
||||
num_sampled_tokens: np.ndarray | None = None,
|
||||
num_active_loras: int = 0,
|
||||
):
|
||||
"""
|
||||
Context manager to select dummy LoRAs for capture/warmup.
|
||||
"""Context manager to select dummy LoRAs for capture/warmup.
|
||||
|
||||
Args:
|
||||
lora_config: LoRA configuration, or None if LoRA is disabled.
|
||||
num_scheduled_tokens: Array of scheduled token counts per request.
|
||||
mapping_type: Which LoRA mapping to build (language or encoder).
|
||||
num_sampled_tokens: Array of sampled token counts per request.
|
||||
num_active_loras: Number of distinct active LoRAs to use.
|
||||
- 0: No LoRA active (set up zero mappings).
|
||||
- >0: Use exactly this many distinct LoRAs.
|
||||
|
||||
"""
|
||||
if num_sampled_tokens is None:
|
||||
num_sampled_tokens = np.ones_like(num_scheduled_tokens, dtype=np.int32)
|
||||
@@ -255,8 +254,7 @@ class LoRAModelRunnerMixin:
|
||||
num_active_loras: int = 0,
|
||||
mapping_type: LoRAMappingType = LoRAMappingType.LANGUAGE,
|
||||
):
|
||||
"""
|
||||
Context manager for dummy runs with LoRA.
|
||||
"""Context manager for dummy runs with LoRA.
|
||||
|
||||
Args:
|
||||
lora_config: LoRA configuration.
|
||||
@@ -265,6 +263,8 @@ class LoRAModelRunnerMixin:
|
||||
remove_lora: Whether to remove LoRAs after the context exits.
|
||||
num_active_loras: Number of distinct active LoRAs to use.
|
||||
LoRA is activated when num_active_loras > 0.
|
||||
mapping_type: Which LoRA mapping to build (language or encoder).
|
||||
|
||||
"""
|
||||
with (
|
||||
self.maybe_setup_dummy_loras(lora_config, remove_lora),
|
||||
|
||||
@@ -415,8 +415,7 @@ def postprocess_mamba_fused_kernel(
|
||||
# the existing 2D-grid contract.
|
||||
TEMPORAL_TILES: tl.constexpr = 1,
|
||||
):
|
||||
"""
|
||||
Fused GPU kernel for postprocess_mamba that computes decisions AND performs
|
||||
"""Fused GPU kernel for postprocess_mamba that computes decisions AND performs
|
||||
mamba state copies without any CPU-GPU synchronization.
|
||||
|
||||
Grid: (num_reqs, num_states [, TEMPORAL_TILES])
|
||||
@@ -769,8 +768,7 @@ class MambaCopyBuffers:
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MambaSpecDecodeGPUContext:
|
||||
"""
|
||||
Context for GPU-side Mamba state copy operations during the
|
||||
"""Context for GPU-side Mamba state copy operations during the
|
||||
fused postprocess path.
|
||||
|
||||
Only used when speculative decoding is enabled on a hybrid model
|
||||
@@ -927,8 +925,7 @@ class MambaSpecDecodeGPUContext:
|
||||
mamba_state_copy_funcs: MambaStateCopyFuncsByType,
|
||||
block_tables: list[torch.Tensor],
|
||||
) -> None:
|
||||
"""
|
||||
Extract and cache memory layout metadata from Mamba state tensors.
|
||||
"""Extract and cache memory layout metadata from Mamba state tensors.
|
||||
|
||||
This method populates the pre-allocated metadata tensors with information
|
||||
needed by `postprocess_mamba_fused_kernel` to perform state copies entirely
|
||||
@@ -960,6 +957,7 @@ class MambaSpecDecodeGPUContext:
|
||||
block_tables: per-mamba-group persistent block-table tensors, in
|
||||
the same order as `mamba_group_ids`. Their `data_ptr()` /
|
||||
`stride(0)` are captured once for the kernel to index into.
|
||||
|
||||
"""
|
||||
if self.is_initialized:
|
||||
return
|
||||
@@ -1096,7 +1094,7 @@ class MambaSpecDecodeGPUContext:
|
||||
seq_lens: torch.Tensor,
|
||||
num_reqs: int,
|
||||
) -> torch.Tensor:
|
||||
"""compute every Mamba group's aligned physical state IDs in one launch."""
|
||||
"""Compute every Mamba group's aligned physical state IDs in one launch."""
|
||||
assert self.is_initialized
|
||||
assert seq_lens.is_cuda
|
||||
assert 0 <= num_reqs <= seq_lens.shape[0]
|
||||
@@ -1137,8 +1135,7 @@ class MambaSpecDecodeGPUContext:
|
||||
num_computed_tokens_gpu: torch.Tensor,
|
||||
num_draft_tokens_gpu: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Run the fused postprocess_mamba kernel on GPU.
|
||||
"""Run the fused postprocess_mamba kernel on GPU.
|
||||
|
||||
This computes decisions and performs mamba state copies entirely on GPU,
|
||||
eliminating the CPU-GPU sync that was previously needed.
|
||||
@@ -1150,6 +1147,7 @@ class MambaSpecDecodeGPUContext:
|
||||
num_scheduled_tokens_gpu: [num_reqs] scheduled token counts
|
||||
num_computed_tokens_gpu: [num_reqs] computed token counts
|
||||
num_draft_tokens_gpu: [num_reqs] draft token counts
|
||||
|
||||
"""
|
||||
if num_reqs == 0 or not self.is_initialized:
|
||||
return
|
||||
@@ -1205,6 +1203,7 @@ class MambaSpecDecodeGPUContext:
|
||||
token_bias_gpu: [max_reqs] accepted-token bias (num_accepted - 1).
|
||||
idx_mapping: optional [num_reqs] batch_idx -> req_state_idx.
|
||||
None means V1 batch order already equals request state order.
|
||||
|
||||
"""
|
||||
if num_reqs == 0 or not self.is_initialized:
|
||||
return
|
||||
@@ -1442,8 +1441,7 @@ def preprocess_mamba(
|
||||
copy_bufs: MambaCopyBuffers,
|
||||
align_ctx: MambaSpecDecodeGPUContext | None = None,
|
||||
):
|
||||
"""
|
||||
Copy the mamba state of previous step to the last
|
||||
"""Copy the mamba state of previous step to the last
|
||||
(1 + num_speculative_blocks) block.
|
||||
"""
|
||||
fused = _resolve_fused_precopy(align_ctx)
|
||||
|
||||
@@ -77,7 +77,8 @@ def compute_plan_fingerprint(
|
||||
def _plan_path(fingerprint: str) -> str:
|
||||
"""Plans are regenerable derived state, so they live under the standard
|
||||
vLLM cache root (like the torch.compile cache) and relocate with
|
||||
``VLLM_CACHE_ROOT`` instead of needing a location knob of their own."""
|
||||
``VLLM_CACHE_ROOT`` instead of needing a location knob of their own.
|
||||
"""
|
||||
# VLLM_CACHE_ROOT is already user-expanded by envs.py.
|
||||
return os.path.join(
|
||||
envs.VLLM_CACHE_ROOT, "startup_plan", f"startup_plan_{fingerprint}.json"
|
||||
@@ -134,7 +135,8 @@ def _applicable_kv_cache_memory_bytes(
|
||||
def maybe_apply_startup_plan(worker: "Worker") -> None:
|
||||
"""If enabled and ``--kv-cache-memory`` was not set explicitly, apply a
|
||||
persisted plan by setting ``worker.cache_config.kv_cache_memory_bytes``.
|
||||
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``."""
|
||||
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``.
|
||||
"""
|
||||
if (
|
||||
not envs.VLLM_ENABLE_STARTUP_PLAN
|
||||
or worker.cache_config.kv_cache_memory_bytes is not None
|
||||
@@ -167,7 +169,8 @@ def maybe_apply_startup_plan(worker: "Worker") -> None:
|
||||
def maybe_save_startup_plan(worker: "Worker", kv_cache_memory_bytes: int) -> None:
|
||||
"""Atomically persist this boot's profiling result for future boots.
|
||||
No-op unless ``VLLM_ENABLE_STARTUP_PLAN=1``; failures are logged,
|
||||
never raised."""
|
||||
never raised.
|
||||
"""
|
||||
if not envs.VLLM_ENABLE_STARTUP_PLAN:
|
||||
return
|
||||
fingerprint = compute_plan_fingerprint(
|
||||
|
||||
@@ -43,8 +43,7 @@ class SMControlContextManager:
|
||||
set_comm_sms: Callable[[int], None],
|
||||
set_compute_sms: Callable[[int], None],
|
||||
):
|
||||
"""
|
||||
Context manager for controlling SM (Streaming Multiprocessor)
|
||||
"""Context manager for controlling SM (Streaming Multiprocessor)
|
||||
allocation. Upon entering the context, it sets the number of SMs
|
||||
allocated for communication and computation to comm_sms and
|
||||
total_sms - comm_sms respectively. Upon exiting, it restores the
|
||||
@@ -57,8 +56,8 @@ class SMControlContextManager:
|
||||
A function that sets the number of SMs for communication.
|
||||
set_compute_sms (Callable[[int], None]):
|
||||
A function that sets the number of SMs for computation.
|
||||
"""
|
||||
|
||||
"""
|
||||
assert current_platform.is_cuda() or current_platform.is_rocm(), (
|
||||
"SM/CU control is supported on CUDA and ROCm platforms"
|
||||
)
|
||||
@@ -221,8 +220,7 @@ def slice_query_start_locs(
|
||||
query_start_loc: torch.Tensor,
|
||||
request_slice: slice,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Creates a new query_start_loc that corresponds to the requests in
|
||||
"""Creates a new query_start_loc that corresponds to the requests in
|
||||
request_slice.
|
||||
|
||||
Note: This function creates a new tensor to hold the new query_start_locs.
|
||||
@@ -237,11 +235,9 @@ def slice_query_start_locs(
|
||||
def _make_metadata_with_slice(
|
||||
ubatch_slice: UBatchSlice, attn_metadata: CommonAttentionMetadata
|
||||
) -> CommonAttentionMetadata:
|
||||
"""
|
||||
This function creates a new CommonAttentionMetadata that corresponds to
|
||||
"""This function creates a new CommonAttentionMetadata that corresponds to
|
||||
the requests included in ubatch_slice
|
||||
"""
|
||||
|
||||
assert not ubatch_slice.is_empty(), f"Ubatch slice {ubatch_slice} is empty"
|
||||
|
||||
request_slice = ubatch_slice.request_slice
|
||||
@@ -338,8 +334,7 @@ def split_attn_metadata(
|
||||
ubatch_slices: list[UBatchSlice],
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> list[CommonAttentionMetadata]:
|
||||
"""
|
||||
Creates a new CommonAttentionMetadata instance that corresponds to the
|
||||
"""Creates a new CommonAttentionMetadata instance that corresponds to the
|
||||
requests for each UBatchSlice in ubatch_slices.
|
||||
|
||||
Note: This function does not modify common_attn_metadata
|
||||
|
||||
@@ -18,9 +18,7 @@ _CURRENT_CONTEXTS: list["UBatchContext | None"] = []
|
||||
|
||||
|
||||
class UBatchContext:
|
||||
"""
|
||||
Context manager for micro-batching synchronization using threading events.
|
||||
"""
|
||||
"""Context manager for micro-batching synchronization using threading events."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
+15
-12
@@ -331,8 +331,7 @@ def select_common_block_size(
|
||||
kv_manager_block_size: int,
|
||||
backends: list[type[AttentionBackend]],
|
||||
) -> int:
|
||||
"""
|
||||
Select a block size that is supported by all backends and is a factor of
|
||||
"""Select a block size that is supported by all backends and is a factor of
|
||||
kv_manager_block_size.
|
||||
|
||||
If kv_manager_block_size is supported by all backends, return it directly.
|
||||
@@ -347,6 +346,7 @@ def select_common_block_size(
|
||||
|
||||
Raises:
|
||||
ValueError: If no valid block size found.
|
||||
|
||||
"""
|
||||
|
||||
def block_size_is_supported(
|
||||
@@ -458,8 +458,7 @@ def allocate_kv_cache(
|
||||
def prepare_kernel_block_sizes(
|
||||
kv_cache_config: KVCacheConfig, attn_groups: list[list[AttentionGroup]]
|
||||
) -> list[int]:
|
||||
"""
|
||||
Generate kernel_block_sizes that matches each block_size.
|
||||
"""Generate kernel_block_sizes that matches each block_size.
|
||||
|
||||
For attention backends that support virtual block splitting,
|
||||
use the supported block sizes from the backend.
|
||||
@@ -471,6 +470,7 @@ def prepare_kernel_block_sizes(
|
||||
|
||||
Returns:
|
||||
List of kernel block sizes for each cache group.
|
||||
|
||||
"""
|
||||
kernel_block_sizes = []
|
||||
for kv_cache_gid, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
@@ -505,8 +505,7 @@ def sanity_check_mm_encoder_outputs(
|
||||
mm_embeddings: MultiModalEmbeddings,
|
||||
expected_num_items: int,
|
||||
) -> None:
|
||||
"""
|
||||
Perform sanity checks for the result of
|
||||
"""Perform sanity checks for the result of
|
||||
[`vllm.model_executor.models.SupportsMultiModal.embed_multimodal`][].
|
||||
"""
|
||||
assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), (
|
||||
@@ -532,8 +531,7 @@ def sanity_check_mm_encoder_outputs(
|
||||
|
||||
|
||||
def request_memory(init_snapshot: MemorySnapshot, cache_config: CacheConfig) -> int:
|
||||
"""
|
||||
Calculate the amount of memory required by vLLM, then validate
|
||||
"""Calculate the amount of memory required by vLLM, then validate
|
||||
that the current amount of free memory is sufficient for that.
|
||||
"""
|
||||
requested_memory = math.ceil(
|
||||
@@ -559,8 +557,7 @@ def add_kv_sharing_layers_to_kv_cache_groups(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
runner_only_attn_layers: set[str] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Sets up KV cache sharing by reusing the allocated KV caches in `kv_caches`
|
||||
"""Sets up KV cache sharing by reusing the allocated KV caches in `kv_caches`
|
||||
for layers that do not allocate its own KV cache, based on the mapping in
|
||||
`shared_kv_cache_layers`. Adds these layers to the corresponding KV cache
|
||||
group, which is needed to ensure that attention metadata is assigned later.
|
||||
@@ -571,6 +568,9 @@ def add_kv_sharing_layers_to_kv_cache_groups(
|
||||
means this layer will perform attention using the keys and values
|
||||
from the KV cache of `shared_kv_cache_layers[layer_name]`.
|
||||
kv_cache_groups: The KV cache groups of the model.
|
||||
runner_only_attn_layers: Attention layers handled by the runner only,
|
||||
which are excluded from the KV cache groups.
|
||||
|
||||
"""
|
||||
if not shared_kv_cache_layers:
|
||||
return
|
||||
@@ -595,8 +595,7 @@ def bind_kv_cache(
|
||||
num_attn_module: int = 1,
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Bind the allocated KV cache to both ModelRunner and forward context so
|
||||
"""Bind the allocated KV cache to both ModelRunner and forward context so
|
||||
that the KV cache can be used in the forward pass.
|
||||
|
||||
This function:
|
||||
@@ -607,9 +606,13 @@ def bind_kv_cache(
|
||||
|
||||
Args:
|
||||
kv_caches: The allocated kv_caches with layer names as keys.
|
||||
num_attn_module: Number of attention modules per layer entry.
|
||||
forward_context: The global forward context containing all Attention
|
||||
layers with layer names as keys.
|
||||
runner_kv_caches: The kv_cache declared by ModelRunner.
|
||||
kv_cache_groups: The KV cache groups of the model, used to resolve
|
||||
layers that share a KV cache.
|
||||
|
||||
"""
|
||||
# Bind kv_caches to ModelRunner
|
||||
assert len(runner_kv_caches) == 0
|
||||
|
||||
@@ -55,8 +55,7 @@ class WorkerBase:
|
||||
distributed_init_method: str,
|
||||
is_driver_worker: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize common worker components.
|
||||
"""Initialize common worker components.
|
||||
|
||||
Args:
|
||||
vllm_config: Complete vLLM configuration
|
||||
@@ -65,6 +64,7 @@ class WorkerBase:
|
||||
distributed_init_method: Distributed initialization method
|
||||
is_driver_worker: Whether this worker handles driver
|
||||
responsibilities
|
||||
|
||||
"""
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
@@ -118,6 +118,7 @@ class WorkerBase:
|
||||
|
||||
Returns:
|
||||
Compilation times (language_model, encoder) in seconds.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -127,7 +128,8 @@ class WorkerBase:
|
||||
|
||||
def synchronize_device(self) -> None:
|
||||
"""Block until in-flight device work completes; backends outside
|
||||
``torch.accelerator`` must override with their own wait."""
|
||||
``torch.accelerator`` must override with their own wait.
|
||||
"""
|
||||
if torch.accelerator.is_available():
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -209,8 +211,7 @@ class WorkerBase:
|
||||
|
||||
|
||||
class WorkerWrapperBase:
|
||||
"""
|
||||
This class represents one process in an executor/engine. It is responsible
|
||||
"""This class represents one process in an executor/engine. It is responsible
|
||||
for lazily initializing the worker and handling the worker's lifecycle.
|
||||
We first instantiate the WorkerWrapper, which remembers the worker module
|
||||
and class name. Then, when we call `update_environment_variables`, and the
|
||||
@@ -222,8 +223,7 @@ class WorkerWrapperBase:
|
||||
rpc_rank: int = 0,
|
||||
global_rank: int | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the worker wrapper with the given vllm_config and rpc_rank.
|
||||
"""Initialize the worker wrapper with the given vllm_config and rpc_rank.
|
||||
Note: rpc_rank is the rank of the worker in the executor. In most cases,
|
||||
it is also the rank of the worker in the distributed group. However,
|
||||
when multiple executors work together, they can be different.
|
||||
@@ -252,8 +252,7 @@ class WorkerWrapperBase:
|
||||
|
||||
@instrument(span_name="Worker init")
|
||||
def init_worker(self, all_kwargs: list[dict[str, Any]]) -> None:
|
||||
"""
|
||||
Here we inject some common logic before initializing the worker.
|
||||
"""Here we inject some common logic before initializing the worker.
|
||||
Arguments are passed to the worker class constructor.
|
||||
"""
|
||||
kwargs = all_kwargs[self.rpc_rank]
|
||||
|
||||
@@ -123,6 +123,7 @@ class WorkspaceManager:
|
||||
|
||||
Returns:
|
||||
List of tensor views into the workspace buffer, one per shape/dtype pair.
|
||||
|
||||
"""
|
||||
actual_bytes = [_compute_bytes(s, d) for s, d in shapes_and_dtypes]
|
||||
aligned_bytes = [round_up(actual, 256) for actual in actual_bytes]
|
||||
@@ -148,6 +149,7 @@ class WorkspaceManager:
|
||||
|
||||
Returns:
|
||||
The current workspace tensor.
|
||||
|
||||
"""
|
||||
ubatch_id = dbo_current_ubatch_id()
|
||||
lane = _workspace_lane.get()
|
||||
@@ -228,6 +230,7 @@ def is_workspace_manager_initialized() -> bool:
|
||||
|
||||
Returns:
|
||||
True if workspace manager is initialized, False otherwise.
|
||||
|
||||
"""
|
||||
return _manager is not None
|
||||
|
||||
@@ -237,6 +240,7 @@ def current_workspace_manager() -> "WorkspaceManager":
|
||||
|
||||
Raises:
|
||||
AssertionError: If workspace manager has not been initialized.
|
||||
|
||||
"""
|
||||
assert _manager is not None, (
|
||||
"WorkspaceManager not initialized. Call init_workspace_manager() "
|
||||
@@ -259,6 +263,7 @@ def init_workspace_manager(
|
||||
device: The device to allocate workspace on.
|
||||
num_ubatches: Number of workspace ubatch slots. Defaults to 1.
|
||||
num_lanes: Number of independent execution lanes per ubatch. Defaults to 1.
|
||||
|
||||
"""
|
||||
global _manager
|
||||
if _manager is not None:
|
||||
@@ -289,6 +294,7 @@ def lock_workspace() -> None:
|
||||
lock_workspace()
|
||||
|
||||
# Now all get_workspace calls must fit in pre-allocated size
|
||||
|
||||
"""
|
||||
current_workspace_manager().lock()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user