diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py index fe9e332bdf..5576c84b05 100644 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -LM eval harness on model to compare vs HF baseline computed offline. +"""LM eval harness on model to compare vs HF baseline computed offline. Configs are found in configs/$MODEL.yaml pytest -s -v test_lm_eval_correctness.py \ diff --git a/.buildkite/performance-benchmarks/scripts/compare-json-results.py b/.buildkite/performance-benchmarks/scripts/compare-json-results.py index c9f8139fe6..823c7488ec 100644 --- a/.buildkite/performance-benchmarks/scripts/compare-json-results.py +++ b/.buildkite/performance-benchmarks/scripts/compare-json-results.py @@ -83,8 +83,7 @@ def compare_data_columns( drop_column: str, debug: bool = False, ): - """ - Align concatenation by keys derived from info_cols instead of row order. + """Align concatenation by keys derived from info_cols instead of row order. - Pick one canonical key list: subset of info_cols present in ALL files. - For each file: set index to those keys, aggregate duplicates (mean for metric, first for names). @@ -363,8 +362,7 @@ def _apply_two_decimals( # Export helpers (Excel + CSV) # ----------------------------- def _sanitize_sheet_name(name: str) -> str: - """ - Excel sheet constraints: + """Excel sheet constraints: - max 31 chars - cannot contain: : \ / ? * [ ] - cannot be empty diff --git a/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py b/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py index e694c38780..deb14a65e4 100644 --- a/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py +++ b/.buildkite/performance-benchmarks/scripts/convert-results-json-to-markdown.py @@ -93,8 +93,7 @@ def results_to_json(latency, throughput, serving): def get_size_with_unit(bytes, suffix="B"): - """ - Scale bytes to its proper format + """Scale bytes to its proper format e.g: 1253656 => '1.20MB' 1253656678 => '1.17GB' diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/scripts/generate-nightly-index.py index 67adf9c92b..8f92996ff9 100644 --- a/.buildkite/scripts/generate-nightly-index.py +++ b/.buildkite/scripts/generate-nightly-index.py @@ -18,8 +18,7 @@ import regex as re def normalize_package_name(name: str) -> str: - """ - Normalize package name according to PEP 503. + """Normalize package name according to PEP 503. https://peps.python.org/pep-0503/#normalized-names Replace runs of underscores, hyphens, and periods with a single hyphen, @@ -55,17 +54,18 @@ class WheelFileInfo: def parse_from_filename(file: str) -> WheelFileInfo: - """ - Parse wheel file name to extract metadata. + """Parse wheel file name to extract metadata. The format of wheel names: {package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl All versions could contain a variant like '+cu129' or '.cpu' or `.rocm` (or not). + Example: vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl vllm-0.10.2rc2+cu129-cp38-abi3-manylinux2014_aarch64.whl vllm-0.11.1rc8.dev14+gaa384b3c0-cp38-abi3-manylinux2014_aarch64.whl vllm-0.11.1rc8.dev14+gaa384b3c0.cu130-cp38-abi3-manylinux1_x86_64.whl + """ wheel_file_re = re.compile( r"^(?P.+)-(?P[^-]+?)(-(?P[^-]+))?-(?P[^-]+)-(?P[^-]+)-(?P[^-]+)\.whl$" @@ -111,9 +111,7 @@ def parse_from_filename(file: str) -> WheelFileInfo: def generate_project_list(subdir_names: list[str], comment: str = "") -> str: - """ - Generate project list HTML content linking to each project & variant subdirectory. - """ + """Generate project list HTML content linking to each project & variant subdirectory.""" href_tags = [] for name in sorted(subdir_names): name = name.strip("/").strip(".") @@ -127,9 +125,7 @@ def generate_package_index_and_metadata( index_base_dir: Path, comment: str = "", ) -> tuple[str, str]: - """ - Generate package index HTML content for a specific package, linking to actual wheel files. - """ + """Generate package index HTML content for a specific package, linking to actual wheel files.""" href_tags = [] metadata = [] for file in sorted(wheel_files, key=lambda x: x.filename): @@ -156,8 +152,7 @@ def generate_index_and_metadata( alias_to_default: str | None = None, comment: str = "", ): - """ - Generate index for all wheel files. + """Generate index for all wheel files. Args: whl_files (list[str]): List of wheel files (must be directly under `wheel_base_dir`). @@ -216,8 +211,8 @@ def generate_index_and_metadata( }, ... ] - """ + """ parsed_files = [parse_from_filename(f) for f in whl_files] if not parsed_files: diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index d63fb567ea..93fb5f5b81 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -54,6 +54,7 @@ steps: - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py + - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp mirror: amd: label: ":amd: (MI300) LoRA TP (Distributed) %N" @@ -79,3 +80,4 @@ steps: - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_gptoss_tp.py - test "$$BUILDKITE_PARALLEL_JOB" != "1" || pytest -v -s -x lora/test_qwen35_densemodel_lora.py - test "$$BUILDKITE_PARALLEL_JOB" != "2" || pytest -v -s -x lora/test_gemma4_tp.py + - test "$$BUILDKITE_PARALLEL_JOB" != "0" || pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp diff --git a/benchmarks/attention_benchmarks/batch_spec.py b/benchmarks/attention_benchmarks/batch_spec.py index 9f15f1d809..85c689b489 100644 --- a/benchmarks/attention_benchmarks/batch_spec.py +++ b/benchmarks/attention_benchmarks/batch_spec.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Simplified batch specification grammar for attention benchmarks. +"""Simplified batch specification grammar for attention benchmarks. Grammar (underscore-separated segments): Format: (?) q(k?) (s(k?))? @@ -24,6 +23,7 @@ Examples: q4s1k -> [(4, 1024)] # 4-token extend (spec decode) 2q1k_32q1s1k -> [(1024, 1024)] * 2 + [(1, 1024)] * 32 # Mixed batch 16q4s1k -> [(4, 1024)] * 16 # 16 spec decode requests + """ from collections import Counter @@ -71,8 +71,7 @@ def _parse_size(size_str: str, k_suffix: str) -> int: def parse_batch_spec(spec: str) -> list[BatchRequest]: - """ - Parse batch specification string into list of BatchRequest objects. + """Parse batch specification string into list of BatchRequest objects. Grammar: (?) q(k?) (s(k?))? @@ -84,6 +83,7 @@ def parse_batch_spec(spec: str) -> list[BatchRequest]: Raises: ValueError: If spec format is invalid + """ requests = [] @@ -103,8 +103,7 @@ def parse_batch_spec(spec: str) -> list[BatchRequest]: def format_batch_spec(requests: list[BatchRequest]) -> str: - """ - Format list of BatchRequest into human-readable string. + """Format list of BatchRequest into human-readable string. Groups requests by type and provides counts and sizes. @@ -113,6 +112,7 @@ def format_batch_spec(requests: list[BatchRequest]) -> str: Returns: Formatted string describing the batch + """ kinds = { "prefill": [], @@ -157,8 +157,7 @@ def format_batch_spec(requests: list[BatchRequest]) -> str: def reorder_for_flashinfer(requests: list[BatchRequest]) -> list[BatchRequest]: - """ - Reorder requests for FlashInfer: decode first, then prefill. + """Reorder requests for FlashInfer: decode first, then prefill. FlashInfer expects decode requests before prefill requests for optimal performance. @@ -168,6 +167,7 @@ def reorder_for_flashinfer(requests: list[BatchRequest]) -> list[BatchRequest]: Returns: Reordered list with decode requests first + """ decodes = [r for r in requests if r.is_decode] non_decodes = [r for r in requests if not r.is_decode] @@ -177,14 +177,14 @@ def reorder_for_flashinfer(requests: list[BatchRequest]) -> list[BatchRequest]: def split_by_type( requests: list[BatchRequest], ) -> dict[str, list[BatchRequest]]: - """ - Split requests by type for analysis. + """Split requests by type for analysis. Args: requests: List of BatchRequest Returns: Dict with keys: 'decode', 'prefill', 'extend' + """ result = { "decode": [], @@ -204,14 +204,14 @@ def split_by_type( def get_batch_stats(requests: list[BatchRequest]) -> dict: - """ - Compute statistics about a batch. + """Compute statistics about a batch. Args: requests: List of BatchRequest Returns: Dict with batch statistics + """ by_type = split_by_type(requests) @@ -232,8 +232,7 @@ def get_batch_stats(requests: list[BatchRequest]) -> dict: def get_batch_type(batch_spec: str, spec_decode_threshold: int = 8) -> str: - """ - Classify a batch spec into a type string. + """Classify a batch spec into a type string. Args: batch_spec: Batch specification string (e.g., "q2k", "8q1s1k", "2q2k_8q1s1k") @@ -241,6 +240,7 @@ def get_batch_type(batch_spec: str, spec_decode_threshold: int = 8) -> str: Returns: Type string: "prefill", "decode", "spec-decode", "extend", or "mixed (types...)" + """ requests = parse_batch_spec(batch_spec) diff --git a/benchmarks/attention_benchmarks/benchmark.py b/benchmarks/attention_benchmarks/benchmark.py index 9c1ea1add7..943133252b 100644 --- a/benchmarks/attention_benchmarks/benchmark.py +++ b/benchmarks/attention_benchmarks/benchmark.py @@ -2,8 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Universal vLLM Attention Benchmark +"""Universal vLLM Attention Benchmark. Benchmark any attention backend with the extended grammar. Supports standard attention (Flash/Triton/FlashInfer) and MLA backends. @@ -23,6 +22,7 @@ Examples: # Parameter sweep (YAML config - recommended) python benchmark.py --config configs/cutlass_numsplits.yaml + """ import argparse @@ -84,8 +84,7 @@ def run_mla_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult: def run_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult: - """ - Run a single benchmark with proper backend selection. + """Run a single benchmark with proper backend selection. Args: config: BenchmarkConfig with backend, batch_spec, and model params @@ -93,6 +92,7 @@ def run_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult: Returns: BenchmarkResult (may have error field set on failure) + """ try: if is_mla_backend(config.backend): @@ -119,8 +119,7 @@ def run_model_parameter_sweep( sweep: ModelParameterSweep, console: Console, ) -> list[BenchmarkResult]: - """ - Run model parameter sweep for given backends and batch specs. + """Run model parameter sweep for given backends and batch specs. Args: backends: List of backend names @@ -131,6 +130,7 @@ def run_model_parameter_sweep( Returns: List of BenchmarkResult objects + """ all_results = [] @@ -304,8 +304,7 @@ def run_parameter_sweep( sweep: ParameterSweep, console: Console, ) -> list[BenchmarkResult]: - """ - Run parameter sweep for given backends and batch specs. + """Run parameter sweep for given backends and batch specs. Args: backends: List of backend names @@ -316,6 +315,7 @@ def run_parameter_sweep( Returns: List of BenchmarkResult objects + """ all_results = [] @@ -395,8 +395,7 @@ def load_config_from_yaml(config_path: str) -> dict: def generate_batch_specs_from_ranges(ranges: list[dict]) -> list[str]: - """ - Generate batch specs from range specifications. + """Generate batch specs from range specifications. Args: ranges: List of range specifications, each containing: @@ -420,6 +419,7 @@ def generate_batch_specs_from_ranges(ranges: list[dict]) -> list[str]: } ] Returns: ["q1kv1k", "q2kv1k", ..., "q16kv1k"] + """ all_specs = [] diff --git a/benchmarks/attention_benchmarks/common.py b/benchmarks/attention_benchmarks/common.py index 45858f9148..b6fd26c803 100644 --- a/benchmarks/attention_benchmarks/common.py +++ b/benchmarks/attention_benchmarks/common.py @@ -21,8 +21,7 @@ from vllm.triton_utils import triton def batch_spec_sort_key(spec: str) -> tuple[int, int, int]: - """ - Extract sorting key from batch spec: (batch_size, max_q_len, max_kv_len). + """Extract sorting key from batch spec: (batch_size, max_q_len, max_kv_len). This ensures results are sorted by batch size first, then query length, then sequence length, rather than alphabetically. @@ -354,13 +353,13 @@ class ResultsFormatter: backends: list[str], compare_to_fastest: bool = True, ): - """ - Print results as a rich table. + """Print results as a rich table. Args: results: List of BenchmarkResult backends: List of backend names being compared compare_to_fastest: Show percentage comparison to fastest + """ # Group by batch spec, preserving first-occurrence order by_spec = {} @@ -489,14 +488,14 @@ class ResultsFormatter: def setup_mla_dims(model_name: str = "deepseek-v3") -> dict: - """ - Get MLA dimensions for known models. + """Get MLA dimensions for known models. Args: model_name: Model identifier Returns: Dict with MLA dimension configuration + """ configs = { "deepseek-v2": { @@ -542,8 +541,7 @@ def get_attention_scale(head_dim: int) -> float: def is_mla_backend(backend: str) -> bool: - """ - Check if backend is an MLA backend using the AttentionBackendEnum. + """Check if backend is an MLA backend using the AttentionBackendEnum. Args: backend: Backend name matching AttentionBackendEnum exactly @@ -551,6 +549,7 @@ def is_mla_backend(backend: str) -> bool: Returns: True if the backend is an MLA backend, False otherwise + """ from vllm.v1.attention.backends.registry import AttentionBackendEnum diff --git a/benchmarks/attention_benchmarks/mla_runner.py b/benchmarks/attention_benchmarks/mla_runner.py index 8a6ae3f87d..04dcd73719 100644 --- a/benchmarks/attention_benchmarks/mla_runner.py +++ b/benchmarks/attention_benchmarks/mla_runner.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -MLA benchmark runner - shared utilities for MLA benchmarks. +"""MLA benchmark runner - shared utilities for MLA benchmarks. This module provides helpers for running MLA backends without needing full VllmConfig integration. @@ -93,8 +92,7 @@ def _ensure_single_rank_model_parallel() -> None: def _add_mock_methods_to_model_config(model_config: ModelConfig) -> None: - """ - Add mock methods for layer-specific queries to ModelConfig. + """Add mock methods for layer-specific queries to ModelConfig. These methods are needed by metadata builders but aren't normally present on ModelConfig when used in benchmark contexts. @@ -125,14 +123,15 @@ def create_minimal_vllm_config( kv_cache_dtype: str = "auto", sparse_mla_force_mqa: bool = False, ) -> VllmConfig: - """ - Create minimal VllmConfig for MLA benchmarks. + """Create minimal VllmConfig for MLA benchmarks. Args: model_name: Model name (deepseek-v2, deepseek-v3, etc.) - used if mla_dims not provided block_size: KV cache block size max_num_seqs: Maximum number of sequences + max_num_batched_tokens: Maximum number of batched tokens + max_model_len: Maximum model context length mla_dims: Optional custom MLA dimensions dict. If not provided, uses setup_mla_dims(model_name) index_topk: Optional topk value for sparse MLA backends. If provided, @@ -140,11 +139,13 @@ def create_minimal_vllm_config( prefill_backend: Prefill backend name (e.g., "fa3", "fa4", "flashinfer", "trtllm"). Configures the attention config to force the specified prefill backend. + kv_cache_dtype: KV cache dtype, e.g. "auto" or "fp8" sparse_mla_force_mqa: If True, forces all sparse MLA tokens through forward_mqa (even prefill tokens). Returns: VllmConfig for benchmarking + """ # Get MLA dimensions - use provided or load from model name if mla_dims is None: @@ -320,8 +321,7 @@ _BACKEND_PROPERTIES = { def _get_backend_config(backend: str) -> dict: - """ - Get backend configuration from AttentionBackendEnum. + """Get backend configuration from AttentionBackendEnum. Uses the registry to get the backend class and extract configuration from its methods (get_impl_cls, get_builder_cls, is_sparse, etc.). @@ -332,6 +332,7 @@ def _get_backend_config(backend: str) -> dict: Returns: Dict with backend configuration + """ from vllm.v1.attention.backend import MultipleOf from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -381,8 +382,7 @@ def _build_attention_metadata( device: torch.device, builder_instance, ) -> tuple: - """ - Build attention metadata from batch requests. + """Build attention metadata from batch requests. Args: requests: List of BatchRequest objects @@ -392,6 +392,7 @@ def _build_attention_metadata( Returns: Tuple of (metadata, kv_cache_num_blocks) + """ q_lens = [r.q_len for r in requests] kv_lens = [r.kv_len for r in requests] @@ -472,8 +473,7 @@ def _create_input_tensors( device: torch.device, dtype: torch.dtype, ): - """ - Create input tensors for both decode and prefill modes. + """Create input tensors for both decode and prefill modes. MLA requires different tensor formats for decode vs prefill: - Decode: Uses kv_lora_rank (512) dimension @@ -490,6 +490,7 @@ def _create_input_tensors( Tuple of (decode_inputs, prefill_inputs) - decode_inputs: Query tensor(s) for decode mode - prefill_inputs: Dict with 'q', 'k_c_normed', 'k_pe', 'k_scale' for prefill + """ if query_format == "tuple": # Decode mode format: (q_nope, q_pe) where q_nope has kv_lora_rank dim @@ -583,8 +584,7 @@ def _create_backend_impl( index_topk: int | None = None, kv_cache_dtype: str = "auto", ): - """ - Create backend implementation instance. + """Create backend implementation instance. Args: backend_cfg: Backend configuration dict from _get_backend_config() @@ -593,9 +593,11 @@ def _create_backend_impl( device: Target device max_num_tokens: Maximum number of tokens for sparse indexer buffer index_topk: Topk value for sparse MLA backends + kv_cache_dtype: KV cache dtype, e.g. "auto" or "fp8" Returns: Tuple of (impl, layer, builder_instance, indexer) + """ # Get classes from backend config (already resolved by _get_backend_config) impl_class = backend_cfg["impl_class"] @@ -701,14 +703,14 @@ def _create_backend_impl( def _extract_mla_dims_from_config(config) -> dict | None: - """ - Extract MLA dimensions from BenchmarkConfig if all required fields are present. + """Extract MLA dimensions from BenchmarkConfig if all required fields are present. Args: config: BenchmarkConfig instance Returns: Dict with MLA dimensions if all fields are provided, None otherwise + """ # Check if all MLA-specific fields are provided if all( @@ -762,8 +764,7 @@ def _run_single_benchmark( output_scale: float | None = None, fuse_quant_op: bool = False, ) -> BenchmarkResult: - """ - Run a single benchmark iteration. + """Run a single benchmark iteration. Args: config: BenchmarkConfig instance @@ -774,6 +775,7 @@ def _run_single_benchmark( mla_dims: MLA dimension configuration device: Target device indexer: Optional MockIndexer for sparse backends + kv_cache_dtype: KV cache dtype. None keeps the backend default. output_scale: Static per-tensor FP8 scale for prefill output. None keeps the plain bf16 output (no quantization). fuse_quant_op: With output_scale set, True lets the prefill kernel write @@ -782,6 +784,7 @@ def _run_single_benchmark( Returns: BenchmarkResult with timing statistics + """ # Parse batch spec requests = parse_batch_spec(config.batch_spec) @@ -1044,8 +1047,7 @@ def _run_mla_benchmark_batched( output_scale: float | None = None, fuse_quant_op: bool = False, ) -> list[BenchmarkResult]: - """ - Unified batched MLA benchmark runner for all backends. + """Unified batched MLA benchmark runner for all backends. Works for: flashattn_mla, flashmla, flashinfer_mla, cutlass_mla, flashinfer_mla_sparse, flashmla_sparse @@ -1061,11 +1063,16 @@ def _run_mla_benchmark_batched( index_topk: Topk value for sparse MLA backends (default 2048) prefill_backend: Prefill backend name (e.g., "fa3", "fa4"). When set, forces the specified FlashAttention version for prefill. + output_scale: Static per-tensor FP8 scale for prefill output. None + keeps the plain bf16 output (no quantization). + fuse_quant_op: With output_scale set, True lets the prefill kernel write + FP8 directly; False runs bf16 attention then a standalone quant. sparse_mla_force_mqa: If True, forces all sparse MLA tokens through forward_mqa (even prefill tokens). Returns: List of BenchmarkResult objects + """ if not configs_with_params: return [] @@ -1277,8 +1284,7 @@ def run_mla_benchmark( output_scale: float | None = None, fuse_quant_op: bool = False, ) -> BenchmarkResult | list[BenchmarkResult]: - """ - Unified MLA benchmark runner for all backends. + """Unified MLA benchmark runner for all backends. Works for: flashattn_mla, flashmla, flashinfer_mla, cutlass_mla, flashinfer_mla_sparse, flashmla_sparse @@ -1303,6 +1309,7 @@ def run_mla_benchmark( Returns: BenchmarkResult (single mode) or list of BenchmarkResult (batched mode) + """ # Normalize to batched mode: (config, threshold, num_splits) if isinstance(config, list): diff --git a/benchmarks/attention_benchmarks/runner.py b/benchmarks/attention_benchmarks/runner.py index 01fd558043..a1b7d04505 100644 --- a/benchmarks/attention_benchmarks/runner.py +++ b/benchmarks/attention_benchmarks/runner.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Standard attention benchmark runner - shared utilities for non-MLA benchmarks. +"""Standard attention benchmark runner - shared utilities for non-MLA benchmarks. This module provides helpers for running standard attention backends (FlashAttention, Triton, FlashInfer) with real vLLM integration. @@ -57,8 +56,7 @@ from vllm.v1.kv_cache_interface import ( def _get_backend_config(backend: str) -> dict: - """ - Get backend configuration from AttentionBackendEnum. + """Get backend configuration from AttentionBackendEnum. Args: backend: Backend name matching AttentionBackendEnum exactly @@ -66,6 +64,7 @@ def _get_backend_config(backend: str) -> dict: Returns: Dict with backend_class + """ from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -409,6 +408,7 @@ def _run_single_benchmark( Returns: (timing_stats, mem_stats) where timing_stats is a dict with mean/std/min/max in seconds per layer. + """ total_q = q_list[0].shape[0] v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim @@ -458,8 +458,7 @@ def _run_single_benchmark( def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult: - """ - Run standard attention benchmark with real kernels. + """Run standard attention benchmark with real kernels. Supports: FLASH_ATTN, TRITON_ATTN, FLASHINFER @@ -468,6 +467,7 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult: Returns: BenchmarkResult with timing and memory statistics + """ device = torch.device(config.device) torch.accelerator.set_device_index(device) diff --git a/benchmarks/benchmark_batch_invariance.py b/benchmarks/benchmark_batch_invariance.py index 7473a41e51..dd22523ce2 100755 --- a/benchmarks/benchmark_batch_invariance.py +++ b/benchmarks/benchmark_batch_invariance.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark to measure the performance overhead of VLLM_BATCH_INVARIANT mode. +"""Benchmark to measure the performance overhead of VLLM_BATCH_INVARIANT mode. This benchmark runs the same workload twice: 1. With VLLM_BATCH_INVARIANT=0 (baseline) @@ -96,8 +95,7 @@ def run_benchmark_with_batch_invariant( batch_invariant: bool, seed: int = 12345, ) -> dict: - """ - Run the benchmark with the specified configuration. + """Run the benchmark with the specified configuration. Returns a dict with timing and throughput metrics. """ diff --git a/benchmarks/benchmark_hash.py b/benchmarks/benchmark_hash.py index 08cdc012d6..c5af4da999 100644 --- a/benchmarks/benchmark_hash.py +++ b/benchmarks/benchmark_hash.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Micro benchmark comparing built-in hash(), SHA-256, and xxHash. +"""Micro benchmark comparing built-in hash(), SHA-256, and xxHash. This focuses on a single test payload shaped like the prefix-cache hash input: (32-byte bytes object, 32-int tuple) diff --git a/benchmarks/benchmark_hidden_state_extraction.py b/benchmarks/benchmark_hidden_state_extraction.py index f0a35a0cf1..878be2c293 100644 --- a/benchmarks/benchmark_hidden_state_extraction.py +++ b/benchmarks/benchmark_hidden_state_extraction.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark hidden state extraction throughput. +"""Benchmark hidden state extraction throughput. Measures two modes: 1. Baseline: bulk inference with max_tokens=1, no extraction. diff --git a/benchmarks/benchmark_long_document_qa_throughput.py b/benchmarks/benchmark_long_document_qa_throughput.py index 6106b90149..1f5b5f5504 100644 --- a/benchmarks/benchmark_long_document_qa_throughput.py +++ b/benchmarks/benchmark_long_document_qa_throughput.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Offline benchmark to test the long document QA throughput. +"""Offline benchmark to test the long document QA throughput. Example usage: # This workload samples 8 different prompts with a default input @@ -49,14 +48,14 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser def test_long_document_qa(llm=None, sampling_params=None, prompts=None): - """ - Test long document QA with the given prompts and sampling parameters. + """Test long document QA with the given prompts and sampling parameters. Print the time spent in processing all the prompts. Args: llm: The language model used for generating responses. sampling_params: Sampling parameter used to generate the response. prompts: A list of prompt strings to be processed by the LLM. + """ start_time = time.time() llm.generate(prompts, sampling_params=sampling_params) @@ -65,8 +64,7 @@ def test_long_document_qa(llm=None, sampling_params=None, prompts=None): def repeat_prompts(prompts, repeat_count, mode: str): - """ - Repeat each prompt in the list for a specified number of times. + """Repeat each prompt in the list for a specified number of times. The order of prompts in the output list depends on the mode. Args: @@ -84,6 +82,7 @@ def repeat_prompts(prompts, repeat_count, mode: str): Raises: ValueError: If an invalid mode is provided. + """ print("Repeat mode: ", mode) if mode == "random": diff --git a/benchmarks/benchmark_prefix_block_hash.py b/benchmarks/benchmark_prefix_block_hash.py index 8bcd8af0d3..522bf9d1ab 100644 --- a/benchmarks/benchmark_prefix_block_hash.py +++ b/benchmarks/benchmark_prefix_block_hash.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Simple benchmark to compare prefix-cache block hashing algorithms. +"""Simple benchmark to compare prefix-cache block hashing algorithms. Example: python benchmark_prefix_block_hash.py --num-blocks 20000 --block-size 32 + """ from __future__ import annotations diff --git a/benchmarks/benchmark_prefix_caching.py b/benchmarks/benchmark_prefix_caching.py index 0145f6b7c8..5abdd65b2a 100644 --- a/benchmarks/benchmark_prefix_caching.py +++ b/benchmarks/benchmark_prefix_caching.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark the efficiency of prefix caching. +"""Benchmark the efficiency of prefix caching. This script allows you to benchmark the performance of a model with and without prefix caching using either fixed prompts diff --git a/benchmarks/benchmark_serving_structured_output.py b/benchmarks/benchmark_serving_structured_output.py index 664fa58dd4..dc8e30268e 100644 --- a/benchmarks/benchmark_serving_structured_output.py +++ b/benchmarks/benchmark_serving_structured_output.py @@ -102,6 +102,7 @@ class SampleRequest: images). prompt_len: The length of the prompt in tokens. expected_output_len: The expected length of the output in tokens. + """ prompt: str @@ -305,8 +306,7 @@ async def get_request( request_rate: float, burstiness: float = 1.0, ) -> AsyncGenerator[tuple[int, SampleRequest], None]: - """ - Asynchronously generates requests at a specified rate + """Asynchronously generates requests at a specified rate with OPTIONAL burstiness. Args: @@ -322,6 +322,7 @@ async def get_request( A lower burstiness value (0 < burstiness < 1) results in more bursty requests, while a higher burstiness value (burstiness > 1) results in a more uniform arrival of requests. + """ input_requests = iter(input_requests) diff --git a/benchmarks/benchmark_topk_topp.py b/benchmarks/benchmark_topk_topp.py index f727f16ea2..14111f733b 100644 --- a/benchmarks/benchmark_topk_topp.py +++ b/benchmarks/benchmark_topk_topp.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark comparing Triton vs PyTorch sort-based top-k/top-p implementations. +"""Benchmark comparing Triton vs PyTorch sort-based top-k/top-p implementations. Compares: - apply_top_k_top_p_triton (Triton binary search) @@ -47,8 +46,7 @@ def calculate_ops_pct( vocab_size: int, batch_size: int, ) -> float: - """ - Calculate the percentage of active top-k and top-p operations. + """Calculate the percentage of active top-k and top-p operations. Returns percentage where 100% = batch_size ops. E.g., if all rows have both top-k and top-p active, returns 200%. @@ -117,8 +115,7 @@ def benchmark_function( warmup_iters: int = 5, benchmark_iters: int = 20, ) -> tuple[float, int]: - """ - Benchmark a function and return (avg_time_ms, peak_memory_bytes). + """Benchmark a function and return (avg_time_ms, peak_memory_bytes). Returns average time in milliseconds and peak memory usage. """ diff --git a/benchmarks/cutlass_benchmarks/utils.py b/benchmarks/cutlass_benchmarks/utils.py deleted file mode 100644 index 659c68bb11..0000000000 --- a/benchmarks/cutlass_benchmarks/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Cutlass bench utils - -import torch - - -def to_fp8(tensor: torch.Tensor) -> torch.Tensor: - finfo = torch.finfo(torch.float8_e4m3fn) - return torch.round(tensor.clamp(min=finfo.min, max=finfo.max)).to( - dtype=torch.float8_e4m3fn - ) - - -def to_int8(tensor: torch.Tensor) -> torch.Tensor: - return torch.round(tensor.clamp(min=-128, max=127)).to(dtype=torch.int8) - - -def to_bf16(tensor: torch.Tensor) -> torch.Tensor: - return tensor.to(dtype=torch.bfloat16) - - -def to_fp16(tensor: torch.Tensor) -> torch.Tensor: - return tensor.to(dtype=torch.float16) - - -def make_rand_tensors( - dtype: torch.dtype, m: int, n: int, k: int -) -> tuple[torch.Tensor, torch.Tensor]: - a = torch.randn((m, k), device="cuda") * 5 - b = torch.randn((n, k), device="cuda").t() * 5 - - if dtype == torch.int8: - return to_int8(a), to_int8(b) - if dtype == torch.float8_e4m3fn: - return to_fp8(a), to_fp8(b) - - raise ValueError("unsupported dtype") diff --git a/benchmarks/cutlass_benchmarks/w8a8_benchmarks.py b/benchmarks/cutlass_benchmarks/w8a8_benchmarks.py deleted file mode 100644 index f7325ddd2c..0000000000 --- a/benchmarks/cutlass_benchmarks/w8a8_benchmarks.py +++ /dev/null @@ -1,372 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import argparse -import copy -import itertools -import pickle as pkl -import time -from collections.abc import Callable, Iterable - -import torch -import torch.utils.benchmark as TBenchmark -from torch.utils.benchmark import Measurement as TMeasurement -from utils import make_rand_tensors -from weight_shapes import WEIGHT_SHAPES - -from vllm import _custom_ops as ops -from vllm.model_executor.layers.quantization.utils.fp8_utils import ( - w8a8_triton_block_scaled_mm, -) -from vllm.utils.argparse_utils import FlexibleArgumentParser -from vllm.utils.math_utils import cdiv - -DEFAULT_MODELS = list(WEIGHT_SHAPES.keys()) -DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512] -DEFAULT_TP_SIZES = [1] - - -# bench -def bench_fn( - label: str, sub_label: str, description: str, fn: Callable, *args, **kwargs -) -> TMeasurement: - min_run_time = 1 - - globals = { - "args": args, - "kwargs": kwargs, - "fn": fn, - } - return TBenchmark.Timer( - stmt="fn(*args, **kwargs)", - globals=globals, - label=label, - sub_label=sub_label, - description=description, - ).blocked_autorange(min_run_time=min_run_time) - - -def bench_int8( - dtype: torch.dtype, - m: int, - k: int, - n: int, - label: str, - sub_label: str, - bench_kernels: list[str] | None = None, -) -> Iterable[TMeasurement]: - """Benchmark INT8-based kernels.""" - assert dtype == torch.int8 - a, b = make_rand_tensors(torch.int8, m, n, k) - scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32) - scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32) - bias = torch.zeros((n,), device="cuda", dtype=torch.bfloat16) - azp = torch.zeros((m,), device="cuda", dtype=torch.int32) - azp_adj = torch.zeros((n,), device="cuda", dtype=torch.int32) - - bench_fns = { - "pytorch_bf16_bf16_bf16_matmul-no-scales": lambda: torch.mm( - a.to(dtype=torch.bfloat16), b.to(dtype=torch.bfloat16) - ), - "pytorch_fp16_fp16_fp16_matmul-no-scales": lambda: torch.mm( - a.to(dtype=torch.float16), b.to(dtype=torch.float16) - ), - "cutlass_i8_i8_bf16_scaled_mm": lambda: ops.cutlass_scaled_mm( - a, b, scale_a, scale_b, torch.bfloat16 - ), - "cutlass_i8_i8_bf16_scaled_mm_bias": lambda: ops.cutlass_scaled_mm( - a, b, scale_a, scale_b, torch.bfloat16, bias - ), - "cutlass_i8_i8_bf16_scaled_mm_azp": lambda: ops.cutlass_scaled_mm_azp( - a, b, scale_a, scale_b, torch.bfloat16, azp_adj - ), - "cutlass_i8_i8_bf16_scaled_mm_azp_bias": lambda: ops.cutlass_scaled_mm_azp( - a, b, scale_a, scale_b, torch.bfloat16, azp_adj, None, bias - ), - "cutlass_i8_i8_bf16_scaled_mm_azp_pt": lambda: ops.cutlass_scaled_mm_azp( - a, b, scale_a, scale_b, torch.bfloat16, azp_adj, azp - ), - "cutlass_i8_i8_bf16_scaled_mm_azp_pt_bias": lambda: ops.cutlass_scaled_mm_azp( - a, b, scale_a, scale_b, torch.bfloat16, azp_adj, azp, bias - ), - } - - timers = [] - for name, fn in bench_fns.items(): - # If bench_kernels is None, run all. Otherwise, run only exact matches. - if bench_kernels is None or name in bench_kernels: - print(f"Running {name}") - timers.append(bench_fn(label, sub_label, name, fn)) - - return timers - - -def bench_fp8( - dtype: torch.dtype, - m: int, - k: int, - n: int, - label: str, - sub_label: str, - bench_kernels: list[str] | None = None, -) -> Iterable[TMeasurement]: - """Benchmark FP8-based kernels.""" - assert dtype == torch.float8_e4m3fn - a, b = make_rand_tensors(torch.float8_e4m3fn, m, n, k) - a_cont = a.contiguous() - scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32) - scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32) - - block_scale_a = torch.rand((m, cdiv(k, 128)), device="cuda", dtype=torch.float32) - block_scale_b = torch.rand( - cdiv(k, 128), cdiv(n, 128), device="cuda", dtype=torch.float32 - ) - block_scale_a_M_major = block_scale_a.t().contiguous().t() - block_scale_b_K_major = block_scale_b.t().contiguous().t() - bias = torch.zeros((n,), device="cuda", dtype=torch.bfloat16) - - print(m, k, n) - - bench_fns = { - "pytorch_bf16_bf16_bf16_matmul-no-scales": lambda: torch.mm( - a.to(dtype=torch.bfloat16), b.to(dtype=torch.bfloat16) - ), - "pytorch_fp16_fp16_fp16_matmul-no-scales": lambda: torch.mm( - a.to(dtype=torch.float16), b.to(dtype=torch.float16) - ), - "pytorch_fp8_fp8_fp16_scaled_mm": lambda: torch._scaled_mm( - a, b, scale_a, scale_b, out_dtype=torch.float16 - ), - "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum": lambda: torch._scaled_mm( - a, b, scale_a, scale_b, out_dtype=torch.float16, use_fast_accum=True - ), - "pytorch_fp8_fp8_bf16_scaled_mm": lambda: torch._scaled_mm( - a, b, scale_a, scale_b, out_dtype=torch.bfloat16 - ), - "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum": lambda: torch._scaled_mm( - a, b, scale_a, scale_b, out_dtype=torch.bfloat16, use_fast_accum=True - ), - "cutlass_fp8_fp8_bf16_scaled_mm": lambda: ops.cutlass_scaled_mm( - a, b, scale_a, scale_b, torch.bfloat16 - ), - "cutlass_fp8_fp8_fp16_scaled_mm": lambda: ops.cutlass_scaled_mm( - a, b, scale_a, scale_b, torch.float16 - ), - "cutlass_fp8_fp8_bf16_scaled_mm_bias": lambda: ops.cutlass_scaled_mm( - a, b, scale_a, scale_b, torch.bfloat16, bias - ), - "cutlass_fp8_fp8_fp16_scaled_mm_bias": lambda: ops.cutlass_scaled_mm( - a, b, scale_a, scale_b, torch.float16, bias.to(dtype=torch.float16) - ), - "triton_fp8_fp8_fp16_scaled_mm_blockwise": lambda: w8a8_triton_block_scaled_mm( - a_cont, b.t(), block_scale_a, block_scale_b.t(), (128, 128) - ), - "cutlass_fp8_fp8_fp16_scaled_mm_blockwise": lambda: ops.cutlass_scaled_mm( - a, b, block_scale_a_M_major, block_scale_b_K_major, torch.float16 - ), - } - - timers = [] - for name, fn in bench_fns.items(): - # If bench_kernels is None, run all. Otherwise, run only exact matches. - if bench_kernels is None or name in bench_kernels: - print(f"Running {name}") - timers.append(bench_fn(label, sub_label, name, fn)) - - return timers - - -def bench( - dtype: torch.dtype, - m: int, - k: int, - n: int, - label: str, - sub_label: str, - bench_kernels: list[str] | None = None, -) -> Iterable[TMeasurement]: - if dtype == torch.int8: - return bench_int8(dtype, m, k, n, label, sub_label, bench_kernels) - if dtype == torch.float8_e4m3fn: - return bench_fp8(dtype, m, k, n, label, sub_label, bench_kernels) - raise ValueError("unsupported type") - - -# runner -def print_timers(timers: Iterable[TMeasurement]): - compare = TBenchmark.Compare(timers) - compare.print() - - -def run( - dtype: torch.dtype, - MKNs: Iterable[tuple[int, int, int]], - bench_kernels: list[str] | None = None, -) -> Iterable[TMeasurement]: - results = [] - for m, k, n in MKNs: - timers = bench( - dtype, - m, - k, - n, - f"scaled-{dtype}-gemm", - f"MKN=({m}x{k}x{n})", - bench_kernels=bench_kernels, - ) - print_timers(timers) - results.extend(timers) - return results - - -def make_output( - data: Iterable[TMeasurement], - MKNs: Iterable[tuple[int, int, int]], - base_description: str, - timestamp=None, -): - print(f"== All Results {base_description} ====") - print_timers(data) - - # pickle all the results - timestamp = int(time.time()) if timestamp is None else timestamp - with open(f"{base_description}-{timestamp}.pkl", "wb") as f: - pkl.dump(data, f) - - -def run_square_bench(args): - dim_sizes = list(range(args.dim_start, args.dim_end + 1, args.dim_increment)) - MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes)) - data = run(args.dtype, MKNs, bench_kernels=args.kernels) - make_output(data, MKNs, f"square_bench-{args.dtype}") - - -def run_range_bench(args): - dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment)) - n = len(dim_sizes) - Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes - Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes - Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes - MKNs = list(zip(Ms, Ks, Ns)) - data = run(args.dtype, MKNs, bench_kernels=args.kernels) - make_output(data, MKNs, f"range_bench-{args.dtype}") - - -def run_model_bench(args): - print("Benchmarking models:") - for i, model in enumerate(args.models): - print(f"[{i}] {model}") - - def model_shapes(model_name: str, tp_size: int) -> list[tuple[int, int]]: - KNs = [] - for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]): - KN[tp_split_dim] = KN[tp_split_dim] // tp_size - KNs.append(KN) - return KNs - - model_bench_data = [] - models_tps = list(itertools.product(args.models, args.tp_sizes)) - for model, tp_size in models_tps: - Ms = args.batch_sizes - KNs = model_shapes(model, tp_size) - MKNs = [] - for m in Ms: - for k, n in KNs: - MKNs.append((m, k, n)) - - data = run(args.dtype, MKNs, bench_kernels=args.kernels) - model_bench_data.append(data) - - # Print all results - for data, model_tp in zip(model_bench_data, models_tps): - model, tp_size = model_tp - print(f"== Results {args.dtype} {model}-TP{tp_size} ====") - print_timers(data) - - timestamp = int(time.time()) - - all_data = [] - for d in model_bench_data: - all_data.extend(d) - # pickle all data - with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f: - pkl.dump(all_data, f) - - -if __name__ == "__main__": - - def to_torch_dtype(dt): - if dt == "int8": - return torch.int8 - if dt == "fp8": - return torch.float8_e4m3fn - raise ValueError("unsupported dtype") - - parser = FlexibleArgumentParser( - description=""" -Benchmark Cutlass GEMM. - - To run square GEMMs: - python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 square_bench --dim-start 128 --dim-end 512 --dim-increment 64 - - To run constant N and K and sweep M: - python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384 - - To run dimensions from a model: - python3 ./benchmarks/cutlass_benchmarks/w8a8_benchmarks.py --dtype fp8 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1 - - Output: - - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs. - """, # noqa: E501 - formatter_class=argparse.RawTextHelpFormatter, - ) - - parser.add_argument( - "--dtype", - type=to_torch_dtype, - required=True, - help="Available options are ['int8', 'fp8']", - ) - parser.add_argument( - "--kernels", - nargs="+", - type=str, - default=None, - help="Exact names of the kernels to benchmark. If not set, runs all kernels.", - ) - - subparsers = parser.add_subparsers(dest="cmd") - - square_parser = subparsers.add_parser("square_bench") - square_parser.add_argument("--dim-start", type=int, required=True) - square_parser.add_argument("--dim-end", type=int, required=True) - square_parser.add_argument("--dim-increment", type=int, required=True) - square_parser.set_defaults(func=run_square_bench) - - range_parser = subparsers.add_parser("range_bench") - range_parser.add_argument("--dim-start", type=int, required=True) - range_parser.add_argument("--dim-end", type=int, required=True) - range_parser.add_argument("--dim-increment", type=int, required=True) - range_parser.add_argument("--m-constant", type=int, default=None) - range_parser.add_argument("--n-constant", type=int, default=None) - range_parser.add_argument("--k-constant", type=int, default=None) - range_parser.set_defaults(func=run_range_bench) - - model_parser = subparsers.add_parser("model_bench") - model_parser.add_argument( - "--models", - nargs="+", - type=str, - default=DEFAULT_MODELS, - choices=WEIGHT_SHAPES.keys(), - ) - model_parser.add_argument( - "--tp-sizes", nargs="+", type=int, default=DEFAULT_TP_SIZES - ) - model_parser.add_argument( - "--batch-sizes", nargs="+", type=int, default=DEFAULT_BATCH_SIZES - ) - model_parser.set_defaults(func=run_model_bench) - - args = parser.parse_args() - args.func(args) diff --git a/benchmarks/cutlass_benchmarks/weight_shapes.py b/benchmarks/cutlass_benchmarks/weight_shapes.py deleted file mode 100644 index 25b96ef566..0000000000 --- a/benchmarks/cutlass_benchmarks/weight_shapes.py +++ /dev/null @@ -1,46 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Weight Shapes are in the format -# ([K, N], TP_SPLIT_DIM) -# Example: -# A shape of ([14336, 4096], 0) indicates the following GEMM shape, -# - TP1 : K = 14336, N = 4096 -# - TP2 : K = 7168, N = 4096 -# A shape of ([4096, 6144], 1) indicates the following GEMM shape, -# - TP1 : K = 4096, N = 6144 -# - TP4 : K = 4096, N = 1536 - -# TP1 shapes -WEIGHT_SHAPES = { - "mistralai/Mistral-7B-v0.1": [ - ([4096, 6144], 1), - ([4096, 4096], 0), - ([4096, 28672], 1), - ([14336, 4096], 0), - ], - "meta-llama/Llama-2-7b-hf": [ - ([4096, 12288], 1), - ([4096, 4096], 0), - ([4096, 22016], 1), - ([11008, 4096], 0), - ], - "meta-llama/Llama-3-8b": [ - ([4096, 6144], 1), - ([4096, 4096], 0), - ([4096, 28672], 1), - ([14336, 4096], 0), - ], - "meta-llama/Llama-2-13b-hf": [ - ([5120, 15360], 1), - ([5120, 5120], 0), - ([5120, 27648], 1), - ([13824, 5120], 0), - ], - "meta-llama/Llama-2-70b-hf": [ - ([8192, 10240], 1), - ([8192, 8192], 0), - ([8192, 57344], 1), - ([28672, 8192], 0), - ], -} diff --git a/benchmarks/fused_kernels/merge_attn_states_benchmarks.py b/benchmarks/fused_kernels/merge_attn_states_benchmarks.py index 26b04299b3..61dfd7ea12 100644 --- a/benchmarks/fused_kernels/merge_attn_states_benchmarks.py +++ b/benchmarks/fused_kernels/merge_attn_states_benchmarks.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark: Fused FP8 output quantization in merge_attn_states +"""Benchmark: Fused FP8 output quantization in merge_attn_states. Compares fused vs unfused approaches for producing FP8-quantized merged attention output: diff --git a/benchmarks/kernels/bench_concat_mla_q.py b/benchmarks/kernels/bench_concat_mla_q.py index 8d940484d6..c528ab86b2 100644 --- a/benchmarks/kernels/bench_concat_mla_q.py +++ b/benchmarks/kernels/bench_concat_mla_q.py @@ -24,9 +24,9 @@ def make_inputs(num_tokens, dtype): """Create inputs matching the real code path. Args: - contiguous_nope: If False, simulate the transposed BMM output - (non-contiguous nope with stride pattern from - [N,B,L].transpose(0,1)). + num_tokens: Number of tokens to generate inputs for. + dtype: dtype of the generated tensors. + """ # Simulate: bmm output [N, B, L].transpose(0, 1) -> [B, N, L] raw = torch.randn(NUM_HEADS, num_tokens, NOPE_DIM, dtype=dtype, device="cuda") diff --git a/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py b/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py index 0dd5c6d848..d0880b3f03 100644 --- a/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py +++ b/benchmarks/kernels/benchmark_2d_silu_mul_fp8_quant.py @@ -106,8 +106,7 @@ class BenchmarkTensors: def reference_quant(x: torch.Tensor, quant_out: torch.Tensor, use_ue8m0: bool): - """ - Reference triton quant kernel from, + """Reference triton quant kernel from, vllm.model_executor.layers.quantization.utils.fp8_utils """ assert quant_out.size() == x.size() diff --git a/benchmarks/kernels/benchmark_cutlass_moe_fp8.py b/benchmarks/kernels/benchmark_cutlass_moe_fp8.py index 03d7fb386f..bff4f73e2c 100644 --- a/benchmarks/kernels/benchmark_cutlass_moe_fp8.py +++ b/benchmarks/kernels/benchmark_cutlass_moe_fp8.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark the performance of the cutlass_moe_fp8 kernel vs the triton_moe +"""Benchmark the performance of the cutlass_moe_fp8 kernel vs the triton_moe kernel. Both kernels take in fp8 quantized weights and 16-bit activations, but use different quantization strategies and backends. """ @@ -190,7 +189,7 @@ def bench_run( torch.accelerator.synchronize() def bench_cuda_graph(graph, num_warmup=5, num_iters=100): - """Benchmark CUDA graph using events like benchmark_moe.py""" + """Benchmark CUDA graph using events like benchmark_moe.py.""" # Warmup for _ in range(num_warmup): graph.replay() diff --git a/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py b/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py index 7379bf8588..a14242732a 100644 --- a/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py +++ b/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark the performance of the cutlass_moe_fp4 kernel vs the triton_moe +"""Benchmark the performance of the cutlass_moe_fp4 kernel vs the triton_moe kernel. The cutlass_moe_fp4 kernel takes in fp4 quantized weights and 16-bit activations. The triton_moe kernel takes in fp8 weights(tensor scaled to fp8) and 16-bit activations. diff --git a/benchmarks/kernels/benchmark_device_communicators.py b/benchmarks/kernels/benchmark_device_communicators.py index 6c3a726a97..9f9a150471 100644 --- a/benchmarks/kernels/benchmark_device_communicators.py +++ b/benchmarks/kernels/benchmark_device_communicators.py @@ -2,8 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark script for device communicators: +"""Benchmark script for device communicators: CustomAllreduce (oneshot, twoshot), PyNcclCommunicator, and SymmMemCommunicator (multimem, two-shot). @@ -17,6 +16,7 @@ Usage: Example: torchrun --nproc_per_node=2 benchmark_device_communicators.py --sequence-lengths 512 1024 2048 --num-warmup 10 --num-trials 100 + """ import json @@ -241,7 +241,6 @@ class CommunicatorBenchmark: self, sequence_length: int, num_warmup: int, num_trials: int ) -> dict[str, float]: """Benchmark allreduce operations for all available communicators.""" - results = {} # Define communicators with their benchmark functions diff --git a/benchmarks/kernels/benchmark_fused_collective.py b/benchmarks/kernels/benchmark_fused_collective.py index c999c16021..aaa39fd9fe 100644 --- a/benchmarks/kernels/benchmark_fused_collective.py +++ b/benchmarks/kernels/benchmark_fused_collective.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark for FlashInfer fused collective operations vs standard operations. +"""Benchmark for FlashInfer fused collective operations vs standard operations. This benchmark compares: 1. FlashInfer's allreduce_fusion with trtllm backend @@ -446,9 +445,15 @@ def run_benchmarks( """Run all benchmarks for given configuration. Args: + num_tokens: Number of tokens in the benchmark input. + hidden_dim: Hidden dimension of the benchmark input. + dtype: dtype of the benchmark input. + use_residual: Whether to include a residual add in the fused op. + no_oneshot: Skip the one-shot allreduce variants. allreduce_params: Shared parameters for FlashInfer fused allreduce. workspaces: Dict mapping backend name ("trtllm", "mnnvl") to workspace. quant_modes: Set of quantization modes: "none", "fp8", "fp4". + """ ( input_tensor, diff --git a/benchmarks/kernels/benchmark_kimi_k3_attn_res.py b/benchmarks/kernels/benchmark_kimi_k3_attn_res.py index c8cf6f9acc..dba82a8722 100644 --- a/benchmarks/kernels/benchmark_kimi_k3_attn_res.py +++ b/benchmarks/kernels/benchmark_kimi_k3_attn_res.py @@ -10,6 +10,7 @@ Example: .venv/bin/python benchmarks/kernels/benchmark_kimi_k3_attn_res.py \ --output /tmp/attn_res.json + """ from __future__ import annotations diff --git a/benchmarks/kernels/benchmark_kimi_k3_kda_decode.py b/benchmarks/kernels/benchmark_kimi_k3_kda_decode.py index fcce640152..6b6d944455 100644 --- a/benchmarks/kernels/benchmark_kimi_k3_kda_decode.py +++ b/benchmarks/kernels/benchmark_kimi_k3_kda_decode.py @@ -16,6 +16,7 @@ state traffic itself. Example: python benchmarks/kernels/benchmark_kimi_k3_kda_decode.py \ --tokens 1 8 32 64 128 --heads 12 + """ import argparse diff --git a/benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py b/benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py index 2669dadb34..855769263d 100644 --- a/benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py +++ b/benchmarks/kernels/benchmark_kimi_k3_latent_moe_tail.py @@ -27,6 +27,7 @@ Examples: For multi-node runs, launch one ``torchrun`` agent per node and use a shared rendezvous endpoint. + """ from __future__ import annotations diff --git a/benchmarks/kernels/benchmark_lora.py b/benchmarks/kernels/benchmark_lora.py index ab930c59d2..606741d047 100644 --- a/benchmarks/kernels/benchmark_lora.py +++ b/benchmarks/kernels/benchmark_lora.py @@ -102,9 +102,7 @@ def make_rand_tensors( num_slices: int, device: str = "cuda", ) -> tuple[torch.Tensor, list[torch.Tensor], torch.Tensor]: - """ - Make LoRA input/output matrices. - """ + """Make LoRA input/output matrices.""" A = torch.rand(a_shape, dtype=a_dtype).to(device) # LoRA weights column major @@ -117,8 +115,7 @@ def make_rand_tensors( def make_prompt_lora_mapping( num_prompts: int, num_active_loras: int, sort_by_lora_id: bool, device: str ) -> torch.Tensor: - """ - All prompts are mapped to a LoRA ID in range [0, num_active_loras). + """All prompts are mapped to a LoRA ID in range [0, num_active_loras). where 0 refers to first lora, 1 refers to second lora and so on. """ assert num_active_loras > 0 @@ -147,9 +144,7 @@ def make_token_lora_mapping( seq_len_tensor: torch.Tensor, device: str, ): - """ - Make token_lora_mapping from prompt_lora_mapping and seq_lens_tensor - """ + """Make token_lora_mapping from prompt_lora_mapping and seq_lens_tensor.""" assert prompt_lora_mapping.shape[0] == num_prompts # token to lora index mapping @@ -174,8 +169,7 @@ def ref_group_gemm( scaling: float, add_inputs: bool | None, ): - """ - Torch group gemm reference implementation to test correctness of + """Torch group gemm reference implementation to test correctness of benchmarking operations. """ batches = seq_lens_cpu.size(0) @@ -198,9 +192,7 @@ def ref_group_gemm( class OpType(Enum): - """ - LoRA Ops to benchmark and its properties. - """ + """LoRA Ops to benchmark and its properties.""" LORA_SHRINK = auto() LORA_EXPAND = auto() @@ -291,9 +283,7 @@ class OpType(Enum): def matmul_dtypes( self, op_dtype: torch.dtype ) -> tuple[torch.dtype, torch.dtype, torch.dtype]: - """ - return a type, b type and c type for A x B = C - """ + """Return a type, b type and c type for A x B = C.""" if self.is_shrink_fn(): return op_dtype, op_dtype, torch.float32 elif self.is_expand_fn(): @@ -338,8 +328,7 @@ class OpType(Enum): top_k_num: int | None = None, num_experts: int | None = None, ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: - """ - Given num_slices, return the shapes of the A, B, and C matrices + """Given num_slices, return the shapes of the A, B, and C matrices in A x B = C, for the op_type """ m, k, n = self.mkn(batch_size, seq_length, hidden_size, lora_rank) @@ -419,9 +408,7 @@ class OpType(Enum): @dataclass class BenchmarkContext: - """ - LoRA benchmark context - """ + """LoRA benchmark context.""" batch_size: int hidden_size: int @@ -467,9 +454,7 @@ class BenchmarkContext: @dataclass class BenchmarkTensors: - """ - Input/Output tensors used for benchmarks - """ + """Input/Output tensors used for benchmarks.""" # matmul tensors input: torch.Tensor @@ -555,9 +540,7 @@ class BenchmarkTensors: ) def sanity_check(self, ctx: BenchmarkContext, op_type: OpType) -> None: - """ - Fails asserts when non-conformality is detected. - """ + """Fails asserts when non-conformality is detected.""" num_tokens = ( self.input.shape[1] if op_type.is_fused_moe_lora_expand_fn() @@ -577,9 +560,7 @@ class BenchmarkTensors: ) def to_device(self, device: str): - """ - Transfer tensors to device if the tensors aren't already on the device - """ + """Transfer tensors to device if the tensors aren't already on the device.""" def to_device(tensor: torch.Tensor): if tensor.device != device: @@ -604,9 +585,7 @@ class BenchmarkTensors: ) def metadata(self, ctx: BenchmarkContext, op_type: OpType) -> tuple[int, int, int]: - """ - Return num_seqs, num_tokens and max_seq_len - """ + """Return num_seqs, num_tokens and max_seq_len.""" num_seqs = self.seq_lens.shape[0] num_tokens = self.get_num_tokens( self.lora_kernel_meta.token_lora_mapping.shape[0], ctx.top_k_num, op_type @@ -630,8 +609,7 @@ class BenchmarkTensors: expert_map: torch.Tensor | None = None, pad_sorted_ids: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Aligns tokens and experts into block-sized chunks for LoRA-based + """Aligns tokens and experts into block-sized chunks for LoRA-based mixture-of-experts (MoE) execution. """ max_num_tokens_padded = topk_ids.numel() + num_experts * (block_size - 1) @@ -959,8 +937,7 @@ class BenchmarkTensors: def test_correctness( self, op_type: OpType, expand_fn_add_inputs: bool | None ) -> bool: - """ - Test correctness of op_type implementation against a grouped gemm + """Test correctness of op_type implementation against a grouped gemm reference implementation. """ seq_lens_cpu = self.seq_lens.to(device="cpu") @@ -1070,15 +1047,13 @@ def bench_torch_mm( op_type: OpType, cuda_graph_nops: int | None = None, ) -> TMeasurement: - """ - Benchmark basic torch.mm as a roofline. + """Benchmark basic torch.mm as a roofline. When all the input tokens have the same LoRA ID, the LoRA kernels are just a matmul. This torch.mm benchmark serves as a roofline for that case. input op_type is used in determining the m, k, n dimensions for the matmul. """ - batch_size, hidden_size, lora_rank, seq_length, dtype = ( ctx.batch_size, ctx.hidden_size, diff --git a/benchmarks/kernels/benchmark_mla_k_concat.py b/benchmarks/kernels/benchmark_mla_k_concat.py index 7debf36348..3cae613e82 100644 --- a/benchmarks/kernels/benchmark_mla_k_concat.py +++ b/benchmarks/kernels/benchmark_mla_k_concat.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark script comparing torch.cat vs direct copy for k_nope/k_pe concatenation +"""Benchmark script comparing torch.cat vs direct copy for k_nope/k_pe concatenation in MLA (Multi-head Latent Attention) prefill. This validates that the optimization from commit 8d4142bd is beneficial across diff --git a/benchmarks/kernels/benchmark_moe_defaults.py b/benchmarks/kernels/benchmark_moe_defaults.py index f6ad59366d..00c6606db0 100644 --- a/benchmarks/kernels/benchmark_moe_defaults.py +++ b/benchmarks/kernels/benchmark_moe_defaults.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark comparing old vs new default fused MoE configs. +"""Benchmark comparing old vs new default fused MoE configs. Runs the triton fused_moe kernel with three configurations for each scenario: 1. Tuned config (from JSON file, if available) — the target to match diff --git a/benchmarks/kernels/benchmark_per_token_group_quant.py b/benchmarks/kernels/benchmark_per_token_group_quant.py index f2195a6d78..fda3d2a652 100644 --- a/benchmarks/kernels/benchmark_per_token_group_quant.py +++ b/benchmarks/kernels/benchmark_per_token_group_quant.py @@ -15,7 +15,7 @@ from vllm.platforms import current_platform @contextmanager def _triton_mode(): - """Temporarily force the Triton fallback path""" + """Temporarily force the Triton fallback path.""" with patch("vllm.platforms.current_platform.is_cuda", return_value=False): yield diff --git a/benchmarks/kernels/benchmark_per_token_quant_fp8.py b/benchmarks/kernels/benchmark_per_token_quant_fp8.py index 6ce97e3036..0069b06c64 100644 --- a/benchmarks/kernels/benchmark_per_token_quant_fp8.py +++ b/benchmarks/kernels/benchmark_per_token_quant_fp8.py @@ -16,7 +16,7 @@ from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE def with_triton_mode(fn): - """Temporarily force the Triton fallback path""" + """Temporarily force the Triton fallback path.""" def wrapped(*args, **kwargs): with patch("vllm.platforms.current_platform.is_cuda", return_value=False): @@ -124,8 +124,7 @@ def compute_geomean_speedups( speedup_cols: list[str], groupby_cols: list[str] | None = None, ) -> pd.DataFrame: - """ - Compute geometric mean speedups over a baseline column. + """Compute geometric mean speedups over a baseline column. Args: df: Input dataframe @@ -135,6 +134,7 @@ def compute_geomean_speedups( Returns: pd.DataFrame with geometric mean speedups + """ from scipy.stats import gmean diff --git a/benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py b/benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py index 4ecd917f31..2d5ec80c6a 100644 --- a/benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py +++ b/benchmarks/kernels/benchmark_rdna_hybrid_w4a16_gemm.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark the RDNAHybridW4A16LinearKernel across decode and prefill shapes. +"""Benchmark the RDNAHybridW4A16LinearKernel across decode and prefill shapes. Usage: python benchmark_int4_gemm.py diff --git a/benchmarks/kernels/benchmark_reshape_and_cache.py b/benchmarks/kernels/benchmark_reshape_and_cache.py index 97af4ac976..f49bf61036 100644 --- a/benchmarks/kernels/benchmark_reshape_and_cache.py +++ b/benchmarks/kernels/benchmark_reshape_and_cache.py @@ -32,7 +32,6 @@ def run_benchmark( device: str = "cuda", ) -> float: """Return latency (seconds) for given num_tokens.""" - if kv_cache_dtype == "fp8" and head_size % 16: raise ValueError("fp8 kv-cache requires head_size to be a multiple of 16.") diff --git a/benchmarks/kernels/benchmark_reshape_and_cache_flash.py b/benchmarks/kernels/benchmark_reshape_and_cache_flash.py index 55c2037251..84d9cc707c 100644 --- a/benchmarks/kernels/benchmark_reshape_and_cache_flash.py +++ b/benchmarks/kernels/benchmark_reshape_and_cache_flash.py @@ -37,7 +37,6 @@ def run_benchmark( device: str = "cuda", ) -> float: """Return latency (seconds) for given num_tokens.""" - if kv_cache_dtype == "fp8" and head_size % 16: raise ValueError("fp8 kv-cache requires head_size to be a multiple of 16.") diff --git a/benchmarks/kernels/benchmark_rmsnorm.py b/benchmarks/kernels/benchmark_rmsnorm.py deleted file mode 100644 index d8d7f5bcf9..0000000000 --- a/benchmarks/kernels/benchmark_rmsnorm.py +++ /dev/null @@ -1,255 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import itertools - -import torch -from flashinfer.norm import fused_add_rmsnorm, rmsnorm -from torch import nn - -from vllm import _custom_ops as vllm_ops -from vllm.triton_utils import triton - - -class HuggingFaceRMSNorm(nn.Module): - def __init__(self, hidden_size: int, eps: float = 1e-6) -> None: - super().__init__() - self.weight = nn.Parameter(torch.ones(hidden_size)) - self.variance_epsilon = eps - - def forward( - self, - x: torch.Tensor, - residual: torch.Tensor | None = None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - orig_dtype = x.dtype - x = x.to(torch.float32) - if residual is not None: - x = x + residual.to(torch.float32) - residual = x.to(orig_dtype) - - variance = x.pow(2).mean(dim=-1, keepdim=True) - x = x * torch.rsqrt(variance + self.variance_epsilon) - x = x.to(orig_dtype) * self.weight - if residual is None: - return x - else: - return x, residual - - -def rmsnorm_naive( - x: torch.Tensor, - weight: torch.Tensor, - residual: torch.Tensor | None = None, - eps: float = 1e-6, -): - naive_norm = HuggingFaceRMSNorm(x.shape[-1], eps=eps) - naive_norm.weight = nn.Parameter(weight) - naive_norm = naive_norm.to(x.device) - - orig_shape = x.shape - x = x.view(-1, x.shape[-1]) - if residual is not None: - residual = residual.view(-1, residual.shape[-1]) - - output = naive_norm(x, residual) - - if isinstance(output, tuple): - output = (output[0].view(orig_shape), output[1].view(orig_shape)) - else: - output = output.view(orig_shape) - return output - - -def rmsnorm_flashinfer( - x: torch.Tensor, - weight: torch.Tensor, - residual: torch.Tensor | None = None, - eps: float = 1e-6, -): - orig_shape = x.shape - x = x.view(-1, x.shape[-1]) - if residual is not None: - residual = residual.view(-1, residual.shape[-1]) - - if residual is not None: - fused_add_rmsnorm(x, residual, weight, eps) - output = (x, residual) - else: - output = rmsnorm(x, weight, eps) - - if isinstance(output, tuple): - output = (output[0].view(orig_shape), output[1].view(orig_shape)) - else: - output = output.view(orig_shape) - return output - - -def rmsnorm_vllm( - x: torch.Tensor, - weight: torch.Tensor, - residual: torch.Tensor | None = None, - eps: float = 1e-6, -): - orig_shape = x.shape - x = x.view(-1, x.shape[-1]) - if residual is not None: - residual = residual.view(-1, residual.shape[-1]) - - if residual is not None: - vllm_ops.fused_add_rms_norm(x, residual, weight, eps) - output = (x, residual) - else: - out = torch.empty_like(x) - vllm_ops.rms_norm(out, x, weight, eps) - output = out - - if isinstance(output, tuple): - output = (output[0].view(orig_shape), output[1].view(orig_shape)) - else: - output = output.view(orig_shape) - return output - - -def calculate_diff(batch_size, seq_len, hidden_size, use_residual=True): - dtype = torch.bfloat16 - x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") - weight = torch.ones(hidden_size, dtype=dtype, device="cuda") - residual = torch.randn_like(x) if use_residual else None - - output_naive = rmsnorm_naive( - x.clone(), weight, residual.clone() if residual is not None else None - ) - output_flashinfer = rmsnorm_flashinfer( - x.clone(), weight, residual.clone() if residual is not None else None - ) - output_vllm = rmsnorm_vllm( - x.clone(), weight, residual.clone() if residual is not None else None - ) - - if use_residual: - output_naive = output_naive[0] - output_flashinfer = output_flashinfer[0] - output_vllm = output_vllm[0] - - print(f"Naive output={output_naive}") - print(f"FlashInfer output={output_flashinfer}") - print(f"vLLM output={output_vllm}") - - if torch.allclose( - output_naive, output_flashinfer, atol=1e-2, rtol=1e-2 - ) and torch.allclose(output_naive, output_vllm, atol=1e-2, rtol=1e-2): - print("✅ All implementations match") - else: - print("❌ Implementations differ") - - -batch_size_range = [2**i for i in range(0, 7, 2)] -seq_length_range = [2**i for i in range(6, 11, 1)] -head_num_range = [32, 48] -configs = list(itertools.product(head_num_range, batch_size_range, seq_length_range)) - - -def get_benchmark(use_residual): - @triton.testing.perf_report( - triton.testing.Benchmark( - x_names=["head_num", "batch_size", "seq_len"], - x_vals=[list(_) for _ in configs], - line_arg="provider", - line_vals=["huggingface", "flashinfer", "vllm"], - line_names=["HuggingFace", "FlashInfer", "vLLM"], - styles=[("blue", "-"), ("green", "-"), ("red", "-")], - ylabel="us", - plot_name=f"rmsnorm-perf-{'with' if use_residual else 'without'}-residual", - args={}, - ) - ) - def benchmark(head_num, batch_size, seq_len, provider): - dtype = torch.bfloat16 - hidden_size = head_num * 128 # assuming head_dim = 128 - - x = torch.randn(batch_size, seq_len, hidden_size, dtype=dtype, device="cuda") - weight = torch.ones(hidden_size, dtype=dtype, device="cuda") - residual = torch.randn_like(x) if use_residual else None - - quantiles = [0.5, 0.2, 0.8] - - if provider == "huggingface": - ms, min_ms, max_ms = triton.testing.do_bench( - lambda: rmsnorm_naive( - x.clone(), - weight, - residual.clone() if residual is not None else None, - ), - quantiles=quantiles, - ) - elif provider == "flashinfer": - ms, min_ms, max_ms = triton.testing.do_bench( - lambda: rmsnorm_flashinfer( - x.clone(), - weight, - residual.clone() if residual is not None else None, - ), - quantiles=quantiles, - ) - else: - ms, min_ms, max_ms = triton.testing.do_bench( - lambda: rmsnorm_vllm( - x.clone(), - weight, - residual.clone() if residual is not None else None, - ), - quantiles=quantiles, - ) - - return 1000 * ms, 1000 * max_ms, 1000 * min_ms - - return benchmark - - -if __name__ == "__main__": - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument( - "--batch-size", - type=int, - default=4, - help="Batch size", - ) - parser.add_argument( - "--seq-len", - type=int, - default=128, - help="Sequence length", - ) - parser.add_argument( - "--hidden-size", - type=int, - default=4096, - help="Hidden size (2nd dimension) of the sequence", - ) - parser.add_argument( - "--use-residual", action="store_true", help="Whether to use residual connection" - ) - parser.add_argument( - "--save-path", - type=str, - default="./configs/rmsnorm/", - help="Path to save rmsnorm benchmark results", - ) - - args = parser.parse_args() - - # Run correctness test - calculate_diff( - batch_size=args.batch_size, - seq_len=args.seq_len, - hidden_size=args.hidden_size, - use_residual=args.use_residual, - ) - - # Get the benchmark function with proper use_residual setting - benchmark = get_benchmark(args.use_residual) - # Run performance benchmark - benchmark.run(print_data=True, save_path=args.save_path) diff --git a/benchmarks/kernels/benchmark_selective_state_update.py b/benchmarks/kernels/benchmark_selective_state_update.py index ddf1efd018..476dce7dff 100644 --- a/benchmarks/kernels/benchmark_selective_state_update.py +++ b/benchmarks/kernels/benchmark_selective_state_update.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark and tuning script for the Mamba selective_state_update kernel. +"""Benchmark and tuning script for the Mamba selective_state_update kernel. Mirrors the fused MoE tuning workflow: sweeps (BLOCK_SIZE_M, num_warps) across an effective_batch grid for a given (headdim, dstate, ngroups, cache_dtype) and @@ -124,8 +123,7 @@ def benchmark_config( num_warmup: int = 20, graph_batch_size: int = 10, ) -> float | None: - """ - Time one (BLOCK_SIZE_M, num_warps) config for selective_state_update. + """Time one (BLOCK_SIZE_M, num_warps) config for selective_state_update. Returns elapsed time in microseconds, or None on error. Uses accelerator graph capture-and-replay to isolate kernel time from @@ -356,8 +354,7 @@ def validate_configs( rtol: float = 1e-2, state_dtype: torch.dtype | None = None, ) -> dict[int, bool]: - """ - For every (effective_batch, batch, nheads) in *active* that has a tuned + """For every (effective_batch, batch, nheads) in *active* that has a tuned config, run the kernel with that config and compare against the reference. Returns {effective_batch: passed}. """ diff --git a/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py b/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py index 9fcf278f2e..22ef378b07 100644 --- a/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py +++ b/benchmarks/kernels/benchmark_silu_mul_fp8_quant.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Comprehensive 3-way SiLU Benchmark Suite +"""Comprehensive 3-way SiLU Benchmark Suite. This benchmark compares three SiLU implementations: 1. SiLU V2 (CUDA) - Optimized CUDA kernel implementation @@ -110,7 +109,7 @@ def silu_mul_fp8_quant_deep_gemm_triton( eps: float = 1e-10, expert_offsets: torch.Tensor = None, ) -> tuple[torch.Tensor, torch.Tensor]: - """Quantize silu(y[..., :H]) * y[..., H:] to FP8 with group per-token scales + """Quantize silu(y[..., :H]) * y[..., H:] to FP8 with group per-token scales. y has shape (E, T, 2*H). The first half of the last dimension is silu-activated, multiplied by the second half, then quantized into FP8. @@ -206,7 +205,7 @@ def benchmark( iterations_per_run: int = 20, ): def generate_data(seed_offset=0): - """Generate input data with given seed offset""" + """Generate input data with given seed offset.""" set_random_seed(42 + seed_offset) y = torch.rand((E, T, 2 * H), dtype=torch.bfloat16, device="cuda").contiguous() diff --git a/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py b/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py index 6bb504b7f0..29ff156b60 100644 --- a/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py +++ b/benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py @@ -10,6 +10,7 @@ with a static scale file. Example: python benchmarks/kernels/benchmark_vit_aiter_fp8_attn.py \ --seq-lens 2304 4096 8192 16384 --head-dim 72 + """ from types import SimpleNamespace diff --git a/benchmarks/kernels/benchmark_w8a8_block_fp8.py b/benchmarks/kernels/benchmark_w8a8_block_fp8.py index 590d4cfdc6..3ad7ec8887 100644 --- a/benchmarks/kernels/benchmark_w8a8_block_fp8.py +++ b/benchmarks/kernels/benchmark_w8a8_block_fp8.py @@ -57,10 +57,12 @@ def w8a8_block_matmul( Bs: The per-block quantization scale for `B`. block_size: The block size for per-block quantization. It should be 2-dim, e.g., [128, 128]. + config: Triton kernel launch config, e.g. tile sizes and num_warps. output_dtype: The dtype of the returned tensor. Returns: torch.Tensor: The result of matmul. + """ assert len(block_size) == 2 block_n, block_k = block_size[0], block_size[1] diff --git a/benchmarks/kernels/ir/bench_ir_ops.py b/benchmarks/kernels/ir/bench_ir_ops.py index b23c4e8ae3..80fdd87acb 100644 --- a/benchmarks/kernels/ir/bench_ir_ops.py +++ b/benchmarks/kernels/ir/bench_ir_ops.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Generic benchmark harness for vLLM IR ops. +"""Generic benchmark harness for vLLM IR ops. Usage: python benchmarks/kernels/ir/bench_ir_ops.py diff --git a/benchmarks/kernels/ir/shapes.py b/benchmarks/kernels/ir/shapes.py index 6cc44cf6ce..40f5f89476 100644 --- a/benchmarks/kernels/ir/shapes.py +++ b/benchmarks/kernels/ir/shapes.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Shape configurations for IR op benchmarks. -""" +"""Shape configurations for IR op benchmarks.""" import torch @@ -20,10 +18,11 @@ COMMON_HIDDEN_SIZES = [ # Each entry maps an op name to a list of kwarg dicts that will be passed # to that op's registered input generator via op.generate_inputs(**kwargs). SHAPE_CONFIGS: dict[str, list[dict]] = { - "rms_norm": [ + op_name: [ {"num_tokens": n, "hidden_size": d, "dtype": dtype} for dtype in [torch.float16, torch.bfloat16, torch.float32] for d in COMMON_HIDDEN_SIZES for n in NUM_TOKENS - ], + ] + for op_name in ("rms_norm", "fused_add_rms_norm") } diff --git a/benchmarks/kernels/utils.py b/benchmarks/kernels/utils.py index a9af811bbe..64d23691bd 100644 --- a/benchmarks/kernels/utils.py +++ b/benchmarks/kernels/utils.py @@ -17,8 +17,7 @@ class CudaGraphBenchParams: @dataclasses.dataclass class ArgPool: - """ - When some argument of the benchmarking function is annotated with this type, + """When some argument of the benchmarking function is annotated with this type, the benchmarking class (BenchMM) will collapse the argument to a pick a single value from the given list of values, during function invocation. For every invocation during a benchmarking run, it will choose a diff --git a/benchmarks/multi_turn/bench_dataset.py b/benchmarks/multi_turn/bench_dataset.py index 8cb8a2f386..9b159a9ee9 100644 --- a/benchmarks/multi_turn/bench_dataset.py +++ b/benchmarks/multi_turn/bench_dataset.py @@ -148,8 +148,7 @@ class LognormalDistribution(Distribution): def _generate_lognormal_by_median( target_average: int, median_ratio: float ) -> tuple[float, float]: - """ - Compute (mu, sigma) for a lognormal distribution given: + """Compute (mu, sigma) for a lognormal distribution given: - a target average (mean of the distribution) - a ratio of median / mean (controls skewness), assume mean > median diff --git a/benchmarks/multi_turn/convert_sharegpt_to_openai.py b/benchmarks/multi_turn/convert_sharegpt_to_openai.py index fccab4d0ce..b415744873 100644 --- a/benchmarks/multi_turn/convert_sharegpt_to_openai.py +++ b/benchmarks/multi_turn/convert_sharegpt_to_openai.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Download dataset from: +"""Download dataset from: https://huggingface.co/datasets/philschmid/sharegpt-raw/blob/main/sharegpt_20230401_clean_lang_split.json Convert to OpenAI API: diff --git a/benchmarks/overheads/benchmark_hashing.py b/benchmarks/overheads/benchmark_hashing.py deleted file mode 100644 index 178599952d..0000000000 --- a/benchmarks/overheads/benchmark_hashing.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import cProfile -import pstats - -from vllm import LLM, SamplingParams -from vllm.utils.argparse_utils import FlexibleArgumentParser - -# A very long prompt, total number of tokens is about 15k. -LONG_PROMPT = ["You are an expert in large language models, aren't you?"] * 1000 -LONG_PROMPT = " ".join(LONG_PROMPT) - - -def main(args): - llm = LLM( - model=args.model, - enforce_eager=True, - enable_prefix_caching=True, - tensor_parallel_size=args.tensor_parallel_size, - ) - - sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len) - profiler = cProfile.Profile() - - print("------warm up------") - for i in range(3): - output = llm.generate(LONG_PROMPT, sampling_params) - print(output[0].outputs[0].text) - - print("------start generating------") - for i in range(3): - profiler.runctx( - "llm.generate(LONG_PROMPT, sampling_params)", globals(), locals() - ) - - # analyze the runtime of hashing function - stats = pstats.Stats(profiler) - stats.sort_stats("cumulative") - total_time = 0 - total_calls = 0 - for func in stats.stats: - if "hash_of_block" in func[2]: - total_time = stats.stats[func][3] - total_calls = stats.stats[func][0] - percentage = (total_time / stats.total_tt) * 100 - print( - f"Hashing took {total_time:.2f} seconds,{percentage:.2f}% of the total runtime." - ) - - -if __name__ == "__main__": - parser = FlexibleArgumentParser( - description="Benchmark the performance of hashing function in" - "automatic prefix caching." - ) - parser.add_argument("--model", type=str, default="lmsys/longchat-7b-16k") - parser.add_argument("--tensor-parallel-size", "-tp", type=int, default=1) - parser.add_argument("--output-len", type=int, default=10) - parser.add_argument( - "--enable-prefix-caching", action="store_true", help="enable prefix caching" - ) - args = parser.parse_args() - main(args) diff --git a/benchmarks/replayssm/e2e_decode_speedup.py b/benchmarks/replayssm/e2e_decode_speedup.py index b7084a2131..ff7c30669a 100644 --- a/benchmarks/replayssm/e2e_decode_speedup.py +++ b/benchmarks/replayssm/e2e_decode_speedup.py @@ -15,6 +15,7 @@ Examples: python e2e_decode_speedup.py --model-id nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16 python e2e_decode_speedup.py --dtype auto --buffer-len 16 \ --model-id nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 # B300 NVFP4 + """ import argparse diff --git a/cmake/external_projects/deepgemm.cmake b/cmake/external_projects/deepgemm.cmake index 2ff69bd8b6..9ac0d05414 100644 --- a/cmake/external_projects/deepgemm.cmake +++ b/cmake/external_projects/deepgemm.cmake @@ -32,7 +32,7 @@ else() # Pinned to the tip of the fork's dev branch: upstream 2.8.0 plus the SM120 # port, the SM90 paged-MQA kv_block=32/next_n=4 port, configurable SwiGLU # alpha/beta, and SiTU for FP8/FP4 Mega MoE. - set(_DEEPGEMM_UPSTREAM_TAG "ad1f1726aa540a76c1d26d6a120effb8de21eaa4") + set(_DEEPGEMM_UPSTREAM_TAG "a6bbb8000161c0dc3a85a0300a905f76898a7913") set(_deepgemm_fc_root "${FETCHCONTENT_BASE_DIR}") if(NOT _deepgemm_fc_root) diff --git a/csrc/cpu/generate_cpu_attn_dispatch.py b/csrc/cpu/generate_cpu_attn_dispatch.py index a82a27b844..12a14ef587 100644 --- a/csrc/cpu/generate_cpu_attn_dispatch.py +++ b/csrc/cpu/generate_cpu_attn_dispatch.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Generate CPU attention dispatch switch cases and kernel instantiations. -""" +"""Generate CPU attention dispatch switch cases and kernel instantiations.""" import os diff --git a/docs/contributing/model/multimodal.md b/docs/contributing/model/multimodal.md index 951d25587a..d374a5ecbc 100644 --- a/docs/contributing/model/multimodal.md +++ b/docs/contributing/model/multimodal.md @@ -454,17 +454,17 @@ return a schema of the tensors outputted by the HF processor that are related to ??? code ```python - def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str: + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: # Mistral3Processor requires text corresponding to the images return self.dummy_inputs.get_dummy_text(mm_counts) def _postprocess_hf_mm_data( self, - mm_data: Mapping[str, object], - hf_processor_mm_kwargs: Mapping[str, object], + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], processed_data: BatchFeature, ) -> BatchFeature: - if not mm_data: + if not hf_data: return processed_data pixel_values = processed_data.get("pixel_values") @@ -486,25 +486,36 @@ return a schema of the tensors outputted by the HF processor that are related to calls the HF processor on the multi-modal data without passing any text. If the HF processor instead requires text corresponding to the multi-modal items, you should override - [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] + [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] to return the dummy text from [BaseDummyInputsBuilder.get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text] like in the example above. If the HF processor expects the multi-modal data - under different keys than those provided by the multi-modal items - (e.g. `audio` instead of `audios`), or requires additional keyword arguments + under different keys than those provided by the multi-modal items, + or requires additional keyword arguments (e.g. `sampling_rate`), you should override - [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data]. + [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs]. If you need to modify the output of the HF processor, you should override [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data]. + Passthrough fields have already been merged into `processed_data` when this + hook is called. + If you need to change how the normalized inputs are passed to the HF + processor (for example, to call a method other than `__call__`, or to use + different kwargs to construct and invoke it), override + [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor]. If the HF processor applies additional transformations to the prompt tokens that are not reflected in its multi-modal outputs (e.g. prepending a BOS token), you should override [_postprocess_prompt][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_prompt] to replicate them. - For even more control over how the HF processor is called, you can override + Only processors that require multiple HF calls or manually construct the + complete output should override [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] - directly. + directly. Such implementations must obtain their normalized inputs through + [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] + and return through + [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] + so passthrough fields and model-specific postprocessing are applied consistently. Since `pixel_values` is now a list with one tensor per image, we can override [_get_mm_fields_config][vllm.multimodal.processing.BaseMultiModalProcessor._get_mm_fields_config] as follows: @@ -724,7 +735,7 @@ Examples: ### Custom HF processor -Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. It is then applied to the multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor] inside [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. +Some models don't define an HF processor class on HF Hub. In that case, you can define a custom HF processor that has the same call signature as HF processors and return it from [BaseProcessingInfo.get_hf_processor][vllm.multimodal.processing.BaseProcessingInfo.get_hf_processor]. The default [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] applies it to the normalized multi-modal data via [InputProcessingContext.call_hf_processor][vllm.multimodal.processing.InputProcessingContext.call_hf_processor]. Override `_call_hf_processor` if the custom processor instead exposes a different callable interface. Examples: diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index a2112a2922..202d8d04c1 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -8,9 +8,16 @@ In vLLM's rendering pipeline (see [BaseRenderer][vllm.renderers.base.BaseRendere Since Transformers 5.10, `ProcessorMixin` now allows multi-modal inputs to be passed by themselves. However, certain subclasses (such as `ChameleonProcessor`) and older out-of-tree implementations may still define their own `__call__` method that assumes the presence of text with corresponding placeholder tokens. This causes a problem as we don't have the original text anymore to pass to these processors. -To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_processor_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_processor_text] returns so that [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] passes it to the HF processor together with the multi-modal inputs to obtain the processed multi-modal data. +To work around this, each model defines how to generate dummy text based on the number of multi-modal inputs, via [get_dummy_text][vllm.multimodal.processing.BaseDummyInputsBuilder.get_dummy_text], which its override of [_get_hf_mm_text][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_text] returns. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] adds this text and `truncation=False` to the normalized HF inputs. The truncation setting is attached only to calls that use dummy text, so unrelated processor calls do not receive a text-specific option. -Similarly, since the multi-modal data extracted by vLLM may not match what a specific HF processor expects, [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main] allows each model to adapt the inputs via [_preprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._preprocess_hf_mm_data] (e.g. renaming keys like `audios` to `audio` or injecting extra keyword arguments such as `sampling_rate`) and the outputs via [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data], without having to reimplement the entire method. +HF processing is split into four overridable stages: + +1. [_get_hf_mm_inputs][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_inputs] extracts and normalizes processor data, kwargs, and passthrough data. It centrally renames vLLM's `audios` field to the standard HF `audio` keyword; processors that require another spelling can adapt it in an override. +2. [_call_hf_processor][vllm.multimodal.processing.BaseMultiModalProcessor._call_hf_processor] invokes the HF processor. Models can override it when they use a different callable or need separate construction and invocation kwargs. +3. [_finalize_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._finalize_hf_mm_data] merges passthrough fields into the processed output. It lazily creates an empty output when there is no processor data. +4. [_postprocess_hf_mm_data][vllm.multimodal.processing.BaseMultiModalProcessor._postprocess_hf_mm_data] performs model-specific output transformations after passthrough fields have been merged. + +This division lets most models customize one stage without reimplementing [_apply_hf_processor_main][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main]. Implementations that genuinely require multiple processor calls or manually construct their complete output may still override that method, but must use `_get_hf_mm_inputs` and `_finalize_hf_mm_data` to retain the common input and output behavior. ## Prompt Update Detection diff --git a/docs/features/lora.md b/docs/features/lora.md index 839624542e..78004722a9 100644 --- a/docs/features/lora.md +++ b/docs/features/lora.md @@ -422,6 +422,18 @@ vllm serve ibm-granite/granite-speech-3.3-2b \ Note: Default multimodal LoRAs are currently only available for `.generate` and chat completions. +## Sequence-Classification LoRA Adapters + +vLLM supports PEFT sequence-classification adapters that save a complete, single-layer linear classification head through `modules_to_save`. The saved module must be named `score` or `classifier`. + +See [classification_with_lora_offline.py](../../examples/pooling/classify/classification_with_lora_offline.py) for an offline classification example using a LoRA adapter. + +This support has the following limitations: + +- All adapters in one engine must have the same `num_labels` and hidden size as the base classification head. An incompatible adapter is rejected when it is loaded. +- A classification head stored as float32 is converted to the runtime head dtype when it is loaded. +- Token-classification adapters are not supported by this feature. + ## Using Tips ### Configuring `max_lora_rank` diff --git a/docs/features/per_request_metrics.md b/docs/features/per_request_metrics.md index 25211ecfc3..da1685f929 100644 --- a/docs/features/per_request_metrics.md +++ b/docs/features/per_request_metrics.md @@ -120,6 +120,18 @@ the same `metrics` response field. As with `n > 1`, metrics are omitted for requests with multiple prompts, because the timing data cannot be attributed to a single prompt's generation. +## Responses API + +Per-request metrics are available on the `/v1/responses` endpoint using the +common `metrics` object described above. Non-streaming responses include it at +the top level. Streaming responses include it in the final response carried by +the `response.completed` event; intermediate events do not include metrics. + +Metrics are omitted for Responses requests that perform multiple +model-generation turns, such as built-in tool-call workflows, because the +response retains timing data for only one generation turn while token usage is +accumulated across all turns. + ## Relationship to Prometheus Metrics The `metrics` response field provides per-request values for a single request. diff --git a/docs/mkdocs/gen_files/generate_argparse.py b/docs/mkdocs/gen_files/generate_argparse.py index 1c7025366f..ecdce7761f 100644 --- a/docs/mkdocs/gen_files/generate_argparse.py +++ b/docs/mkdocs/gen_files/generate_argparse.py @@ -98,8 +98,7 @@ mock_if_no_torch("torch", PydanticMagicMock(name="torch", nn=mock_nn)) # Mock torch.library.infer_schema for vllm.ir.ops.IrOpInplaceOverload.__init__ # We need to return the corresponding number of inputs, as IR infra will assert it def get_outputs(native_fn: Callable) -> str: - """ - Extract output schema from function's return type annotation, + """Extract output schema from function's return type annotation, e.g. 'Tensor' or 'Tensor, Tensor'. """ import typing @@ -249,11 +248,12 @@ def create_parser(add_cli_args, **kwargs) -> FlexibleArgumentParser: """Create a parser for the given class with markdown formatting. Args: - cls: The class to create a parser for + add_cli_args: The `add_cli_args` classmethod to build the parser from. **kwargs: Additional keyword arguments to pass to `cls.add_cli_args`. Returns: FlexibleArgumentParser: A parser with markdown formatting for the class. + """ try: parser = FlexibleArgumentParser(add_json_tip=False) diff --git a/docs/mkdocs/gen_files/generate_attention_backends.py b/docs/mkdocs/gen_files/generate_attention_backends.py index 942165eaa9..725d7072ca 100644 --- a/docs/mkdocs/gen_files/generate_attention_backends.py +++ b/docs/mkdocs/gen_files/generate_attention_backends.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Generates documentation table for attention backends showing feature support. +"""Generates documentation table for attention backends showing feature support. This script parses all registered attention backends using AST (no imports needed) and generates a markdown table showing what features each backend supports, @@ -304,6 +303,7 @@ def parse_mla_prefill_registry() -> dict[str, str]: Returns: A dict mapping backend names to their class paths. + """ if not MLA_PREFILL_REGISTRY_FILE.exists(): return {} @@ -325,6 +325,7 @@ def parse_mla_prefill_priorities() -> dict[str, list[str]]: Returns: A dict with keys like 'blackwell' and 'default' containing lists of backend enum names in priority order. + """ if not MLA_PREFILL_SELECTOR_FILE.exists(): return {} @@ -549,6 +550,7 @@ def parse_mla_prefill_backend_file(class_path: str) -> dict[str, Any] | None: Returns: A dict with backend properties, or None if parsing fails. + """ file_path = get_file_from_class_path(class_path) if file_path is None: diff --git a/docs/mkdocs/gen_files/generate_examples.py b/docs/mkdocs/gen_files/generate_examples.py index c9c10018ec..a514dcf879 100644 --- a/docs/mkdocs/gen_files/generate_examples.py +++ b/docs/mkdocs/gen_files/generate_examples.py @@ -49,8 +49,7 @@ def title(text: str) -> str: @dataclass class Example: - """ - Example class for generating documentation content from a given path. + """Example class for generating documentation content from a given path. Attributes: path (Path): The path to the main directory or file. @@ -64,6 +63,7 @@ class Example: Methods: generate() -> str: Generates the documentation content. + """ path: Path @@ -116,8 +116,7 @@ class Example: raise ValueError(f"Title not found in {self.main_file}") def fix_relative_links(self, content: str) -> str: - """ - Fix relative links in markdown content by converting them to gh-file + """Fix relative links in markdown content by converting them to gh-file format. Args: @@ -125,6 +124,7 @@ class Example: Returns: str: Content with relative links converted to gh-file format + """ # Regex to match markdown links [text](relative_path) # This matches links that don't start with http, https, ftp, or # diff --git a/docs/mkdocs/gen_files/generated_content.py b/docs/mkdocs/gen_files/generated_content.py index fa3f31b294..d669194424 100644 --- a/docs/mkdocs/gen_files/generated_content.py +++ b/docs/mkdocs/gen_files/generated_content.py @@ -35,6 +35,7 @@ def fill_markers(doc_path: str, blocks: dict[str, str]) -> None: FileNotFoundError: If the source page does not exist. ValueError: If an expected marker is missing, or any `gen:` marker is left unsubstituted after filling. + """ source = DOCS_DIR / doc_path if not source.exists(): diff --git a/docs/mkdocs/hooks/autoref_code.py b/docs/mkdocs/hooks/autoref_code.py index 647f74f202..58cec14d45 100644 --- a/docs/mkdocs/hooks/autoref_code.py +++ b/docs/mkdocs/hooks/autoref_code.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -MkDocs hook to automatically convert inline code references to API doc links. +"""MkDocs hook to automatically convert inline code references to API doc links. For example, `WeightTransferConfig` becomes [`WeightTransferConfig`][vllm.config.WeightTransferConfig] diff --git a/docs/mkdocs/hooks/url_schemes.py b/docs/mkdocs/hooks/url_schemes.py index d208b9d677..cee418936f 100644 --- a/docs/mkdocs/hooks/url_schemes.py +++ b/docs/mkdocs/hooks/url_schemes.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -MkDocs hook + markdown extension to enable the following links to render correctly, +"""MkDocs hook + markdown extension to enable the following links to render correctly, including inside content included via pymdownx.snippets: - Relative file links outside of the `docs/` directory, e.g.: @@ -71,9 +70,7 @@ class UrlSchemesPreprocessor(Preprocessor): return lines def replace_relative_link(match: re.Match) -> str: - """ - Replace relative file links with URLs if they point outside the docs dir. - """ + """Replace relative file links pointing outside the docs dir with URLs.""" title = match.group("title") path = match.group("path") path = ((DOC_DIR / page.file.src_uri).parent / path).resolve() @@ -91,9 +88,7 @@ class UrlSchemesPreprocessor(Preprocessor): return f"[{gh_icon} {title}]({url})" def replace_github_link(match: re.Match) -> str: - """ - Replace GitHub issue, PR, and project links with enhanced Markdown links. - """ + """Replace GitHub issue, PR, and project links with rich Markdown links.""" repo = match.group("repo") type = match.group("type") number = match.group("number") diff --git a/docs/models/pooling_models/embed.md b/docs/models/pooling_models/embed.md index a60bd4307a..13a2e0d381 100644 --- a/docs/models/pooling_models/embed.md +++ b/docs/models/pooling_models/embed.md @@ -47,7 +47,7 @@ You can compute pairwise similarity scores to build a similarity matrix using th | `JinaEmbeddingsV5Model`C | Qwen3-decoder or EuroBERT-encoder backbone with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small`, `jinaai/jina-embeddings-v5-text-nano` (see note) | ✅︎ | ✅︎ | | `LlamaBidirectionalModel`C | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ | | `LlamaModel`C, `LlamaForCausalLM`C, `MistralModel`C, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ | -| `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | | +| `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | ✅︎ | | | `NomicBertModel` | Nomic BERT | `nomic-ai/nomic-embed-text-v1`, `nomic-ai/nomic-embed-text-v2-moe`, `Snowflake/snowflake-arctic-embed-m-long`, etc. | | | | `Qwen2Model`C, `Qwen2ForCausalLM`C | Qwen2-based | `ssmits/Qwen2-7B-Instruct-embed-base` (see note), `Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc. | ✅︎ | ✅︎ | | `Qwen3Model`C, `Qwen3ForCausalLM`C | Qwen3-based | `Qwen/Qwen3-Embedding-0.6B`, etc. | ✅︎ | ✅︎ | diff --git a/examples/applications/api_server/server.py b/examples/applications/api_server/server.py index e56a7931f6..36474e2101 100644 --- a/examples/applications/api_server/server.py +++ b/examples/applications/api_server/server.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -NOTE: This API server is used only for demonstrating usage of AsyncEngine +"""NOTE: This API server is used only for demonstrating usage of AsyncEngine and simple performance benchmarks. It is not intended for production use. For production use, we recommend using our OpenAI compatible server. We are also not going to accept PRs modifying this file. diff --git a/examples/applications/chatbot/streamlit_openai_chatbot_webserver.py b/examples/applications/chatbot/streamlit_openai_chatbot_webserver.py index d60dbf4d75..eb18b41a5a 100644 --- a/examples/applications/chatbot/streamlit_openai_chatbot_webserver.py +++ b/examples/applications/chatbot/streamlit_openai_chatbot_webserver.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -vLLM Chat Assistant - A Streamlit Web Interface +"""vLLM Chat Assistant - A Streamlit Web Interface. A streamlined chat interface that quickly integrates with vLLM API server. @@ -78,6 +77,7 @@ def create_new_chat_session(): - current_session: Sets to new session ID - active_session: Sets to new session ID - messages: Resets to empty list + """ session_id = datetime.now().strftime("%Y-%m-%d %H:%M:%S") st.session_state.sessions[session_id] = [] @@ -101,6 +101,7 @@ def switch_to_chat_session(session_id): - current_session: Updated to specified session_id - active_session: Updated to specified session_id - messages: Loaded from sessions[session_id] + """ st.session_state.current_session = session_id st.session_state.active_session = session_id @@ -134,6 +135,7 @@ def get_llm_response(messages, model, reason, content_ph=None, reasoning_ph=None Note: The function uses streamlit placeholders for live updates. When reason=True, the reasoning process appears above the response. + """ full_text = "" think_text = "" @@ -256,6 +258,7 @@ def server_supports_reasoning(): Returns: bool: True if the model supports reasoning, False otherwise + """ resp = client.chat.completions.create( model=model, diff --git a/examples/applications/rag/retrieval_augmented_generation_with_langchain.py b/examples/applications/rag/retrieval_augmented_generation_with_langchain.py index d9a4cadb03..1681ea1466 100644 --- a/examples/applications/rag/retrieval_augmented_generation_with_langchain.py +++ b/examples/applications/rag/retrieval_augmented_generation_with_langchain.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Retrieval Augmented Generation (RAG) Implementation with Langchain +"""Retrieval Augmented Generation (RAG) Implementation with Langchain ================================================================== This script demonstrates a RAG implementation using LangChain, Milvus @@ -35,6 +34,7 @@ Notes: - Ensure both vLLM services are running before executing - Default ports: 8000 (embedding), 8001 (chat) - First run may take time to download models + """ import argparse @@ -52,9 +52,7 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter def load_and_split_documents(config: dict[str, Any]): - """ - Load and split documents from web URL - """ + """Load and split documents from web URL.""" try: loader = WebBaseLoader(web_paths=(config["url"],)) docs = loader.load() @@ -70,9 +68,7 @@ def load_and_split_documents(config: dict[str, Any]): def init_vectorstore(config: dict[str, Any], documents: list[Document]): - """ - Initialize vector store with documents - """ + """Initialize vector store with documents.""" return Milvus.from_documents( documents=documents, embedding=OpenAIEmbeddings( @@ -86,9 +82,7 @@ def init_vectorstore(config: dict[str, Any], documents: list[Document]): def init_llm(config: dict[str, Any]): - """ - Initialize llm - """ + """Initialize llm.""" return ChatOpenAI( model=config["chat_model"], openai_api_key=config["vllm_api_key"], @@ -97,9 +91,7 @@ def init_llm(config: dict[str, Any]): def get_qa_prompt(): - """ - Get question answering prompt template - """ + """Get question answering prompt template.""" template = """You are an assistant for question-answering tasks. Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that you don't know. @@ -112,16 +104,12 @@ Answer: def format_docs(docs: list[Document]): - """ - Format documents for prompt - """ + """Format documents for prompt.""" return "\n\n".join(doc.page_content for doc in docs) def create_qa_chain(retriever: Any, llm: ChatOpenAI, prompt: PromptTemplate): - """ - Set up question answering chain - """ + """Set up question answering chain.""" return ( { "context": retriever | format_docs, @@ -134,9 +122,7 @@ def create_qa_chain(retriever: Any, llm: ChatOpenAI, prompt: PromptTemplate): def get_parser() -> argparse.ArgumentParser: - """ - Parse command line arguments - """ + """Parse command line arguments.""" parser = argparse.ArgumentParser(description="RAG with vLLM and langchain") # Add command line arguments @@ -192,10 +178,7 @@ def get_parser() -> argparse.ArgumentParser: def init_config(args: Namespace): - """ - Initialize configuration settings from command line arguments - """ - + """Initialize configuration settings from command line arguments.""" return { "vllm_api_key": args.vllm_api_key, "vllm_embedding_endpoint": args.vllm_embedding_endpoint, diff --git a/examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py b/examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py index be4796acd1..0df32206bf 100644 --- a/examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py +++ b/examples/applications/rag/retrieval_augmented_generation_with_llamaindex.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -RAG (Retrieval Augmented Generation) Implementation with LlamaIndex +"""RAG (Retrieval Augmented Generation) Implementation with LlamaIndex ================================================================ This script demonstrates a RAG system using: @@ -35,6 +34,7 @@ Notes: - Ensure both vLLM services are running before executing - Default ports: 8000 (embedding), 8001 (chat) - First run may take time to download models + """ import argparse @@ -50,7 +50,7 @@ from llama_index.vector_stores.milvus import MilvusVectorStore def init_config(args: Namespace): - """Initialize configuration with command line arguments""" + """Initialize configuration with command line arguments.""" return { "url": args.url, "embedding_model": args.embedding_model, @@ -66,12 +66,12 @@ def init_config(args: Namespace): def load_documents(url: str) -> list: - """Load and process web documents""" + """Load and process web documents.""" return SimpleWebPageReader(html_to_text=True).load_data([url]) def setup_models(config: dict[str, Any]): - """Configure embedding and chat models""" + """Configure embedding and chat models.""" Settings.embed_model = OpenAILikeEmbedding( api_base=config["embedding_endpoint"], api_key=config["vllm_api_key"], @@ -96,14 +96,14 @@ def setup_models(config: dict[str, Any]): def setup_vector_store(db_path: str) -> MilvusVectorStore: - """Initialize vector store""" + """Initialize vector store.""" sample_emb = Settings.embed_model.get_text_embedding("test") print(f"Embedding dimension: {len(sample_emb)}") return MilvusVectorStore(uri=db_path, dim=len(sample_emb), overwrite=True) def create_index(documents: list, vector_store: MilvusVectorStore): - """Create document index""" + """Create document index.""" storage_context = StorageContext.from_defaults(vector_store=vector_store) return VectorStoreIndex.from_documents( documents, @@ -112,13 +112,13 @@ def create_index(documents: list, vector_store: MilvusVectorStore): def query_document(index: VectorStoreIndex, question: str, top_k: int): - """Query document with given question""" + """Query document with given question.""" query_engine = index.as_query_engine(similarity_top_k=top_k) return query_engine.query(question) def get_parser() -> argparse.ArgumentParser: - """Parse command line arguments""" + """Parse command line arguments.""" parser = argparse.ArgumentParser(description="RAG with vLLM and LlamaIndex") # Add command line arguments diff --git a/examples/deployment/async_llm_streaming.py b/examples/deployment/async_llm_streaming.py index ef69089a04..bf9b36d05c 100644 --- a/examples/deployment/async_llm_streaming.py +++ b/examples/deployment/async_llm_streaming.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Simple example demonstrating streaming offline inference with AsyncLLM (V1 engine). +"""Simple example demonstrating streaming offline inference with AsyncLLM (V1 engine). This script shows the core functionality of vLLM's AsyncLLM engine for streaming token-by-token output in offline inference scenarios. It demonstrates DELTA mode @@ -20,8 +19,7 @@ from vllm.v1.engine.async_llm import AsyncLLM async def stream_response(engine: AsyncLLM, prompt: str, request_id: str) -> None: - """ - Stream response from AsyncLLM and display tokens as they arrive. + """Stream response from AsyncLLM and display tokens as they arrive. This function demonstrates the core streaming pattern: 1. Create SamplingParams with DELTA output kind diff --git a/examples/deployment/llm_engine_example.py b/examples/deployment/llm_engine_example.py index d9215255a8..0d25a157b4 100644 --- a/examples/deployment/llm_engine_example.py +++ b/examples/deployment/llm_engine_example.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file demonstrates using the `LLMEngine` +"""This file demonstrates using the `LLMEngine` for processing prompts with various sampling parameters. """ diff --git a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py index 830da9b5d6..c44c902690 100644 --- a/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py +++ b/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -disagg_encoder_proxy.py +"""disagg_encoder_proxy.py Proxy that routes OpenAI-compatible “/v1/chat/completions” requests to two clusters: @@ -200,8 +199,7 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict: def extract_mm_items(request_data: dict) -> list[dict]: - """ - Return *all* image/audio/video items that appear anywhere in `messages`. + """Return *all* image/audio/video items that appear anywhere in `messages`. Each returned dict looks like: { "type": "image_url", "image_url": {...} } @@ -224,8 +222,7 @@ async def fanout_encoder_primer( req_id: str, consumer_zmq: str | None = None, ) -> tuple[dict[int, dict], dict[str, Any]]: - """ - 1. Build one request *per MM item* with all text removed. + """1. Build one request *per MM item* with all text removed. 2. Send them concurrently to the encode cluster. 3. Raise if any of them fails. @@ -402,8 +399,7 @@ async def maybe_prefill( p_url: str, req_id: str, ) -> dict: - """ - - Do prefill-only task if p_url exist; + """- Do prefill-only task if p_url exist; - Return a new body carrying kv transfer params (for nixl connector) - Else, skip and return the original request data for decode @@ -849,14 +845,14 @@ async def _post_if_available( payload: dict, headers: dict, ) -> dict | None: - """ - POST `payload` to `url`. + """POST `payload` to `url`. Returns ------- • The decoded JSON body on success (2xx) • None if the endpoint does not exist (404) • Raises for anything else. + """ try: resp = await session.post(url, json=payload, headers=headers) @@ -877,9 +873,7 @@ async def _post_if_available( async def _profile_cmd(cmd: str, payload: dict, e_url: str, p_url: str, d_url: str): - """ - Fire & forget to both clusters, tolerate 404. - """ + """Fire & forget to both clusters, tolerate 404.""" headers = {"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY', '')}"} encode_task = _post_if_available( diff --git a/examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py index 57deef6a15..ae35ec6d76 100644 --- a/examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_demo.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file provides a disaggregated prefilling proxy demo to demonstrate an +"""This file provides a disaggregated prefilling proxy demo to demonstrate an example usage of XpYd disaggregated prefilling. We can launch multiple vllm instances (2 for prefill and 2 for decode), and launch this proxy demo through: diff --git a/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py index cc1cc402d2..3102f344b0 100644 --- a/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_multiturn.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer +"""Disaggregated Prefill/Decode Proxy with Bidirectional KV Transfer. This proxy sits between clients and a vLLM Prefill/Decode (P/D) deployment, routing multi-turn chat requests so that each turn reuses KV cache blocks @@ -54,7 +53,7 @@ Benchmarking: it the benchmark sends OpenAI-schema-compliant payloads and every turn lands as a cache MISS in this proxy. - Example: +Example: python benchmarks/multi_turn/benchmark_serving_multi_turn.py \\ --model --served-model-name \\ --url http://:8000 \\ @@ -67,6 +66,7 @@ Benchmarking: Dependencies: pip install fastapi uvicorn httpx + """ from __future__ import annotations diff --git a/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py index a35a44fbba..53d8b210d6 100644 --- a/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Push-mode disaggregated prefilling proxy demo. +"""Push-mode disaggregated prefilling proxy demo. Companion to ``disagg_proxy_demo.py`` (pull mode). The client-facing API is the same; the difference is in how P and D coordinate the KV transfer: diff --git a/examples/disaggregated/example_connector/decode_example.py b/examples/disaggregated/example_connector/decode_example.py index 2d575840e6..9b76e2f001 100644 --- a/examples/disaggregated/example_connector/decode_example.py +++ b/examples/disaggregated/example_connector/decode_example.py @@ -6,7 +6,7 @@ from vllm.config import KVTransferConfig def read_prompts(): - """Read prompts from output.txt""" + """Read prompts from output.txt.""" prompts = [] try: with open("output.txt") as f: diff --git a/examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py b/examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py index b67c245931..55f4fe5200 100644 --- a/examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py +++ b/examples/disaggregated/flexkv_connector/prefix_caching_flexkv.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use FlexKV with vLLM for prefix caching. +"""This example shows how to use FlexKV with vLLM for prefix caching. FlexKV is a distributed KV Store and multi-level cache management system for ultra-large-scale LLM inference. diff --git a/examples/disaggregated/kv_load_failure_recovery_offline/decode_example.py b/examples/disaggregated/kv_load_failure_recovery_offline/decode_example.py index db9c5a85f7..3c3d5dbff9 100644 --- a/examples/disaggregated/kv_load_failure_recovery_offline/decode_example.py +++ b/examples/disaggregated/kv_load_failure_recovery_offline/decode_example.py @@ -7,7 +7,7 @@ from vllm.config import KVTransferConfig def read_prompts(): - """Read prompts from prefill_output.txt""" + """Read prompts from prefill_output.txt.""" prompts = [] try: with open("prefill_output.txt") as f: diff --git a/examples/disaggregated/kv_load_failure_recovery_offline/load_recovery_example_connector.py b/examples/disaggregated/kv_load_failure_recovery_offline/load_recovery_example_connector.py index 5bae044267..2b2e94aa74 100644 --- a/examples/disaggregated/kv_load_failure_recovery_offline/load_recovery_example_connector.py +++ b/examples/disaggregated/kv_load_failure_recovery_offline/load_recovery_example_connector.py @@ -119,8 +119,7 @@ class LoadRecoveryExampleConnector(ExampleConnector): def update_state_after_alloc( self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int ): - """ - Update KVConnector state after block allocation. + """Update KVConnector state after block allocation. If blocks were allocated, add to _requests_need_load, such that we load the KVs in the next forward pass. diff --git a/examples/disaggregated/lmcache/cpu_offload_lmcache.py b/examples/disaggregated/lmcache/cpu_offload_lmcache.py index b67a929e5d..8a84675952 100644 --- a/examples/disaggregated/lmcache/cpu_offload_lmcache.py +++ b/examples/disaggregated/lmcache/cpu_offload_lmcache.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file demonstrates the example usage of CPU offloading +"""This file demonstrates the example usage of CPU offloading with LMCache in vLLM v1. Note that `lmcache` is needed to run this example. diff --git a/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py index e048aecff9..3a64139c63 100644 --- a/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py +++ b/examples/disaggregated/lmcache/disagg_prefill_lmcache_v1/disagg_proxy_server.py @@ -14,9 +14,7 @@ from fastapi.responses import StreamingResponse @asynccontextmanager async def lifespan(app: FastAPI): - """ - Lifespan context manager to handle startup and shutdown events. - """ + """Lifespan context manager to handle startup and shutdown events.""" # Startup: Initialize clients prefiller_base_url = ( f"http://{global_args.prefiller_host}:{global_args.prefiller_port}/v1" @@ -106,9 +104,7 @@ app.state.decode_client = None async def send_request_to_service( client: httpx.AsyncClient, endpoint: str, req_data: dict ): - """ - Send a request to a service using a persistent client. - """ + """Send a request to a service using a persistent client.""" req_data = req_data.copy() req_data["max_tokens"] = 1 if "max_completion_tokens" in req_data: @@ -128,9 +124,7 @@ async def send_request_to_service( async def stream_service_response( client: httpx.AsyncClient, endpoint: str, req_data: dict ): - """ - Asynchronously stream the response from a service using a persistent client. - """ + """Asynchronously stream the response from a service using a persistent client.""" headers = {"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"} async with client.stream( "POST", endpoint, json=req_data, headers=headers diff --git a/examples/disaggregated/lmcache/kv_cache_sharing_lmcache_v1.py b/examples/disaggregated/lmcache/kv_cache_sharing_lmcache_v1.py index 489ff13212..a50d6cbbc3 100644 --- a/examples/disaggregated/lmcache/kv_cache_sharing_lmcache_v1.py +++ b/examples/disaggregated/lmcache/kv_cache_sharing_lmcache_v1.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file demonstrates the example usage of remote KV cache sharing +"""This file demonstrates the example usage of remote KV cache sharing with LMCache. We will launch 2 vllm instances, and launch an additional LMCache server. KV cache is transferred in the following manner: diff --git a/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py b/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py index 09880a32aa..6815f42061 100644 --- a/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py +++ b/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py @@ -65,9 +65,7 @@ async def get_prefiller_info(prefill_clients: list, ready: asyncio.Event): @asynccontextmanager async def lifespan(app: FastAPI): - """ - Lifespan context manager to handle startup and shutdown events. - """ + """Lifespan context manager to handle startup and shutdown events.""" # Startup: Initialize client pools for prefiller and decoder services app.state.prefill_clients = [] app.state.decode_clients = [] @@ -180,10 +178,12 @@ def _parse_prefill_urls(prefill_list): """Parse prefill URLs from --prefill arguments. Format: --prefill URL [BOOTSTRAP_PORT] + Example: --prefill http://prefill1:8080 9000 # With bootstrap port --prefill http://prefill2:8080 none # Explicitly no bootstrap port --prefill http://prefill3:8080 # Defaults to no bootstrap port + """ if not prefill_list: return [] @@ -228,8 +228,7 @@ def _parse_decode_urls(decode_list): def get_next_client(app, service_type: str): - """ - Get the next client in round-robin fashion. + """Get the next client in round-robin fashion. Args: app: The FastAPI app instance @@ -237,6 +236,7 @@ def get_next_client(app, service_type: str): Returns: The next client to use + """ if service_type == "prefill": return next(app.state.prefill_iterator) @@ -250,9 +250,7 @@ def get_next_client(app, service_type: str): async def send_request_to_service( client_info: dict, dp_rank: int, endpoint: str, req_data: dict, request_id: str ): - """ - Send a request to a service using a client from the pool. - """ + """Send a request to a service using a client from the pool.""" req_data = req_data.copy() req_data["kv_transfer_params"] = { "do_remote_decode": True, @@ -288,9 +286,7 @@ async def stream_service_response( req_data: dict, request_id: str, ): - """ - Asynchronously stream response from a service using a client from the pool. - """ + """Asynchronously stream response from a service using a client from the pool.""" headers = { "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}", "X-Request-Id": request_id, diff --git a/examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py b/examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py index 801b4b7697..db033e1861 100644 --- a/examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py +++ b/examples/features/automatic_prefix_caching/automatic_prefix_caching_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Demonstration script for Automatic Prefix Caching (APC) in vLLM. +"""Demonstration script for Automatic Prefix Caching (APC) in vLLM. Automatic Prefix Caching (APC) allows the vLLM engine to reuse cached KV (key-value) pairs from previous prompts if a new query shares the same diff --git a/examples/features/batch_invariance/reproducibility_offline.py b/examples/features/batch_invariance/reproducibility_offline.py index 72c1e841dc..54afcdee9f 100644 --- a/examples/features/batch_invariance/reproducibility_offline.py +++ b/examples/features/batch_invariance/reproducibility_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Demonstrates how to achieve reproducibility in vLLM. +"""Demonstrates how to achieve reproducibility in vLLM. Main article: https://docs.vllm.ai/en/latest/usage/reproducibility.html """ diff --git a/examples/features/context_extension/context_extension_offline.py b/examples/features/context_extension/context_extension_offline.py index 3874288b5e..cf270a2230 100644 --- a/examples/features/context_extension/context_extension_offline.py +++ b/examples/features/context_extension/context_extension_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This script demonstrates how to extend the context length +"""This script demonstrates how to extend the context length of a Qwen model using the YARN method (rope_parameters) and run a simple chat example. diff --git a/examples/features/data_parallel/data_parallel_offline.py b/examples/features/data_parallel/data_parallel_offline.py index c38ff7297a..86e61e04f0 100644 --- a/examples/features/data_parallel/data_parallel_offline.py +++ b/examples/features/data_parallel/data_parallel_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Usage: +"""Usage: Single node: python examples/features/data_parallel/data_parallel_offline.py \ --model="ibm-research/PowerMoE-3b" \ diff --git a/examples/features/kv_events/kv_events_subscriber.py b/examples/features/kv_events/kv_events_subscriber.py index 9c1b5138bc..91d3829c54 100644 --- a/examples/features/kv_events/kv_events_subscriber.py +++ b/examples/features/kv_events/kv_events_subscriber.py @@ -18,7 +18,7 @@ class EventBatch(msgspec.Struct, array_like=True, omit_defaults=True, gc=False): class KVCacheEvent(msgspec.Struct, omit_defaults=True, gc=False, tag=True): - """Base class for all KV cache-related events""" + """Base class for all KV cache-related events.""" class BlockStored(KVCacheEvent): diff --git a/examples/features/logits_processor/custom.py b/examples/features/logits_processor/custom.py index ce000872dc..c5028aa797 100644 --- a/examples/features/logits_processor/custom.py +++ b/examples/features/logits_processor/custom.py @@ -48,7 +48,7 @@ from vllm.v1.sample.logits_processor.builtin import process_dict_updates # Hypothetical custom logits processor class DummyLogitsProcessor(LogitsProcessor): - """Fake logit processor to support unit testing and examples""" + """Fake logit processor to support unit testing and examples.""" @classmethod def validate_params(cls, params: SamplingParams): diff --git a/examples/features/logits_processor/custom_req.py b/examples/features/logits_processor/custom_req.py index 5763fff541..eb57029efa 100644 --- a/examples/features/logits_processor/custom_req.py +++ b/examples/features/logits_processor/custom_req.py @@ -58,7 +58,7 @@ class DummyPerReqLogitsProcessor: token id identified by `target_token`""" def __init__(self, target_token: int) -> None: - """Specify `target_token`""" + """Specify `target_token`.""" self.target_token = target_token def __call__( @@ -103,6 +103,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor): Returns: `Callable` request logits processor, or None + """ target_token: Any | None = params.extra_args and params.extra_args.get( "target_token" diff --git a/examples/features/logits_processor/custom_req_init.py b/examples/features/logits_processor/custom_req_init.py index acd2c47f23..38554f06d8 100644 --- a/examples/features/logits_processor/custom_req_init.py +++ b/examples/features/logits_processor/custom_req_init.py @@ -59,7 +59,7 @@ class DummyPerReqLogitsProcessor: token id identified by `target_token`""" def __init__(self, target_token: int) -> None: - """Specify `target_token`""" + """Specify `target_token`.""" self.target_token = target_token def __call__( @@ -111,6 +111,7 @@ class WrappedPerReqLogitsProcessor(AdapterLogitsProcessor): Returns: `Callable` request logits processor, or None + """ if ( not self.is_cuda diff --git a/examples/features/lora/lora_with_quantization_offline.py b/examples/features/lora/lora_with_quantization_offline.py index ef6dbd4534..23f165cc5f 100644 --- a/examples/features/lora/lora_with_quantization_offline.py +++ b/examples/features/lora/lora_with_quantization_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use LoRA with different quantization techniques +"""This example shows how to use LoRA with different quantization techniques for offline inference. Requires HuggingFace credentials for access. @@ -73,7 +72,6 @@ def initialize_engine( model: str, quantization: str, lora_repo: str | None ) -> LLMEngine: """Initialize the LLMEngine.""" - engine_args = EngineArgs( model=model, quantization=quantization, @@ -86,7 +84,6 @@ def initialize_engine( def main(): """Main function that sets up and runs the prompt processing.""" - test_configs = [ # QLoRA (https://arxiv.org/abs/2305.14314) { diff --git a/examples/features/lora/multilora_offline.py b/examples/features/lora/multilora_offline.py index 92021f9fb2..d870ca7562 100644 --- a/examples/features/lora/multilora_offline.py +++ b/examples/features/lora/multilora_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use the multi-LoRA functionality +"""This example shows how to use the multi-LoRA functionality for offline inference. Requires HuggingFace credentials for access to Llama2. diff --git a/examples/features/pause_resume/data_parallel_pause_resume.py b/examples/features/pause_resume/data_parallel_pause_resume.py index 1f11536e53..fa26d0aa3c 100644 --- a/examples/features/pause_resume/data_parallel_pause_resume.py +++ b/examples/features/pause_resume/data_parallel_pause_resume.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test pause/resume with Data Parallel (DP) via HTTP API. +"""Test pause/resume with Data Parallel (DP) via HTTP API. This example demonstrates coordinated pause/resume across multiple DP ranks. The pause synchronizes across all DP engines via all-reduce. diff --git a/examples/features/prompt_embed/prompt_embed_offline.py b/examples/features/prompt_embed/prompt_embed_offline.py index 9e90aa46b5..2b5d3121c8 100644 --- a/examples/features/prompt_embed/prompt_embed_offline.py +++ b/examples/features/prompt_embed/prompt_embed_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Demonstrates how to generate prompt embeddings using +"""Demonstrates how to generate prompt embeddings using Hugging Face Transformers and use them as input to vLLM for both single and batch inference. diff --git a/examples/features/reset_kv/reset_kv_offline.py b/examples/features/reset_kv/reset_kv_offline.py index 3fbe7fa754..c19d88a35f 100644 --- a/examples/features/reset_kv/reset_kv_offline.py +++ b/examples/features/reset_kv/reset_kv_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file demonstrates preempt requests when using the `LLMEngine` +"""This file demonstrates preempt requests when using the `LLMEngine` for processing prompts with various sampling parameters. """ diff --git a/examples/features/sharded_state/load_sharded_state_offline.py b/examples/features/sharded_state/load_sharded_state_offline.py index e867db5d12..66d8a75107 100644 --- a/examples/features/sharded_state/load_sharded_state_offline.py +++ b/examples/features/sharded_state/load_sharded_state_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Validates the loading of a model saved with the sharded_state format. +"""Validates the loading of a model saved with the sharded_state format. This script demonstrates how to load a model that was previously saved using save_sharded_state_offline.py and validates it by running inference. Example usage: diff --git a/examples/features/sharded_state/save_sharded_state_offline.py b/examples/features/sharded_state/save_sharded_state_offline.py index 675f2e35a5..7ed0de1f10 100644 --- a/examples/features/sharded_state/save_sharded_state_offline.py +++ b/examples/features/sharded_state/save_sharded_state_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Saves each worker's model state dict directly to a checkpoint, which enables a +"""Saves each worker's model state dict directly to a checkpoint, which enables a fast load path for large tensor-parallel models where each worker only needs to read its own shard rather than the entire checkpoint. diff --git a/examples/features/speculative_decoding/mlpspeculator_offline.py b/examples/features/speculative_decoding/mlpspeculator_offline.py index 6a533eb5c9..b863b28222 100644 --- a/examples/features/speculative_decoding/mlpspeculator_offline.py +++ b/examples/features/speculative_decoding/mlpspeculator_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file demonstrates the usage of text generation with an LLM model, +"""This file demonstrates the usage of text generation with an LLM model, comparing the performance with and without speculative decoding. Note that this example is out of date and not supported in vLLM v1. diff --git a/examples/features/structured_outputs/structured_outputs_offline.py b/examples/features/structured_outputs/structured_outputs_offline.py index 6b6099f71b..e716935755 100644 --- a/examples/features/structured_outputs/structured_outputs_offline.py +++ b/examples/features/structured_outputs/structured_outputs_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file demonstrates the example usage of structured outputs +"""This file demonstrates the example usage of structured outputs in vLLM. It shows how to apply different constraints such as choice, regex, json schema, and grammar to produce structured and formatted results based on specific prompts. diff --git a/examples/features/torchrun/torchrun_dp_example_offline.py b/examples/features/torchrun/torchrun_dp_example_offline.py index f18f6042e9..1f8ac60377 100644 --- a/examples/features/torchrun/torchrun_dp_example_offline.py +++ b/examples/features/torchrun/torchrun_dp_example_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -experimental support for data-parallel inference with torchrun +"""experimental support for data-parallel inference with torchrun Note the data load balancing and distribution is done out of the vllm engine, no internal lb supported in external_launcher mode. diff --git a/examples/features/torchrun/torchrun_example_offline.py b/examples/features/torchrun/torchrun_example_offline.py index e41bcd420c..b4b7cbd6a9 100644 --- a/examples/features/torchrun/torchrun_example_offline.py +++ b/examples/features/torchrun/torchrun_example_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -experimental support for tensor-parallel inference with torchrun, +"""experimental support for tensor-parallel inference with torchrun, see https://github.com/vllm-project/vllm/issues/11400 for the motivation and use case for this example. run the script with `torchrun --nproc-per-node=4 torchrun_example_offline.py`, diff --git a/examples/generate/multimodal/audio_language_offline.py b/examples/generate/multimodal/audio_language_offline.py index a2e9b19756..8b2215d8c1 100644 --- a/examples/generate/multimodal/audio_language_offline.py +++ b/examples/generate/multimodal/audio_language_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference +"""This example shows how to use vLLM for running offline inference with the correct prompt format on audio language models. For most models, the prompt format should follow corresponding examples @@ -286,8 +285,7 @@ def run_minicpmo(question: str, audio_count: int) -> ModelRequestData: # Phi-4-multimodal-instruct def run_phi4mm(question: str, audio_count: int) -> ModelRequestData: - """ - Phi-4-multimodal-instruct supports both image and audio inputs. Here, we + """Phi-4-multimodal-instruct supports both image and audio inputs. Here, we show how to process audio inputs. """ model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct") diff --git a/examples/generate/multimodal/encoder_decoder_multimodal_offline.py b/examples/generate/multimodal/encoder_decoder_multimodal_offline.py index 1ab087de3f..b4ea893fc0 100644 --- a/examples/generate/multimodal/encoder_decoder_multimodal_offline.py +++ b/examples/generate/multimodal/encoder_decoder_multimodal_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference with +"""This example shows how to use vLLM for running offline inference with the explicit/implicit prompt format on enc-dec LMMs for text generation. """ @@ -56,8 +55,7 @@ def run_whisper(): def run_fireredasr2(): - """ - FireRedASR2 – Automatic Speech Recognition model. + """FireRedASR2 – Automatic Speech Recognition model. This model uses a Conformer encoder + Qwen2 LLM decoder architecture for speech-to-text transcription. Audio is passed via the implicit diff --git a/examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py b/examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py index e1c4fd76d7..60aba7abd0 100644 --- a/examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py +++ b/examples/generate/multimodal/openai_chat_completion_client_for_multimodal.py @@ -43,7 +43,6 @@ headers = {"User-Agent": "vLLM Example Client"} def encode_base64_content_from_url(content_url: str) -> str: """Encode a content retrieved from a remote url to base64 format.""" - with requests.get(content_url, headers=headers) as response: response.raise_for_status() result = base64.b64encode(response.content).decode("utf-8") @@ -53,7 +52,6 @@ def encode_base64_content_from_url(content_url: str) -> str: def encode_base64_content_from_file(file_path: str) -> str: """Encode a local file content to base64 format.""" - with open(file_path, "rb") as file: file_content = file.read() result = base64.b64encode(file_content).decode("utf-8") diff --git a/examples/generate/multimodal/qwen2_5_omni/only_thinker.py b/examples/generate/multimodal/qwen2_5_omni/only_thinker.py index 476ca91d9f..66bfd49003 100644 --- a/examples/generate/multimodal/qwen2_5_omni/only_thinker.py +++ b/examples/generate/multimodal/qwen2_5_omni/only_thinker.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference +"""This example shows how to use vLLM for running offline inference with the correct prompt format on Qwen2.5-Omni (thinker only). """ diff --git a/examples/generate/multimodal/qwen3_omni/only_thinker.py b/examples/generate/multimodal/qwen3_omni/only_thinker.py index 84dd5228e5..57e845ac4f 100644 --- a/examples/generate/multimodal/qwen3_omni/only_thinker.py +++ b/examples/generate/multimodal/qwen3_omni/only_thinker.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference +"""This example shows how to use vLLM for running offline inference with the correct prompt format on Qwen3-Omni (thinker only). """ diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py index 8b12df3991..cccb75d838 100644 --- a/examples/generate/multimodal/vision_language_multi_image_offline.py +++ b/examples/generate/multimodal/vision_language_multi_image_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference with +"""This example shows how to use vLLM for running offline inference with multi-image input on vision language models for text generation, using the chat template defined by the model. """ @@ -913,11 +912,9 @@ def load_phi3v(question: str, image_urls: list[str]) -> ModelRequestData: def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData: - """ - Phi-4-multimodal-instruct supports both image and audio inputs. Here, we + """Phi-4-multimodal-instruct supports both image and audio inputs. Here, we show how to process multi images inputs. """ - model_path = snapshot_download("microsoft/Phi-4-multimodal-instruct") # Since the vision-lora and speech-lora co-exist with the base model, # we have to manually specify the path of the lora weights. diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index 33e1a8a36b..7cd471d266 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference with +"""This example shows how to use vLLM for running offline inference with the correct prompt format on vision language models for text generation. For most models, the prompt format should follow corresponding examples @@ -1660,8 +1659,7 @@ def run_phi3v(questions: list[str], modality: str) -> ModelRequestData: # Phi-4-multimodal-instruct def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData: - """ - Phi-4-multimodal-instruct supports both image and audio inputs. Here, we + """Phi-4-multimodal-instruct supports both image and audio inputs. Here, we show how to process image inputs. """ assert modality == "image" @@ -2283,8 +2281,7 @@ MODELS_SUPPORT_VIT_CUDA_GRAPH = [ def get_multi_modal_input(args): - """ - return { + """Return { "data": image or video, "question": question, } diff --git a/examples/generate/trace_replay_offline.py b/examples/generate/trace_replay_offline.py index 2baa03307d..498a46cbd4 100644 --- a/examples/generate/trace_replay_offline.py +++ b/examples/generate/trace_replay_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Trace-replay with vLLM offline inference. +"""Trace-replay with vLLM offline inference. Trace-replay lets you supply a known sequence of decode token IDs alongside the prompt. Instead of sampling from the model distribution, the engine diff --git a/examples/pooling/classify/classification_with_lora_offline.py b/examples/pooling/classify/classification_with_lora_offline.py new file mode 100644 index 0000000000..a44066a653 --- /dev/null +++ b/examples/pooling/classify/classification_with_lora_offline.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Example of offline classification with a LoRA adapter.""" + +from vllm import LLM +from vllm.lora.request import LoRARequest +from vllm.transformers_utils.repo_utils import hf_api + + +def main(): + lora_path = hf_api().snapshot_download( + repo_id="AmirMohseni/skywork-qwen3-0.6b-reward-lora" + ) + llm = LLM( + model="Skywork/Skywork-Reward-V2-Qwen3-0.6B", + runner="pooling", + enable_lora=True, + max_lora_rank=8, + max_model_len=512, + ) + + prompt = "Which response is more helpful and correct?" + (output,) = llm.classify( + prompt, + lora_request=LoRARequest("reward-lora", 1, lora_path), + ) + print(f"Prompt: {prompt!r}") + print(f"Class Probabilities: {output.outputs.probs}") + + +if __name__ == "__main__": + main() diff --git a/examples/pooling/embed/openai_embedding_long_text/client.py b/examples/pooling/embed/openai_embedding_long_text/client.py index 7cc33b1f24..44e415f670 100644 --- a/examples/pooling/embed/openai_embedding_long_text/client.py +++ b/examples/pooling/embed/openai_embedding_long_text/client.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example script demonstrating long text embedding with chunked processing in vLLM. +"""Example script demonstrating long text embedding with chunked processing in vLLM. This example shows how to use vLLM's chunked processing feature to handle text inputs that exceed the model's maximum token length. The feature automatically diff --git a/examples/pooling/embed/vision_embedding_offline.py b/examples/pooling/embed/vision_embedding_offline.py index 2c82d0374d..5175a6cd46 100644 --- a/examples/pooling/embed/vision_embedding_offline.py +++ b/examples/pooling/embed/vision_embedding_offline.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -This example shows how to use vLLM for running offline inference with +"""This example shows how to use vLLM for running offline inference with the correct prompt format on vision language models for multimodal embedding. For most models, the prompt format should follow corresponding examples diff --git a/examples/pooling/embed/vision_embedding_online.py b/examples/pooling/embed/vision_embedding_online.py index 79d6e397c8..765528978b 100644 --- a/examples/pooling/embed/vision_embedding_online.py +++ b/examples/pooling/embed/vision_embedding_online.py @@ -37,8 +37,7 @@ def create_chat_embeddings( add_special_tokens: bool = False, padding: str | NotGiven = NOT_GIVEN, ) -> CreateEmbeddingResponse: - """ - Convenience function for accessing vLLM's Chat Embeddings API, + """Convenience function for accessing vLLM's Chat Embeddings API, which is an extension of OpenAI's existing Embeddings API. """ return client.post( @@ -56,13 +55,11 @@ def create_chat_embeddings( def run_clip(client: OpenAI, model: str): - """ - Start the server using: + """Start the server using: vllm serve openai/clip-vit-base-patch32 \ --runner pooling """ - response = create_chat_embeddings( client, messages=[ @@ -97,8 +94,7 @@ def run_clip(client: OpenAI, model: str): def run_dse_qwen2_vl(client: OpenAI, model: str): - """ - Start the server using: + """Start the server using: vllm serve MrLight/dse-qwen2-2b-mrl-v1 \ --runner pooling \ @@ -159,14 +155,12 @@ def run_dse_qwen2_vl(client: OpenAI, model: str): def run_qwen3_vl(client: OpenAI, model: str): - """ - Start the server using: + """Start the server using: vllm serve Qwen/Qwen3-VL-Embedding-2B \ --runner pooling \ --max-model-len 8192 """ - default_instruction = "Represent the user's input." print("Text embedding output:") @@ -266,14 +260,12 @@ def run_qwen3_vl(client: OpenAI, model: str): def run_siglip(client: OpenAI, model: str): - """ - Start the server using: + """Start the server using: vllm serve google/siglip-base-patch16-224 \ --runner pooling \ --chat-template template_basic.jinja """ - response = create_chat_embeddings( client, messages=[ @@ -311,8 +303,7 @@ def run_siglip(client: OpenAI, model: str): def run_vlm2vec(client: OpenAI, model: str): - """ - Start the server using: + """Start the server using: vllm serve TIGER-Lab/VLM2Vec-Full \ --runner pooling \ @@ -320,7 +311,6 @@ def run_vlm2vec(client: OpenAI, model: str): --max-model-len 4096 \ --chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja """ - response = create_chat_embeddings( client, messages=[ diff --git a/examples/pooling/plugin/prithvi_geospatial_mae_offline.py b/examples/pooling/plugin/prithvi_geospatial_mae_offline.py index f7b30d9313..6bbde74264 100644 --- a/examples/pooling/plugin/prithvi_geospatial_mae_offline.py +++ b/examples/pooling/plugin/prithvi_geospatial_mae_offline.py @@ -85,8 +85,7 @@ def generate_datamodule(): def process_channel_group(orig_img, channels): - """ - Args: + """Args: orig_img: torch.Tensor representing original image (reference) with shape = (bands, H, W). channels: list of indices representing RGB channels. @@ -94,8 +93,8 @@ def process_channel_group(orig_img, channels): Returns: torch.Tensor with shape (num_channels, height, width) for original image - """ + """ orig_img = orig_img[channels, ...] valid_mask = torch.ones_like(orig_img, dtype=torch.bool) valid_mask[orig_img == NO_DATA_FLOAT] = False @@ -121,8 +120,8 @@ def read_geotiff(file_path: str): Returns: np.ndarray with shape (bands, height, width) meta info dict - """ + """ with rasterio.open(file_path) as src: img = src.read() meta = src.meta @@ -142,8 +141,8 @@ def save_geotiff(image, output_path: str, meta: dict): image: np.ndarray with shape (bands, height, width) output_path: path where to save the image meta: dict with meta info. - """ + """ with rasterio.open(output_path, "w", **meta) as dest: for i in range(image.shape[0]): dest.write(image[i, :, :], i + 1) @@ -172,12 +171,14 @@ def load_example( images in *file_paths*. std: list containing std values for each band in the images in *file_paths*. + indices: optional band indices to select from each image. All bands + are kept when None. Returns: np.array containing created example list of meta info for each image in *file_paths* - """ + """ imgs = [] metas = [] temporal_coords = [] diff --git a/examples/pooling/reward/sequence_reward_offline.py b/examples/pooling/reward/sequence_reward_offline.py index 0727bceee1..4453f4a390 100644 --- a/examples/pooling/reward/sequence_reward_offline.py +++ b/examples/pooling/reward/sequence_reward_offline.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example offline usage of sequence reward models. +"""Example offline usage of sequence reward models. The key distinction between sequence classification and token classification lies in their output granularity: sequence classification produces a single diff --git a/examples/pooling/reward/sequence_reward_online.py b/examples/pooling/reward/sequence_reward_online.py index 40d8d28e39..c8362265dd 100644 --- a/examples/pooling/reward/sequence_reward_online.py +++ b/examples/pooling/reward/sequence_reward_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example online usage of sequence reward models. +"""Example online usage of sequence reward models. Run `vllm serve --runner pooling` to start up the server in vLLM. e.g. diff --git a/examples/pooling/reward/token_reward_offline.py b/examples/pooling/reward/token_reward_offline.py index 4705c04912..1820c390f8 100644 --- a/examples/pooling/reward/token_reward_offline.py +++ b/examples/pooling/reward/token_reward_offline.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example offline usage of token reward models. +"""Example offline usage of token reward models. The key distinction between sequence classification and token classification lies in their output granularity: sequence classification produces a single diff --git a/examples/pooling/reward/token_reward_online.py b/examples/pooling/reward/token_reward_online.py index 64ee0c9dfd..38f2394451 100644 --- a/examples/pooling/reward/token_reward_online.py +++ b/examples/pooling/reward/token_reward_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example online usage of token reward models. +"""Example online usage of token reward models. Run `vllm serve --runner pooling` to start up the server in vLLM. e.g. diff --git a/examples/pooling/score/cohere_rerank_client.py b/examples/pooling/score/cohere_rerank_client.py index b32209967b..e7bf720cac 100644 --- a/examples/pooling/score/cohere_rerank_client.py +++ b/examples/pooling/score/cohere_rerank_client.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example of using the OpenAI entrypoint's rerank API which is compatible with +"""Example of using the OpenAI entrypoint's rerank API which is compatible with the Cohere SDK: https://github.com/cohere-ai/cohere-python Note that `pip install cohere` is needed to run this example. diff --git a/examples/pooling/score/colbert_rerank_online.py b/examples/pooling/score/colbert_rerank_online.py index 4cc509b95a..b8bc05b9e0 100644 --- a/examples/pooling/score/colbert_rerank_online.py +++ b/examples/pooling/score/colbert_rerank_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example of using ColBERT late interaction models for reranking and scoring. +"""Example of using ColBERT late interaction models for reranking and scoring. ColBERT (Contextualized Late Interaction over BERT) uses per-token embeddings and MaxSim scoring for document reranking, providing better accuracy than diff --git a/examples/pooling/score/colmodernvbert_rerank_online.py b/examples/pooling/score/colmodernvbert_rerank_online.py index de827ae062..95ca36ed71 100644 --- a/examples/pooling/score/colmodernvbert_rerank_online.py +++ b/examples/pooling/score/colmodernvbert_rerank_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example of using ColModernVBERT late interaction model for reranking. +"""Example of using ColModernVBERT late interaction model for reranking. ColModernVBERT is a multi-modal ColBERT-style model combining a SigLIP vision encoder with a ModernBERT text encoder. It produces per-token diff --git a/examples/pooling/score/colqwen3_5_rerank_online.py b/examples/pooling/score/colqwen3_5_rerank_online.py index 00746634d5..a2cd635361 100644 --- a/examples/pooling/score/colqwen3_5_rerank_online.py +++ b/examples/pooling/score/colqwen3_5_rerank_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example of using ColQwen3.5 late interaction model for reranking. +"""Example of using ColQwen3.5 late interaction model for reranking. ColQwen3.5 is a multi-modal ColBERT-style model based on Qwen3.5. It produces per-token embeddings and uses MaxSim scoring for retrieval diff --git a/examples/pooling/score/colqwen3_rerank_online.py b/examples/pooling/score/colqwen3_rerank_online.py index 0e61531bfd..22d5fee079 100644 --- a/examples/pooling/score/colqwen3_rerank_online.py +++ b/examples/pooling/score/colqwen3_rerank_online.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Example of using ColQwen3 late interaction model for reranking and scoring. +"""Example of using ColQwen3 late interaction model for reranking and scoring. ColQwen3 is a multi-modal ColBERT-style model based on Qwen3-VL. It produces per-token embeddings and uses MaxSim scoring for retrieval diff --git a/examples/pooling/score/convert_model_to_seq_cls.py b/examples/pooling/score/convert_model_to_seq_cls.py index 21e2941664..494f683ae7 100644 --- a/examples/pooling/score/convert_model_to_seq_cls.py +++ b/examples/pooling/score/convert_model_to_seq_cls.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Script to convert Large Language Models (LLMs) to Sequence Classification models. +"""Script to convert Large Language Models (LLMs) to Sequence Classification models. This is particularly useful for converting reranker models that use next-token prediction to a sequence classification format for compatibility with standard classification and rerank pipelines. @@ -35,8 +34,7 @@ import transformers def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device): - """ - This method extracts the difference between weights for 'true' and 'false' tokens + """This method extracts the difference between weights for 'true' and 'false' tokens from the language model head to create a single classification weight vector. Args: @@ -47,6 +45,7 @@ def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device): device: Target device (cpu/cuda) Reference: https://huggingface.co/Qwen/Qwen3-Reranker-0.6B/discussions/3 + """ assert len(tokens) == 2, ( "Method requires exactly two tokens for binary classification" @@ -73,8 +72,7 @@ def from_2_way_softmax(causal_lm, seq_cls_model, tokenizer, tokens, device): def no_post_processing(causal_lm, seq_cls_model, tokenizer, tokens, device): - """ - Directly use token weights from the language model head for classification. + """Directly use token weights from the language model head for classification. This method maps each classification label directly to a corresponding token in the vocabulary without additional transformation. @@ -85,6 +83,7 @@ def no_post_processing(causal_lm, seq_cls_model, tokenizer, tokens, device): tokenizer: Model tokenizer tokens: List of tokens representing class labels device: Target device (cpu/cuda) + """ # Get the language model head weights (vocabulary_size x hidden_size) lm_head_weights = causal_lm.lm_head.weight @@ -110,8 +109,7 @@ method_map = { def converting( model_name, classifier_from_tokens, path, method, use_sep_token=False, device="cpu" ): - """ - Main conversion function to transform a CausalLM model to SequenceClassification. + """Main conversion function to transform a CausalLM model to SequenceClassification. Args: model_name: Name or path of the pretrained model @@ -120,6 +118,7 @@ def converting( method: Conversion method ('from_2_way_softmax' or 'no_post_processing') use_sep_token: Whether to use separating token in the sequence classification model device: Device to load the model on ('cpu' or 'cuda') + """ assert method in method_map, f"Unknown method: {method}" diff --git a/examples/pooling/score/qwen3_reranker_offline.py b/examples/pooling/score/qwen3_reranker_offline.py index c79ebf97fe..dbe8d47522 100644 --- a/examples/pooling/score/qwen3_reranker_offline.py +++ b/examples/pooling/score/qwen3_reranker_offline.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -What is the difference between the official original version and one +"""What is the difference between the official original version and one that has been converted into a sequence classification model? Qwen3-Reranker is a language model that doing reranker by using the @@ -34,8 +33,7 @@ model_name = "Qwen/Qwen3-Reranker-0.6B" def get_llm() -> LLM: - """ - Initializes and returns the LLM model for Qwen3-Reranker. + """Initializes and returns the LLM model for Qwen3-Reranker. Returns: LLM: Configured vLLM instance for reranking tasks. @@ -43,6 +41,7 @@ def get_llm() -> LLM: Note: This function loads the ORIGINAL Qwen3-Reranker model with specific overrides to make it compatible with vLLM's score API. + """ return LLM( # Specify the original model from HuggingFace diff --git a/examples/pooling/score/qwen3_reranker_online.py b/examples/pooling/score/qwen3_reranker_online.py index f117b3b5b8..324e3f5bc3 100644 --- a/examples/pooling/score/qwen3_reranker_online.py +++ b/examples/pooling/score/qwen3_reranker_online.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -What is the difference between the official original version and one +"""What is the difference between the official original version and one that has been converted into a sequence classification model? Qwen3-Reranker is a language model that doing reranker by using the diff --git a/examples/pooling/score/rerank_api_online.py b/examples/pooling/score/rerank_api_online.py index 908d6a9240..caf0586282 100644 --- a/examples/pooling/score/rerank_api_online.py +++ b/examples/pooling/score/rerank_api_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example of using the OpenAI entrypoint's rerank API which is compatible with +"""Example of using the OpenAI entrypoint's rerank API which is compatible with Jina and Cohere https://jina.ai/reranker run: vllm serve BAAI/bge-reranker-base diff --git a/examples/pooling/score/score_api_online.py b/examples/pooling/score/score_api_online.py index af2886f136..a60cf04bf8 100644 --- a/examples/pooling/score/score_api_online.py +++ b/examples/pooling/score/score_api_online.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example online usage of Score API. +"""Example online usage of Score API. Run `vllm serve --runner pooling` to start up the server in vLLM. """ diff --git a/examples/pooling/score/using_template_offline.py b/examples/pooling/score/using_template_offline.py index f434e699ff..abe8157d4c 100644 --- a/examples/pooling/score/using_template_offline.py +++ b/examples/pooling/score/using_template_offline.py @@ -68,13 +68,13 @@ def get_chat_template(model: str) -> str: def get_hf_overrides(model: str) -> dict[str, Any]: """Convert Large Language Models (LLMs) to Sequence Classification models. - note: + Note: Some reranking models require special configuration overrides to work correctly with vLLM's score API. Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_offline.py Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/convert_model_to_seq_cls.py - """ + """ model_name_to_hf_overrides_map = { "BAAI/bge-reranker-v2-gemma": { "architectures": ["GemmaForSequenceClassification"], @@ -122,7 +122,6 @@ def get_hf_overrides(model: str) -> dict[str, Any]: def main(args: Namespace): """Main execution function for the reranking example.""" - # Get the overrides for the specified model args.hf_overrides = get_hf_overrides(args.model) diff --git a/examples/pooling/score/using_template_online.py b/examples/pooling/score/using_template_online.py index f0bfa7d157..f7dc109dd8 100644 --- a/examples/pooling/score/using_template_online.py +++ b/examples/pooling/score/using_template_online.py @@ -1,15 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Example of using the rerank API with template. +"""Example of using the rerank API with template. This script demonstrates how to interact with a vLLM server running a reranking model via the REST API. Before running this script, start the vLLM server with one of the supported reranking models using the commands below. -note: +Note: Some reranking models require special configuration overrides to work correctly with vLLM's score API. Reference: https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_online.py @@ -21,6 +20,7 @@ run: vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}' --chat-template examples/pooling/score/template/mxbai_rerank_v2.jinja vllm serve nvidia/llama-nemotron-rerank-1b-v2 --runner pooling --trust-remote-code --chat-template examples/pooling/score/template/nemotron-rerank.jinja vllm serve Qwen/Qwen3-Reranker-0.6B --runner pooling --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}' --chat-template examples/pooling/score/template/qwen3_reranker.jinja + """ import json diff --git a/examples/pooling/score/vision_rerank_api_online.py b/examples/pooling/score/vision_rerank_api_online.py index dce2efd1dc..cc1f2e97bf 100644 --- a/examples/pooling/score/vision_rerank_api_online.py +++ b/examples/pooling/score/vision_rerank_api_online.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Example Python client for multimodal rerank API which is compatible with +"""Example Python client for multimodal rerank API which is compatible with Jina and Cohere https://jina.ai/reranker Run `vllm serve --runner pooling` to start up the server in vLLM. diff --git a/examples/pooling/score/vision_reranker_offline.py b/examples/pooling/score/vision_reranker_offline.py index ef2b354877..24c8b9068f 100644 --- a/examples/pooling/score/vision_reranker_offline.py +++ b/examples/pooling/score/vision_reranker_offline.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use vLLM for running offline inference with +"""This example shows how to use vLLM for running offline inference with vision language reranker models for multimodal scoring tasks. Vision language rerankers score the relevance between a text query and diff --git a/examples/pooling/score/vision_score_api_online.py b/examples/pooling/score/vision_score_api_online.py index 543d4bfa21..3a3ac62cf7 100644 --- a/examples/pooling/score/vision_score_api_online.py +++ b/examples/pooling/score/vision_score_api_online.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Example online usage of Score API. +"""Example online usage of Score API. Run `vllm serve --runner pooling` to start up the server in vLLM. e.g. diff --git a/examples/pooling/token_classify/forced_alignment_offline.py b/examples/pooling/token_classify/forced_alignment_offline.py index e97a373576..13facab480 100644 --- a/examples/pooling/token_classify/forced_alignment_offline.py +++ b/examples/pooling/token_classify/forced_alignment_offline.py @@ -3,8 +3,7 @@ # Adapted from Qwen3-ForcedAligner inference: # https://github.com/QwenLM/Qwen3-ASR -""" -Offline forced alignment example using Qwen3-ForcedAligner-0.6B. +"""Offline forced alignment example using Qwen3-ForcedAligner-0.6B. Forced alignment takes audio and reference text as input and produces word-level timestamps. The model predicts a time bin at each diff --git a/examples/pooling/token_classify/forced_alignment_online.py b/examples/pooling/token_classify/forced_alignment_online.py index e967a8ba70..ab3026160d 100644 --- a/examples/pooling/token_classify/forced_alignment_online.py +++ b/examples/pooling/token_classify/forced_alignment_online.py @@ -3,8 +3,7 @@ # Adapted from Qwen3-ForcedAligner inference: # https://github.com/QwenLM/Qwen3-ASR -""" -Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf. +"""Online forced alignment example using Qwen3-ForcedAligner-0.6B-hf. Forced alignment takes audio and reference text as input and produces word-level timestamps. The model predicts a time bin at each diff --git a/examples/pooling/token_classify/ner_online.py b/examples/pooling/token_classify/ner_online.py index 9ec2bd45a0..bd60d68626 100644 --- a/examples/pooling/token_classify/ner_online.py +++ b/examples/pooling/token_classify/ner_online.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from https://huggingface.co/boltuix/NeuroBERT-NER -""" -Example online usage of Pooling API for Named Entity Recognition (NER). +"""Example online usage of Pooling API for Named Entity Recognition (NER). Run `vllm serve --runner pooling` to start up the server in vLLM. e.g. diff --git a/examples/pooling/token_embed/colqwen3_token_embed_online.py b/examples/pooling/token_embed/colqwen3_token_embed_online.py index cac11188e8..c1a8f3a56e 100644 --- a/examples/pooling/token_embed/colqwen3_token_embed_online.py +++ b/examples/pooling/token_embed/colqwen3_token_embed_online.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Example online usage of Pooling API for ColQwen3 multi-vector retrieval. +"""Example online usage of Pooling API for ColQwen3 multi-vector retrieval. ColQwen3 is a multi-modal late interaction model based on Qwen3-VL that produces per-token embeddings (320-dim, L2-normalized) for both text and diff --git a/examples/pooling/token_embed/jina_reranker_v3_online.py b/examples/pooling/token_embed/jina_reranker_v3_online.py index 8350aee2f1..b9018f0f23 100644 --- a/examples/pooling/token_embed/jina_reranker_v3_online.py +++ b/examples/pooling/token_embed/jina_reranker_v3_online.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Example online usage of the Jina Reranker v3 score and rerank APIs with a task +"""Example online usage of the Jina Reranker v3 score and rerank APIs with a task instruction. Run `vllm serve jinaai/jina-reranker-v3 --runner pooling` to start up the diff --git a/examples/pooling/token_embed/multi_vector_retrieval_online.py b/examples/pooling/token_embed/multi_vector_retrieval_online.py index c2aae88294..f36e3dc31d 100644 --- a/examples/pooling/token_embed/multi_vector_retrieval_online.py +++ b/examples/pooling/token_embed/multi_vector_retrieval_online.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example online usage of Pooling API for multi vector retrieval. +"""Example online usage of Pooling API for multi vector retrieval. Run `vllm serve --runner pooling` to start up the server in vLLM. e.g. diff --git a/examples/ray_serving/batch_llm_inference.py b/examples/ray_serving/batch_llm_inference.py index 22408dc950..1b265efb17 100644 --- a/examples/ray_serving/batch_llm_inference.py +++ b/examples/ray_serving/batch_llm_inference.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This example shows how to use Ray Data for data parallel batch inference. +"""This example shows how to use Ray Data for data parallel batch inference. Ray Data is a data processing framework that can process very large datasets with first-class support for vLLM. diff --git a/examples/ray_serving/ray_serve_deepseek.py b/examples/ray_serving/ray_serve_deepseek.py index af53443b91..a1e9348115 100644 --- a/examples/ray_serving/ray_serve_deepseek.py +++ b/examples/ray_serving/ray_serve_deepseek.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Deploy DeepSeek R1 or V3 with Ray Serve LLM. +"""Deploy DeepSeek R1 or V3 with Ray Serve LLM. Ray Serve LLM is a scalable and production-grade model serving library built on the Ray distributed computing framework and first-class support for the vLLM engine. diff --git a/examples/reasoning/openai_chat_completion_tool_calls_with_reasoning.py b/examples/reasoning/openai_chat_completion_tool_calls_with_reasoning.py index 1dfc308464..53b30f534d 100644 --- a/examples/reasoning/openai_chat_completion_tool_calls_with_reasoning.py +++ b/examples/reasoning/openai_chat_completion_tool_calls_with_reasoning.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -An example demonstrates how to use tool calling with reasoning models +"""An example demonstrates how to use tool calling with reasoning models like QwQ-32B. The reasoning will not be parsed by the tool calling process; only the final output will be parsed. diff --git a/examples/reasoning/openai_chat_completion_with_reasoning.py b/examples/reasoning/openai_chat_completion_with_reasoning.py index 87043897b0..a72606108f 100644 --- a/examples/reasoning/openai_chat_completion_with_reasoning.py +++ b/examples/reasoning/openai_chat_completion_with_reasoning.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -An example shows how to generate chat completions from reasoning models +"""An example shows how to generate chat completions from reasoning models like DeepSeekR1. To run this example, you need to start the vLLM server diff --git a/examples/reasoning/openai_chat_completion_with_reasoning_streaming.py b/examples/reasoning/openai_chat_completion_with_reasoning_streaming.py index 8e262701b7..36f4c1f8e6 100644 --- a/examples/reasoning/openai_chat_completion_with_reasoning_streaming.py +++ b/examples/reasoning/openai_chat_completion_with_reasoning_streaming.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -An example shows how to generate chat completions from reasoning models +"""An example shows how to generate chat completions from reasoning models like DeepSeekR1. To run this example, you need to start the vLLM server with the reasoning diff --git a/examples/reasoning/openai_responses_client.py b/examples/reasoning/openai_responses_client.py index b4eb246715..94ecd84468 100644 --- a/examples/reasoning/openai_responses_client.py +++ b/examples/reasoning/openai_responses_client.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Set up this example by starting a vLLM OpenAI-compatible server. +"""Set up this example by starting a vLLM OpenAI-compatible server. Reasoning models can be used through the Responses API as seen here https://platform.openai.com/docs/api-reference/responses For example: diff --git a/examples/rl/rlhf_async_new_apis.py b/examples/rl/rlhf_async_new_apis.py index ea7099962f..081023156e 100644 --- a/examples/rl/rlhf_async_new_apis.py +++ b/examples/rl/rlhf_async_new_apis.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Demonstrates async reinforcement learning using vLLM and Ray, +"""Demonstrates async reinforcement learning using vLLM and Ray, with native weight syncing APIs and batch-invariant generation. The script separates training and inference workloads onto distinct GPUs diff --git a/examples/rl/rlhf_http_ipc.py b/examples/rl/rlhf_http_ipc.py index c688707df9..0ba9350216 100644 --- a/examples/rl/rlhf_http_ipc.py +++ b/examples/rl/rlhf_http_ipc.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the +"""RLHF weight syncing against a `vllm serve` HTTP server, using CUDA IPC for the data plane. * OpenAI-compatible API for inference requests diff --git a/examples/rl/rlhf_http_nccl.py b/examples/rl/rlhf_http_nccl.py index 97b0db0b0b..d095b647aa 100644 --- a/examples/rl/rlhf_http_nccl.py +++ b/examples/rl/rlhf_http_nccl.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the +"""RLHF weight syncing against a `vllm serve` HTTP server, using NCCL for the data plane. * OpenAI-compatible API for inference requests diff --git a/examples/rl/rlhf_ipc_fsdp_ep.py b/examples/rl/rlhf_ipc_fsdp_ep.py index e3fa1c44f1..4eb3fbcbbb 100644 --- a/examples/rl/rlhf_ipc_fsdp_ep.py +++ b/examples/rl/rlhf_ipc_fsdp_ep.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC** +"""RLHF with FSDP2 training and vLLM expert-parallel inference using **CUDA IPC** weight transfer and **packed** tensors. Multi-rank version of `rlhf_http_ipc.py`: the trainer is 4 FSDP2 Ray actors diff --git a/examples/rl/rlhf_nccl_fsdp_ep.py b/examples/rl/rlhf_nccl_fsdp_ep.py index e4380bea94..b9b72c012e 100644 --- a/examples/rl/rlhf_nccl_fsdp_ep.py +++ b/examples/rl/rlhf_nccl_fsdp_ep.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs). +"""RLHF with FSDP2 training (4 GPUs) and vLLM expert-parallel inference (4 GPUs). 8-GPU layout: Training — 4 GPUs, PyTorch FSDP2 (fully_shard), as Ray actors @@ -74,8 +73,7 @@ BASE_URL = f"http://localhost:{SERVER_PORT}" @ray.remote(num_gpus=1) class FSDPTrainWorker: - """ - One FSDP2 training worker per GPU. Four of these form the FSDP group. + """One FSDP2 training worker per GPU. Four of these form the FSDP group. Rank 0 additionally drives weight transfer to the vLLM server. """ diff --git a/examples/rl/routed_experts_e2e.py b/examples/rl/routed_experts_e2e.py index 1666bc3ffe..04b5871fd1 100644 --- a/examples/rl/routed_experts_e2e.py +++ b/examples/rl/routed_experts_e2e.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -End-to-end example for routed experts capture with hybrid models. +"""End-to-end example for routed experts capture with hybrid models. Validates that: 1. routed_experts is returned in CompletionOutput for MoE models. diff --git a/examples/speech_to_text/openai/openai_transcription_client.py b/examples/speech_to_text/openai/openai_transcription_client.py index f928c06d45..cc6757bd96 100644 --- a/examples/speech_to_text/openai/openai_transcription_client.py +++ b/examples/speech_to_text/openai/openai_transcription_client.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This script demonstrates how to use the vLLM API server to perform audio +"""This script demonstrates how to use the vLLM API server to perform audio transcription with the `openai/whisper-large-v3` model. Before running this script, you must start the vLLM server with the following command: @@ -35,8 +34,7 @@ def sync_openai( hotwords: str = None, prompt: str | None = None, ): - """ - Perform synchronous transcription using OpenAI-compatible API. + """Perform synchronous transcription using OpenAI-compatible API. The optional ``prompt`` is the OpenAI-API ``prompt`` field (style / vocabulary hint). It is wired through model-by-model: Whisper uses it @@ -69,9 +67,7 @@ async def stream_openai_response( hotwords: str = None, prompt: str | None = None, ): - """ - Perform asynchronous transcription using OpenAI-compatible API. - """ + """Perform asynchronous transcription using OpenAI-compatible API.""" print("\ntranscription result [stream]:", end=" ") with open(audio_path, "rb") as f: transcription = await client.audio.transcriptions.create( @@ -98,9 +94,7 @@ async def stream_openai_response( def stream_api_response(audio_path: str, model: str, openai_api_base: str): - """ - Perform streaming transcription using raw HTTP requests to the vLLM API server. - """ + """Stream a transcription using raw HTTP requests to the vLLM API server.""" import json import os diff --git a/examples/speech_to_text/realtime/openai_realtime_client.py b/examples/speech_to_text/realtime/openai_realtime_client.py index fda3d7cb45..a290b63e83 100644 --- a/examples/speech_to_text/realtime/openai_realtime_client.py +++ b/examples/speech_to_text/realtime/openai_realtime_client.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This script demonstrates how to use the vLLM Realtime WebSocket API to perform +"""This script demonstrates how to use the vLLM Realtime WebSocket API to perform audio transcription by uploading an audio file. Before running this script, you must start the vLLM server with a realtime-capable @@ -34,9 +33,7 @@ from vllm.multimodal.media.audio import load_audio def audio_to_pcm16_base64(audio_path: str) -> str: - """ - Load an audio file and convert it to base64-encoded PCM16 @ 16kHz. - """ + """Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.""" # Load audio and resample to 16kHz mono audio, _ = load_audio(audio_path, sr=16000, mono=True) # Convert to PCM16 @@ -46,9 +43,7 @@ def audio_to_pcm16_base64(audio_path: str) -> str: async def realtime_transcribe(audio_path: str, host: str, port: int, model: str): - """ - Connect to the Realtime API and transcribe an audio file. - """ + """Connect to the Realtime API and transcribe an audio file.""" uri = f"ws://{host}:{port}/v1/realtime" async with websockets.connect(uri) as ws: diff --git a/examples/speech_to_text/realtime/openai_realtime_microphone_client.py b/examples/speech_to_text/realtime/openai_realtime_microphone_client.py index a3c07673ff..c29486120c 100644 --- a/examples/speech_to_text/realtime/openai_realtime_microphone_client.py +++ b/examples/speech_to_text/realtime/openai_realtime_microphone_client.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API. +"""Minimal Gradio demo for real-time speech transcription using the vLLM Realtime API. Start the vLLM server first: diff --git a/examples/tool_calling/openai_chat_completion_client_with_tools.py b/examples/tool_calling/openai_chat_completion_client_with_tools.py index 0bd1d05322..508c6900b6 100644 --- a/examples/tool_calling/openai_chat_completion_client_with_tools.py +++ b/examples/tool_calling/openai_chat_completion_client_with_tools.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Set up this example by starting a vLLM OpenAI-compatible server with tool call +"""Set up this example by starting a vLLM OpenAI-compatible server with tool call options enabled. For example: IMPORTANT: for mistral, you must use one of the provided mistral tool call diff --git a/examples/tool_calling/openai_chat_completion_client_with_tools_required.py b/examples/tool_calling/openai_chat_completion_client_with_tools_required.py index c00d712b35..f38011e10d 100644 --- a/examples/tool_calling/openai_chat_completion_client_with_tools_required.py +++ b/examples/tool_calling/openai_chat_completion_client_with_tools_required.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -To run this example, you can start the vLLM server +"""To run this example, you can start the vLLM server without any specific flags: ```bash diff --git a/examples/tool_calling/openai_chat_completion_client_with_tools_xlam.py b/examples/tool_calling/openai_chat_completion_client_with_tools_xlam.py index f0b0a2db44..004b5fc7fc 100644 --- a/examples/tool_calling/openai_chat_completion_client_with_tools_xlam.py +++ b/examples/tool_calling/openai_chat_completion_client_with_tools_xlam.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Set up this example by starting a vLLM OpenAI-compatible server with tool call +"""Set up this example by starting a vLLM OpenAI-compatible server with tool call options enabled for xLAM-2 models: vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam @@ -105,8 +104,7 @@ tool_functions = { def process_response(response, tool_functions, original_query): - """Process a non-streaming response with possible tool calls""" - + """Process a non-streaming response with possible tool calls.""" print("\n--- Response Output ---") # Check if the response has content @@ -195,7 +193,7 @@ def process_response(response, tool_functions, original_query): def run_test_case(query, test_name): - """Run a single test case with the given query""" + """Run a single test case with the given query.""" print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}") print(f"Query: '{query}'") diff --git a/examples/tool_calling/openai_chat_completion_client_with_tools_xlam_streaming.py b/examples/tool_calling/openai_chat_completion_client_with_tools_xlam_streaming.py index 94e664c9ec..43e4ed21a0 100644 --- a/examples/tool_calling/openai_chat_completion_client_with_tools_xlam_streaming.py +++ b/examples/tool_calling/openai_chat_completion_client_with_tools_xlam_streaming.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 -""" -Set up this example by starting a vLLM OpenAI-compatible server with tool call +"""Set up this example by starting a vLLM OpenAI-compatible server with tool call options enabled for xLAM-2 models: vllm serve --model Salesforce/Llama-xLAM-2-8b-fc-r --enable-auto-tool-choice --tool-call-parser xlam @@ -107,7 +106,7 @@ tool_functions = { def process_stream(response, tool_functions, original_query): - """Process a streaming response with possible tool calls""" + """Process a streaming response with possible tool calls.""" # Track multiple tool calls tool_calls = {} # Dictionary to store tool calls by ID @@ -223,7 +222,7 @@ def process_stream(response, tool_functions, original_query): def run_test_case(query, test_name): - """Run a single test case with the given query""" + """Run a single test case with the given query.""" print(f"\n{'=' * 50}\nTEST CASE: {test_name}\n{'=' * 50}") print(f"Query: '{query}'") diff --git a/examples/tool_calling/openai_responses_client_with_mcp_tools.py b/examples/tool_calling/openai_responses_client_with_mcp_tools.py index 1e685a4f34..0b6ea412bf 100644 --- a/examples/tool_calling/openai_responses_client_with_mcp_tools.py +++ b/examples/tool_calling/openai_responses_client_with_mcp_tools.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Example demonstrating MCP (Model Context Protocol) tools with the Responses API. +"""Example demonstrating MCP (Model Context Protocol) tools with the Responses API. This example shows how to use MCP tools with different allowed_tools configurations: 1. No filter (allows all tools from the MCP server) diff --git a/examples/tool_calling/openai_responses_client_with_tools.py b/examples/tool_calling/openai_responses_client_with_tools.py index a47210f600..6530f331e4 100644 --- a/examples/tool_calling/openai_responses_client_with_tools.py +++ b/examples/tool_calling/openai_responses_client_with_tools.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Set up this example by starting a vLLM OpenAI-compatible server with tool call +"""Set up this example by starting a vLLM OpenAI-compatible server with tool call options enabled. Reasoning models can be used through the Responses API as seen here https://platform.openai.com/docs/api-reference/responses @@ -17,8 +16,7 @@ from openai import OpenAI def get_weather(latitude: float, longitude: float) -> str: - """ - Mock function to simulate getting weather data. + """Mock function to simulate getting weather data. In a real application, this would call an external weather API. """ return f"Current temperature at ({latitude}, {longitude}) is 20°C." diff --git a/pyproject.toml b/pyproject.toml index 1a77433f9c..b5c03b22b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,9 +60,13 @@ include = ["vllm*"] "vllm/third_party/**" = ["ALL"] "vllm/version.py" = ["F401"] "vllm/_version.py" = ["ALL"] +# test args are mostly pytest fixtures, which don't benefit from descriptions +"tests/**" = ["D417"] [tool.ruff.lint] select = [ + # pydocstyle + "D", # pycodestyle "E", # Pyflakes @@ -83,6 +87,12 @@ select = [ "INP", ] ignore = [ + # don't enforce that everything has a docstring + "D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107", + # unwanted pydocstyle rules + "D205", "D209", "D301", "D400", "D401", "D404", "D415", + # incompatible pydocstyle pairs; pin the half `ruff` already picks + "D203", "D213", # star imports "F405", "F403", # lambda expression assignment @@ -95,6 +105,9 @@ ignore = [ "UP032", ] +[tool.ruff.lint.pydocstyle] +ignore-var-parameters = true + [tool.ruff.format] docstring-code-format = true diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 052896c9f2..996caab96a 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -1174,7 +1174,7 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers -transformers==5.16.1 +transformers==5.17.0 # via # -r requirements/test/../common.txt # -r requirements/test/cuda.in diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index b16c64ed34..5e3faaebcc 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -39,7 +39,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_ datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.16.1 +transformers==5.17.0 schemathesis>=4.0.0 # Required for openai schema test. # quantization buildkite-test-collector==0.1.9 diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index db6e359182..ea94558000 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1281,7 +1281,7 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers -transformers==5.16.1 +transformers==5.17.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index 35c4156c5d..cebf3b782b 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -29,7 +29,7 @@ opencv-python-headless >= 4.13.0 # required for video test datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.16.1 +transformers==5.17.0 schemathesis>=4.0.0 # Required for openai schema test. # quantization buildkite-test-collector==0.1.9 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index 8397c98171..c8c08fabe5 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -35,7 +35,7 @@ open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_ datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test -transformers==5.16.1 +transformers==5.17.0 schemathesis>=4.0.0 # Required for openai schema test # quantization bitsandbytes==0.50.0 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index a6b9271989..631c3952fd 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -1233,7 +1233,7 @@ tqdm==4.67.3 # sentence-transformers # tilelang # transformers -transformers==5.16.1 +transformers==5.17.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index 3aab5bdb00..796e76eb44 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -19,7 +19,7 @@ lm_eval[api]>=0.4.12 modelscope<1.38 runai-model-streamer[s3,gcs,azure]>=0.15.7 tensorizer==2.10.1 -transformers==5.16.1 +transformers==5.17.0 # --- Audio Processing --- av diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 2e2fbd88ed..4e4fd628b6 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1081,7 +1081,7 @@ tqdm==4.70.0 # pqdm # sentence-transformers # transformers -transformers==5.16.1 +transformers==5.17.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/rust/proto/control.proto b/rust/proto/control.proto index d2816f68d3..ef3218d35c 100644 --- a/rust/proto/control.proto +++ b/rust/proto/control.proto @@ -68,6 +68,7 @@ message ParallelismInfo { uint32 data_parallel_rank = 4; uint32 decode_context_parallel_size = 5; uint64 world_size = 6; + uint32 data_parallel_size_local = 7; } message GetModelInfoRequest {} diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index de9c5ae65f..680ebdfbda 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -50,6 +50,7 @@ impl ControlServiceImpl { data_parallel_rank: ready.data_parallel_rank, decode_context_parallel_size: ready.decode_context_parallel_size, world_size: ready.world_size, + data_parallel_size_local: self.client().engine_count() as u32, } } diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index a21e013323..f4b35c9df4 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -31,7 +31,7 @@ use vllm_chat::{ }; use vllm_engine_core_client::mock_engine::{ DEFAULT_MOCK_BLOCK_SIZE, DEFAULT_MOCK_MAX_MODEL_LEN, DEFAULT_MOCK_NUM_GPU_BLOCKS, - default_ready_response, + MockEngineConfig, connect_to_bootstrapped_frontend, default_ready_response, }; use vllm_engine_core_client::protocol::decode_value; use vllm_engine_core_client::protocol::dtype::TensorDtype; @@ -1948,6 +1948,7 @@ async fn control_reports_server_and_model_info() { assert_eq!(parallelism.tensor_parallel_size, 1); assert_eq!(parallelism.pipeline_parallel_size, 1); assert_eq!(parallelism.data_parallel_size, 1); + assert_eq!(parallelism.data_parallel_size_local, 1); assert_eq!(parallelism.data_parallel_rank, 0); assert_eq!(parallelism.decode_context_parallel_size, 1); let rl = server.rl_capabilities.expect("RL capabilities"); @@ -2191,84 +2192,98 @@ async fn control_forwards_weight_update_without_pause_guard() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn control_aggregates_multi_engine_capacity() { - let ipc = IpcNamespace::new().expect("create ipc namespace"); - let handshake_address = ipc.handshake_endpoint(); + // Local capacity and ownership must not expand to the deployment-wide DP size. + for (start_rank, global_size) in [(0, 2), (0, 8), (4, 8)] { + let ipc = IpcNamespace::new().expect("create ipc namespace"); + let input_address = ipc.input_endpoint(); + let output_address = ipc.output_endpoint(); - let mut ready_0 = default_ready_response(); - ready_0.max_model_len = 8_192; - ready_0.num_gpu_blocks = 10; - ready_0.effective_data_parallel_size = 2; - ready_0.tensor_parallel_size = 2; - ready_0.pipeline_parallel_size = 3; - ready_0.world_size = 12; - ready_0.weight_transfer_backend = Some("nccl".to_string()); - ready_0.enable_sleep_mode = true; - ready_0.supports_draft_weight_updates = true; - ready_0.effective_attention_block_size = Some(64); + let mut ready_0 = default_ready_response(); + ready_0.max_model_len = 8_192; + ready_0.num_gpu_blocks = 10; + ready_0.effective_data_parallel_size = global_size as u64; + ready_0.data_parallel_rank = start_rank; + ready_0.tensor_parallel_size = 2; + ready_0.pipeline_parallel_size = 3; + ready_0.world_size = 12; + ready_0.weight_transfer_backend = Some("nccl".to_string()); + ready_0.enable_sleep_mode = true; + ready_0.supports_draft_weight_updates = true; + ready_0.effective_attention_block_size = Some(64); - let mut ready_1 = default_ready_response(); - ready_1.max_model_len = 4_096; - ready_1.num_gpu_blocks = 20; - ready_1.effective_data_parallel_size = 2; - ready_1.tensor_parallel_size = 2; - ready_1.pipeline_parallel_size = 3; - ready_1.world_size = 12; - ready_1.data_parallel_rank = 1; + let mut ready_1 = default_ready_response(); + ready_1.max_model_len = 4_096; + ready_1.num_gpu_blocks = 20; + ready_1.effective_data_parallel_size = global_size as u64; + ready_1.tensor_parallel_size = 2; + ready_1.pipeline_parallel_size = 3; + ready_1.world_size = 12; + ready_1.data_parallel_rank = start_rank + 1; - let engine_tasks = [ready_0, ready_1].map(|ready| { - let engine_id = EngineId::from_engine_index( - ready.data_parallel_rank.try_into().expect("test rank fits engine identity"), + let client_config = EngineCoreClientConfig { + transport_mode: TransportMode::Bootstrapped { + input_address: input_address.clone(), + output_address: output_address.clone(), + engine_start_index: start_rank, + engine_count: 2, + data_parallel_size: global_size, + ready_timeout: Duration::from_secs(2), + }, + coordinator_mode: None, + model_name: "test-model".to_string(), + client_index: 0, + }; + let client_task = tokio::spawn(EngineCoreClient::connect(client_config)); + let mut engine_sockets = Vec::new(); + for ready in [ready_0, ready_1] { + let engine_id = EngineId::from_engine_index( + ready.data_parallel_rank.try_into().expect("test rank fits engine identity"), + ); + engine_sockets.push( + connect_to_bootstrapped_frontend( + &input_address, + &output_address, + engine_id, + MockEngineConfig { + ready_response: ready, + ..Default::default() + }, + ) + .await + .expect("connect mock engine"), + ); + } + let client = client_task.await.unwrap().expect("connect multi-engine client"); + let chat = ChatLlm::from_shared_backend( + Llm::new(client), + Arc::new(FakeTextBackend) as Arc, ); - MockEngineTask::new(spawn_mock_engine_task_with_ready( - handshake_address.clone(), - engine_id, - ready, - |_, _| boxed_test_future(async {}), - )) - }); + let state = AppState::new(vec!["test-model".to_string()], chat); + let service = ControlServiceImpl::new(Arc::new(state)); - let client = EngineCoreClient::connect(EngineCoreClientConfig { - transport_mode: TransportMode::HandshakeOwner { - handshake_address, - advertised_host: "127.0.0.1".to_string(), - engine_count: 2, - ready_timeout: Duration::from_secs(2), - local_input_address: Some(ipc.input_endpoint()), - local_output_address: Some(ipc.output_endpoint()), - }, - coordinator_mode: None, - model_name: "test-model".to_string(), - client_index: 0, - }) - .await - .expect("connect multi-engine client"); - let chat = ChatLlm::from_shared_backend( - Llm::new(client), - Arc::new(FakeTextBackend) as Arc, - ); - let state = AppState::new(vec!["test-model".to_string()], chat); - let service = ControlServiceImpl::new(Arc::new(state)); + let server = pb::control_server::Control::get_server_info( + &service, + tonic::Request::new(pb::GetServerInfoRequest {}), + ) + .await + .expect("get server info") + .into_inner(); + assert_eq!(server.max_model_len, 4_096); + assert_eq!(server.total_kv_blocks, 30); + assert!(server.effective_attention_block_size.is_none()); + let rl = server.rl_capabilities.expect("RL capabilities"); + assert!(!rl.weight_transfer_enabled); + assert!(rl.weight_transfer_backend.is_empty()); + assert!(!rl.sleep_mode_enabled); + assert!(!rl.draft_weight_updates_enabled); + let parallelism = server.parallelism.unwrap(); + assert_eq!(parallelism.data_parallel_size, global_size as u32); + assert_eq!(parallelism.data_parallel_rank, start_rank); + assert_eq!(parallelism.data_parallel_size_local, 2); + assert_eq!(parallelism.world_size, 12); - let server = pb::control_server::Control::get_server_info( - &service, - tonic::Request::new(pb::GetServerInfoRequest {}), - ) - .await - .expect("get server info") - .into_inner(); - assert_eq!(server.max_model_len, 4_096); - assert_eq!(server.total_kv_blocks, 30); - assert!(server.effective_attention_block_size.is_none()); - let rl = server.rl_capabilities.expect("RL capabilities"); - assert!(!rl.weight_transfer_enabled); - assert!(rl.weight_transfer_backend.is_empty()); - assert!(!rl.sleep_mode_enabled); - assert!(!rl.draft_weight_updates_enabled); - let parallelism = server.parallelism.unwrap(); - assert_eq!(parallelism.data_parallel_size, 2); - assert_eq!(parallelism.world_size, 12); - - drop(engine_tasks); + drop(engine_sockets); + } } #[test] diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 308ec3ca3f..dfb7e6cbb1 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -21,6 +21,7 @@ use thiserror_ext::AsReport as _; use tracing::{error, info, trace}; use tracing_futures::Instrument as _; use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs}; +use vllm_engine_core_client::protocol::multimodal::MmFeatureSpec; use vllm_llm::{ CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStreamExt as _, TokenUsage, }; @@ -28,7 +29,7 @@ use vllm_llm::{ use self::convert::{ResponseOptions, prepare_generate_request}; use self::types::{ GenerateLogprob, GenerateResponse, GenerateResponseChoice, GenerateResponseStreamChoice, - GenerateStreamResponse, + GenerateStreamResponse, MultiModalPlaceholders, PlaceholderRangeInfo, }; pub(crate) use self::types::{GenerateRequest, GenerateSamplingParams}; pub(crate) use self::validate::validate_request_compat; @@ -64,6 +65,11 @@ pub async fn generate( } else { None }; + let mm_placeholders = body + .return_token_ids + .unwrap_or(false) + .then(|| extract_mm_placeholders(mm_features.as_deref())) + .flatten(); let prepared = match prepare_generate_request(body, &lora_resolution, request_context, mm_features) { @@ -98,6 +104,7 @@ pub async fn generate( prepared.request_id, api_server_options, prepared.options, + mm_placeholders, ); let sse_stream = generate_sse_stream(chunk_stream).instrument(request_span); @@ -120,6 +127,7 @@ pub async fn generate( prepared.request_id, api_server_options, prepared.options, + mm_placeholders, ) { Ok(response) => response, Err(error) => return error.into_response(), @@ -143,19 +151,26 @@ async fn generate_chunk_stream( include_logprobs, // Ignored: raw generate streaming has no prompt-logprobs wire shape. include_prompt_logprobs: _, + return_token_ids, }: ResponseOptions, + mut mm_placeholders: Option, mut y: TryYielder, ) -> Result<(), ApiError> { pin_mut!(stream); let mut prompt_tokens = None; + let mut prompt_token_ids = None; let mut usage = TokenUsage::default(); while let Some(next) = stream.next().await { match next { Ok(output) => { - if prompt_tokens.is_none() { - prompt_tokens = - output.prompt_info.as_ref().map(|info| info.prompt_token_ids.len()); + if prompt_tokens.is_none() + && let Some(info) = output.prompt_info.as_ref() + { + prompt_tokens = Some(info.prompt_token_ids.len()); + if return_token_ids { + prompt_token_ids = Some(info.prompt_token_ids.to_vec()); + } } usage.prompt_token_count = prompt_tokens.unwrap_or_default(); usage.cached_token_count = usage.cached_token_count.max(output.cached_token_count); @@ -195,6 +210,7 @@ async fn generate_chunk_stream( None }; + let prompt_token_ids = prompt_token_ids.take(); y.yield_ok(GenerateStreamResponse { request_id: request_id.clone(), choices: vec![GenerateResponseStreamChoice { @@ -205,6 +221,8 @@ async fn generate_chunk_stream( }], usage: include_continuous_usage .then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)), + mm_placeholders: prompt_token_ids.as_ref().and_then(|_| mm_placeholders.take()), + prompt_token_ids, }) .await; } @@ -223,6 +241,8 @@ async fn generate_chunk_stream( request_id, choices: Vec::new(), usage: Some(Usage::from_token_usage(usage, enable_prompt_tokens_details)), + prompt_token_ids: None, + mm_placeholders: None, }) .await; } @@ -244,7 +264,9 @@ fn collect_generate( include_continuous_usage: _, include_logprobs, include_prompt_logprobs, + return_token_ids, }: ResponseOptions, + mm_placeholders: Option, ) -> Result { let logprobs = if include_logprobs { let logprobs = collected.logprobs.as_ref().ok_or_else(|| { @@ -292,11 +314,34 @@ fn collect_generate( token_ids: collected.token_ids, }], prompt_logprobs, + prompt_token_ids: return_token_ids.then_some(collected.prompt_token_ids), + mm_placeholders: return_token_ids.then_some(mm_placeholders).flatten(), kv_transfer_params: collected.kv_transfer_params, ec_transfer_params: collected.ec_transfer_params, }) } +fn extract_mm_placeholders(features: Option<&[MmFeatureSpec]>) -> Option { + let features = features?; + if features.is_empty() { + return None; + } + + let mut placeholders = MultiModalPlaceholders::new(); + for feature in features { + placeholders.entry(feature.modality.as_str().to_string()).or_default().push( + PlaceholderRangeInfo { + offset: feature.mm_position.offset, + length: feature.mm_position.length, + }, + ); + } + for ranges in placeholders.values_mut() { + ranges.sort_by_key(|range| range.offset); + } + Some(placeholders) +} + fn raw_logprobs_to_openai_chat(logprobs: &Logprobs) -> Result { let content = logprobs .positions @@ -419,6 +464,7 @@ mod tests { use std::sync::Arc; use futures::{TryStreamExt as _, stream}; + use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange}; use vllm_llm::GeneratePromptInfo; use super::*; @@ -451,6 +497,13 @@ mod tests { }), ]); + let mm_placeholders = MultiModalPlaceholders::from([( + "image".to_string(), + vec![PlaceholderRangeInfo { + offset: 1, + length: 4, + }], + )]); let chunks: Vec<_> = generate_chunk_stream( stream, "raw-stream".to_string(), @@ -461,14 +514,20 @@ mod tests { ResponseOptions { include_usage: true, include_continuous_usage: true, + return_token_ids: true, ..Default::default() }, + Some(mm_placeholders.clone()), ) .try_collect() .await .expect("collect chunks"); assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].prompt_token_ids, Some(vec![11, 22])); + assert_eq!(chunks[0].mm_placeholders, Some(mm_placeholders)); + assert!(chunks[1].prompt_token_ids.is_none()); + assert!(chunks[1].mm_placeholders.is_none()); assert_eq!( chunks[0].usage.as_ref().expect("chunk usage").prompt_tokens, 2 @@ -499,6 +558,209 @@ mod tests { ); } + fn stream_output( + prompt_token_ids: Option<&[u32]>, + token_ids: Vec, + finish_reason: Option, + ) -> GenerateOutput { + GenerateOutput { + request_id: String::new(), + prompt_info: prompt_token_ids.map(|ids| GeneratePromptInfo { + prompt_token_ids: Arc::from(ids), + prompt_logprobs: None, + }), + token_ids, + logprobs: None, + finish_reason, + cached_token_count: 0, + kv_transfer_params: None, + ec_transfer_params: None, + } + } + + async fn collect_chunks( + outputs: Vec, + return_token_ids: bool, + mm_placeholders: Option, + ) -> Vec { + generate_chunk_stream( + stream::iter(outputs.into_iter().map(Ok)), + "raw-stream".to_string(), + ApiServerOptions::default(), + ResponseOptions { + return_token_ids, + ..Default::default() + }, + mm_placeholders, + ) + .try_collect() + .await + .expect("collect chunks") + } + + #[tokio::test] + async fn generate_chunk_stream_omits_prompt_metadata_by_default() { + let chunks = collect_chunks( + vec![ + stream_output(Some(&[11, 22]), vec![33], None), + stream_output(Some(&[11, 22]), vec![44], Some(FinishReason::stop_eos())), + ], + false, + Some(MultiModalPlaceholders::from([( + "image".to_string(), + vec![PlaceholderRangeInfo { + offset: 1, + length: 4, + }], + )])), + ) + .await; + + assert_eq!(chunks.len(), 2); + assert!(chunks.iter().all(|c| c.prompt_token_ids.is_none())); + assert!(chunks.iter().all(|c| c.mm_placeholders.is_none())); + } + + #[tokio::test] + async fn generate_chunk_stream_sends_prompt_metadata_on_first_emitted_chunk() { + // An empty first output is skipped and must not consume the metadata. + let chunks = collect_chunks( + vec![ + stream_output(Some(&[11, 22]), Vec::new(), None), + stream_output(Some(&[11, 22]), vec![33], None), + stream_output(Some(&[11, 22]), vec![44], Some(FinishReason::stop_eos())), + ], + true, + None, + ) + .await; + + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].prompt_token_ids, Some(vec![11, 22])); + assert!(chunks[0].mm_placeholders.is_none()); + assert!(chunks[1].prompt_token_ids.is_none()); + assert!(chunks[1].mm_placeholders.is_none()); + } + + #[test] + fn collect_generate_omits_prompt_metadata_by_default() { + let output = CollectedGenerateOutput { + request_id: "raw-1".to_string(), + prompt_logprobs: None, + token_ids: vec![30], + logprobs: None, + finish_reason: FinishReason::stop_eos(), + usage: TokenUsage { + prompt_token_count: 2, + output_token_count: 1, + cached_token_count: 0, + }, + kv_transfer_params: None, + ec_transfer_params: None, + prompt_token_ids: vec![10, 20], + }; + + let response = collect_generate( + output, + "raw-1".to_string(), + ApiServerOptions::default(), + ResponseOptions::default(), + Some(MultiModalPlaceholders::from([( + "image".to_string(), + vec![PlaceholderRangeInfo { + offset: 0, + length: 1, + }], + )])), + ) + .expect("response"); + + assert!(response.prompt_token_ids.is_none()); + assert!(response.mm_placeholders.is_none()); + } + + #[test] + fn extract_mm_placeholders_returns_none_without_features() { + assert!(extract_mm_placeholders(None).is_none()); + assert!(extract_mm_placeholders(Some(&[])).is_none()); + } + + #[test] + fn extract_mm_placeholders_groups_ranges_in_prompt_order() { + let feature = |modality: MmModality, offset: usize, length: usize| MmFeatureSpec { + data: None, + modality, + identifier: format!("{}-{offset}", modality.as_str()), + mm_position: PlaceholderRange { + offset, + length, + is_embed: None, + }, + mm_hash: None, + }; + let features = vec![ + feature(MmModality::Image, 8, 2), + feature(MmModality::Audio, 4, 3), + feature(MmModality::Image, 1, 5), + ]; + + let placeholders = extract_mm_placeholders(Some(&features)).expect("metadata"); + + assert_eq!( + placeholders["image"], + vec![ + PlaceholderRangeInfo { + offset: 1, + length: 5, + }, + PlaceholderRangeInfo { + offset: 8, + length: 2, + }, + ] + ); + assert_eq!( + placeholders["audio"], + vec![PlaceholderRangeInfo { + offset: 4, + length: 3, + }] + ); + } + + #[test] + fn collect_generate_returns_prompt_token_ids_when_requested() { + let output = CollectedGenerateOutput { + request_id: "raw-1".to_string(), + prompt_logprobs: None, + token_ids: vec![30], + logprobs: None, + finish_reason: FinishReason::stop_eos(), + usage: TokenUsage { + prompt_token_count: 2, + output_token_count: 1, + cached_token_count: 0, + }, + kv_transfer_params: None, + ec_transfer_params: None, + prompt_token_ids: vec![10, 20], + }; + + let response = collect_generate( + output, + "raw-1".to_string(), + ApiServerOptions::default(), + ResponseOptions { + return_token_ids: true, + ..Default::default() + }, + None, + ) + .expect("response"); + + assert_eq!(response.prompt_token_ids, Some(vec![10, 20])); + } + #[test] fn collect_generate_maps_prompt_logprobs_for_single_token_prompt() { let output_without_payload = |prompt_token_ids: Vec| CollectedGenerateOutput { @@ -525,6 +787,7 @@ mod tests { include_prompt_logprobs: true, ..Default::default() }, + None, ) .expect("single-token prompt without payload maps to [None]"); let prompt_logprobs = response.prompt_logprobs.expect("prompt logprobs present"); @@ -539,6 +802,7 @@ mod tests { include_prompt_logprobs: true, ..Default::default() }, + None, ) .expect_err("multi-token prompt without payload is an engine failure"); } diff --git a/rust/src/server/src/routes/inference/generate/convert.rs b/rust/src/server/src/routes/inference/generate/convert.rs index 08c4ce18be..8093a34ca2 100644 --- a/rust/src/server/src/routes/inference/generate/convert.rs +++ b/rust/src/server/src/routes/inference/generate/convert.rs @@ -30,6 +30,8 @@ pub(super) struct ResponseOptions { pub include_logprobs: bool, /// Whether the caller requested top-level prompt logprobs. pub include_prompt_logprobs: bool, + /// Whether the caller requested final prompt token metadata. + pub return_token_ids: bool, } /// Validate and lower one raw generate request into the internal @@ -56,6 +58,7 @@ pub(super) fn prepare_generate_request( .unwrap_or(false); let include_logprobs = request.sampling_params.inner.logprobs.is_some(); let include_prompt_logprobs = request.sampling_params.inner.prompt_logprobs.is_some(); + let return_token_ids = request.return_token_ids.unwrap_or(false); let mut sampling_params = request.sampling_params.inner; sampling_params.vllm_xargs = merge_kv_transfer_params( sampling_params.vllm_xargs, @@ -93,6 +96,7 @@ pub(super) fn prepare_generate_request( include_continuous_usage, include_logprobs, include_prompt_logprobs, + return_token_ids, }, }) } diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index b9220aa7b7..406ec58569 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -46,6 +46,7 @@ pub struct GenerateRequest { pub ec_transfer_params: Option>, /// Raw multimodal input; server resolves media. Mutually exclusive with `features`. pub content_parts: Option>, + pub return_token_ids: Option, #[serde(flatten)] pub other: Map, } @@ -81,6 +82,8 @@ pub(super) struct GenerateStreamResponse { pub request_id: String, pub choices: Vec, pub usage: Option, + pub prompt_token_ids: Option>, + pub mm_placeholders: Option, } /// Mirrors the Python vLLM `GenerateResponse` class. @@ -89,10 +92,20 @@ pub(super) struct GenerateResponse { pub request_id: String, pub choices: Vec, pub prompt_logprobs: Option>>>, + pub prompt_token_ids: Option>, + pub mm_placeholders: Option, pub kv_transfer_params: Option, pub ec_transfer_params: Option, } +pub(super) type MultiModalPlaceholders = HashMap>; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub(super) struct PlaceholderRangeInfo { + pub offset: usize, + pub length: usize, +} + /// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads. #[derive(Debug, Clone, Serialize)] pub(super) struct GenerateLogprob { diff --git a/rust/src/server/src/routes/render.rs b/rust/src/server/src/routes/render.rs index a4ae0298e9..2d38b63f49 100644 --- a/rust/src/server/src/routes/render.rs +++ b/rust/src/server/src/routes/render.rs @@ -105,6 +105,7 @@ fn lower_render_request( kv_transfer_params: None, ec_transfer_params: None, content_parts: None, + return_token_ids: None, other: Default::default(), }; validate_generate_request(&request, &state.served_model_names)?; diff --git a/scripts/autotune_helion_kernels.py b/scripts/autotune_helion_kernels.py index 864aabfc09..e6352a40af 100644 --- a/scripts/autotune_helion_kernels.py +++ b/scripts/autotune_helion_kernels.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Autotune registered Helion kernels for optimal configurations. +"""Autotune registered Helion kernels for optimal configurations. Usage: # Autotune all registered kernels diff --git a/scripts/benchmark_helion_kernels.py b/scripts/benchmark_helion_kernels.py index fea30d6f9c..04350896af 100644 --- a/scripts/benchmark_helion_kernels.py +++ b/scripts/benchmark_helion_kernels.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Benchmark a registered Helion kernel against a baseline. +"""Benchmark a registered Helion kernel against a baseline. For each input case produced by the kernel's registered input generator, this checks the Helion kernel's numerics once against its eager reference, measures @@ -394,6 +393,7 @@ def check_kernel_correctness( ``passed=False`` with the exception text in ``error``; iteration continues regardless. An empty input mapping returns an empty list, which the CLI reports as a skipped check. + """ if inputs_dict is None: inputs_dict = kernel.get_inputs() diff --git a/setup.py b/setup.py index 4012000e63..b24bcc1a2c 100644 --- a/setup.py +++ b/setup.py @@ -512,8 +512,7 @@ class precompiled_wheel_utils: *, rocm: bool = False, ) -> tuple[list[dict], str]: - """ - Fetches metadata for a specific variant of the precompiled wheel. + """Fetches metadata for a specific variant of the precompiled wheel. For non-ROCm, fetches vllm metadata. @@ -591,7 +590,6 @@ class precompiled_wheel_utils: @staticmethod def detect_system_cuda_variant() -> str: """Auto-detect CUDA variant from torch, nvidia-smi, or env default.""" - # Map CUDA major version to hosted wheel variants on wheels.vllm.ai supported = {12: "cu129", 13: "cu130"} @@ -886,8 +884,7 @@ class precompiled_wheel_utils: @staticmethod def determine_wheel_url() -> tuple[str, str | None]: - """ - Try to determine the precompiled wheel URL or path to use. + """Try to determine the precompiled wheel URL or path to use. The order of preference is: 1. user-specified wheel location (can be either local or remote, via VLLM_PRECOMPILED_WHEEL_LOCATION) diff --git a/tests/basic_correctness/test_basic_correctness.py b/tests/basic_correctness/test_basic_correctness.py index 3412b30814..1b3c92ee5b 100644 --- a/tests/basic_correctness/test_basic_correctness.py +++ b/tests/basic_correctness/test_basic_correctness.py @@ -89,7 +89,7 @@ TARGET_TEST_SUITE = _resolve_target_test_suite() condition=current_platform.is_rocm(), ) def test_vllm_gc_ed(): - """Verify vllm instance is GC'ed when it is deleted""" + """Verify vllm instance is GC'ed when it is deleted.""" llm = LLM("hmellor/tiny-random-LlamaForCausalLM") weak_llm = weakref.ref(llm) del llm diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index c3f66e24ab..6b4895c643 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -51,8 +51,7 @@ def _wake_up_with_poisoned_mappings(allocator, byte_value: int = 0xA5) -> None: @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") def test_python_error(): - """ - Test if Python error occurs when there's low-level + """Test if Python error occurs when there's low-level error happening from the C++ side. """ allocator = get_mem_allocator_instance() diff --git a/tests/benchmarks/test_endpoint_request_func_timing.py b/tests/benchmarks/test_endpoint_request_func_timing.py index 34394e3b01..57ac5824ab 100644 --- a/tests/benchmarks/test_endpoint_request_func_timing.py +++ b/tests/benchmarks/test_endpoint_request_func_timing.py @@ -228,7 +228,7 @@ def test_decode_span_identity_is_exact( api_url: str, chat: bool, ) -> None: - """latency - ttft must equal sum(itl) exactly. + """Latency - ttft must equal sum(itl) exactly. On a scripted clock this can only drift if a code path reads the clock an extra time or attributes a read to the wrong event. diff --git a/tests/ci_envs.py b/tests/ci_envs.py index f3a54f308c..171fac996a 100644 --- a/tests/ci_envs.py +++ b/tests/ci_envs.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -These envs only work for a small part of the tests, fix what you need! -""" +"""These envs only work for a small part of the tests, fix what you need!""" import os from collections.abc import Callable diff --git a/tests/compile/backend.py b/tests/compile/backend.py index 87f98946a8..167de6ad68 100644 --- a/tests/compile/backend.py +++ b/tests/compile/backend.py @@ -29,8 +29,7 @@ logger = init_logger("vllm.tests.compile.backend") class LazyInitPass(InductorPass): - """ - If there's a pass that we want to initialize lazily in a test, + """If there's a pass that we want to initialize lazily in a test, we can wrap it in LazyInitPass, which will initialize the pass when invoked and then immediately invoke it. """ @@ -45,8 +44,7 @@ class LazyInitPass(InductorPass): class TestBackend: - """ - This class provides a simple Inductor backend that can be used for testing. + """This class provides a simple Inductor backend that can be used for testing. It takes a list of custom passes and runs them after Inductor's passes. It also saves the graph before and after the custom passes for inspection. diff --git a/tests/compile/conftest.py b/tests/compile/conftest.py index 7d15b5c47e..972ec3fef5 100644 --- a/tests/compile/conftest.py +++ b/tests/compile/conftest.py @@ -11,8 +11,7 @@ from vllm.platforms.interface import DeviceCapability @pytest.fixture def mock_cuda_platform(): - """ - Fixture that returns a factory for creating mocked CUDA platforms. + """Fixture that returns a factory for creating mocked CUDA platforms. Usage: def test_something(mock_cuda_platform): @@ -51,8 +50,7 @@ def mock_cuda_platform(): @pytest.fixture def mock_xpu_platform(): - """ - Fixture that returns a factory for creating mocked XPU platforms. + """Fixture that returns a factory for creating mocked XPU platforms. Usage: def test_something(mock_xpu_platform): diff --git a/tests/compile/correctness_e2e/test_sequence_parallel.py b/tests/compile/correctness_e2e/test_sequence_parallel.py index eed5efb7b3..e5f8d1870e 100644 --- a/tests/compile/correctness_e2e/test_sequence_parallel.py +++ b/tests/compile/correctness_e2e/test_sequence_parallel.py @@ -1,11 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -WARNING: This test runs in both single-node (4 GPUs) and multi-node - (2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is - important to set the distributed backend to "mp" to avoid Ray scheduling - all workers in a node other than the head node, which can cause the test - to fail. +"""WARNING: This test runs in both single-node (4 GPUs) and multi-node +(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is +important to set the distributed backend to "mp" to avoid Ray scheduling +all workers in a node other than the head node, which can cause the test +to fail. """ import json diff --git a/tests/compile/fullgraph/test_full_cudagraph.py b/tests/compile/fullgraph/test_full_cudagraph.py index c2af7be048..d91cd84816 100644 --- a/tests/compile/fullgraph/test_full_cudagraph.py +++ b/tests/compile/fullgraph/test_full_cudagraph.py @@ -17,8 +17,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum @contextlib.contextmanager def temporary_environ(env_vars): - """ - Temporarily set environment variables and restore them afterward. + """Temporarily set environment variables and restore them afterward. We have to do this vs monkeypatch because monkeypatch doesn't work with "module" scoped fixtures. """ @@ -125,8 +124,7 @@ def llm_pair(request): indirect=True, ) class TestFullCUDAGraph: - """ - Use a class such that an llm pair is constructed once for all + """Use a class such that an llm pair is constructed once for all batch_size/max_tokens combinations and released immediately after. Module-scope fixtures would stick around the whole time, @@ -149,11 +147,9 @@ class TestFullCUDAGraph: ], ) def test_full_cudagraph(self, batch_size, max_tokens, llm_pair: tuple[LLM, LLM]): - """ - Test various batch sizes and max_tokens to ensure that the + """Test various batch sizes and max_tokens to ensure that the full cudagraph compilation works for padded cases too. """ - full_cudagraph_llm, piecewise_llm = llm_pair prompts = ["the quick brown fox"] * batch_size diff --git a/tests/compile/fullgraph/test_multiple_graphs.py b/tests/compile/fullgraph/test_multiple_graphs.py index 6d3788af9d..e265663334 100644 --- a/tests/compile/fullgraph/test_multiple_graphs.py +++ b/tests/compile/fullgraph/test_multiple_graphs.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test (piecewise) compilation with a simple model where multiple submodules +"""Test (piecewise) compilation with a simple model where multiple submodules are compiled and graph captured separately. """ diff --git a/tests/compile/fullgraph/test_simple.py b/tests/compile/fullgraph/test_simple.py index f1ea0e414d..4d10a92a07 100644 --- a/tests/compile/fullgraph/test_simple.py +++ b/tests/compile/fullgraph/test_simple.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test the piecewise compilation with a simple model so that we +"""Test the piecewise compilation with a simple model so that we can exactly calculate the expected output and side effects. """ @@ -52,8 +51,7 @@ class SillyModel(nn.Module): self.intermediate_unbacked = intermediate_unbacked def forward(self, x: torch.Tensor) -> torch.Tensor: - """ - Overall effect: + """Overall effect: x = 3 * x + 19 global_counter += 2 """ diff --git a/tests/compile/fullgraph/test_toy_llama.py b/tests/compile/fullgraph/test_toy_llama.py index 69c758702e..7823c362f6 100644 --- a/tests/compile/fullgraph/test_toy_llama.py +++ b/tests/compile/fullgraph/test_toy_llama.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test the piecewise compilation with a simple model, comparing the output +"""Test the piecewise compilation with a simple model, comparing the output with and without the piecewise compilation. This is a tractable model, the weights and computation are specially designed diff --git a/tests/compile/passes/distributed/test_async_tp.py b/tests/compile/passes/distributed/test_async_tp.py index 7b547c9720..ff0b048962 100644 --- a/tests/compile/passes/distributed/test_async_tp.py +++ b/tests/compile/passes/distributed/test_async_tp.py @@ -56,10 +56,7 @@ class TestMMRSModel(torch.nn.Module): torch.nn.init.normal_(self.gate_proj, std=0.02) def forward(self, hidden_states): - """ - Forward pass implementing the mm + reduce scatter in the FX graph - - """ + """Forward pass implementing the mm + reduce scatter in the FX graph.""" # Reshape input view = hidden_states.reshape(-1, self.hidden_size) @@ -88,9 +85,7 @@ class TestAGMMModel(torch.nn.Module): torch.nn.init.normal_(self.weight, std=0.02) def forward(self, hidden_states): - """ - Forward pass implementing the mm + all gather in the FX graph - """ + """Forward pass implementing the mm + all gather in the FX graph.""" # Reshape input view = hidden_states.reshape(-1, self.hidden_size) all_gather = tensor_model_parallel_all_gather(view, dim=0) @@ -122,10 +117,7 @@ class _BaseScaledMMModel(torch.nn.Module): class TestScaledMMRSModel(_BaseScaledMMModel): def forward(self, input: torch.Tensor): - """ - Forward pass implementing the scaled_mm + reduce scatter in the FX graph - - """ + """Forward pass implementing the scaled_mm + reduce scatter in the FX graph.""" fp8_input = input.to(FP8_DTYPE) scale_a = torch.ones(input.shape[0], 1, dtype=torch.float32) scaled_mm = torch._scaled_mm( @@ -147,9 +139,7 @@ class TestScaledMMRSModel(_BaseScaledMMModel): class TestAGScaledMMModel(_BaseScaledMMModel): def forward(self, input: torch.Tensor): - """ - Forward pass implementing the all gather + scaled_mm in the FX graph - """ + """Forward pass implementing the all gather + scaled_mm in the FX graph.""" # Reshape input fp8_input = input.to(FP8_DTYPE) all_gather = tensor_model_parallel_all_gather(fp8_input, dim=0) @@ -173,8 +163,7 @@ class TestAGScaledMMModel(_BaseScaledMMModel): class TestCutlassScaledMMRSModel(_BaseScaledMMModel): def forward(self, input: torch.Tensor): - """ - Forward pass implementing the cutlass_scaled_mm + reduce scatter + """Forward pass implementing the cutlass_scaled_mm + reduce scatter in the FX graph """ @@ -200,8 +189,7 @@ class TestCutlassScaledMMRSModel(_BaseScaledMMModel): class TestAGCutlassScaledMMModel(_BaseScaledMMModel): def forward(self, input: torch.Tensor): - """ - Forward pass implementing the all gather + cutlass_scaled_mm + """Forward pass implementing the all gather + cutlass_scaled_mm in the FX graph """ # Reshape input diff --git a/tests/compile/passes/ir/test_clone_cleanup.py b/tests/compile/passes/ir/test_clone_cleanup.py index b6626a5cb7..8a51e6d539 100644 --- a/tests/compile/passes/ir/test_clone_cleanup.py +++ b/tests/compile/passes/ir/test_clone_cleanup.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Comprehensive tests for UnsafeCloneEliminationPass. +"""Comprehensive tests for UnsafeCloneEliminationPass. This test suite exercises all possible valid FX graph patterns involving clones: 1. Clone with no users (dead code) diff --git a/tests/compile/passes/ir/test_inplace_functionalization.py b/tests/compile/passes/ir/test_inplace_functionalization.py index 1e8d566216..773ea93ce6 100644 --- a/tests/compile/passes/ir/test_inplace_functionalization.py +++ b/tests/compile/passes/ir/test_inplace_functionalization.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for IR inplace functionalization pass integration. +"""Tests for IR inplace functionalization pass integration. This test suite verifies that the inplace functionalization pass, lowering pass, and clone cleanup pass work together correctly with donated buffer tracking. @@ -108,8 +107,7 @@ class MixedModel(nn.Module): class ModelWithTritonAfterMaybeInplace(nn.Module): - """ - Model using maybe_inplace followed by a Triton kernel. + """Model using maybe_inplace followed by a Triton kernel. Test clone elimination can handle Triton in the graph """ @@ -410,8 +408,7 @@ def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int): reason="Only test on cuda and rocm platform", ) def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache): - """ - Test piecewise compilation with donated buffers across graph splits. + """Test piecewise compilation with donated buffers across graph splits. Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching. """ torch.set_default_device(current_platform.device_type) diff --git a/tests/compile/passes/test_double_aiter_rms_quant_fusion.py b/tests/compile/passes/test_double_aiter_rms_quant_fusion.py index 6a620d11a4..f05f821852 100644 --- a/tests/compile/passes/test_double_aiter_rms_quant_fusion.py +++ b/tests/compile/passes/test_double_aiter_rms_quant_fusion.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for the DoubleQuant fan-out variants registered by +"""Unit tests for the DoubleQuant fan-out variants registered by ``RocmAiterRMSNormQuantFusionPass``. Both variants target a 1-to-2 fan-out where one ``rms_norm`` output feeds @@ -90,8 +89,7 @@ def test_double_aiter_rms_fp8_group_quant_fusion( model_cls: type[torch.nn.Module], monkeypatch: pytest.MonkeyPatch, ) -> None: - """ - Both fan-out shapes (with and without an intermediate view) must fuse + """Both fan-out shapes (with and without an intermediate view) must fuse into ``rocm_aiter_rmsnorm_fp8_group_quant``: the no-view shape via ``DoubleAiterRMSFp8GroupQuantPattern`` and the viewed shape via the new ``DoubleAiterRMSFp8GroupQuantViewPattern`` sibling. diff --git a/tests/compile/passes/test_fuse_mla_dual_rms_norm.py b/tests/compile/passes/test_fuse_mla_dual_rms_norm.py index 6f20d4e158..0f631b9472 100644 --- a/tests/compile/passes/test_fuse_mla_dual_rms_norm.py +++ b/tests/compile/passes/test_fuse_mla_dual_rms_norm.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit test for the MLADualRMSNormFusionPass. +"""Unit test for the MLADualRMSNormFusionPass. The pass fuses paired q/kv RMS norms in MLA attention into a single fused_mla_dual_rms_norm op backed by AITER's fused_qk_rmsnorm kernel. @@ -38,13 +37,12 @@ FP8_DTYPE = current_platform.fp8_dtype() class MLADualRMSNormTestModel(torch.nn.Module): - """ - Minimal model reproducing the MLA dual RMS norm pattern: - linear -> split([q_dim, kv_dim]) - +-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear - +-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim]) - +-- kv_c (getitem 0) -> rms_norm(kv_w, eps) - +-- k_pe + """Minimal model reproducing the MLA dual RMS norm pattern: + linear -> split([q_dim, kv_dim]) + +-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear + +-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim]) + +-- kv_c (getitem 0) -> rms_norm(kv_w, eps) + +-- k_pe """ def __init__( @@ -155,13 +153,12 @@ def test_fuse_mla_dual_rms_norm( class MLADualRMSNormFp8PerTokenTestModel(torch.nn.Module): - """ - Minimal model reproducing the FP8 MLA attention path with *per-token* quant: - linear -> split([q_dim, kv_dim]) - +-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant - +-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim]) - +-- kv_c (getitem 0) -> rms_norm (bf16) - +-- k_pe + """Minimal model reproducing the FP8 MLA attention path with *per-token* quant: + linear -> split([q_dim, kv_dim]) + +-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant + +-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim]) + +-- kv_c (getitem 0) -> rms_norm (bf16) + +-- k_pe """ def __init__( diff --git a/tests/compile/passes/test_fusion_attn.py b/tests/compile/passes/test_fusion_attn.py index 4df02cab95..37f0d4126c 100644 --- a/tests/compile/passes/test_fusion_attn.py +++ b/tests/compile/passes/test_fusion_attn.py @@ -107,7 +107,6 @@ class AttentionQuantPatternModel(torch.nn.Module): def build_attn_metadata(self, batch_size: int) -> AttentionMetadata: """Initialize attention metadata.""" - # TODO (Rohan138) reuse utils from vllm/v1/worker/gpu/attn_utils.py # Create common attn metadata @@ -304,7 +303,7 @@ def test_attention_quant_pattern( dist_init, disable_vllm_compile_cache, ): - """Test AttentionStaticQuantPattern fusion pass""" + """Test AttentionStaticQuantPattern fusion pass.""" if backend == AttentionBackendEnum.FLASHINFER and ( not current_platform.is_device_capability((10, 0)) or not has_flashinfer() ): diff --git a/tests/compile/passes/test_mla_attn_quant_fusion.py b/tests/compile/passes/test_mla_attn_quant_fusion.py index 11c455bfa2..f1de3dde0c 100644 --- a/tests/compile/passes/test_mla_attn_quant_fusion.py +++ b/tests/compile/passes/test_mla_attn_quant_fusion.py @@ -141,7 +141,6 @@ class MLAAttentionQuantPatternModel(torch.nn.Module): the full output[:num_actual_toks] buffer after both forward_mha and forward_mqa have written their results. """ - batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size) common_attn_metadata = create_common_attn_metadata( batch_spec, self.block_size, self.device, arange_block_indices=True @@ -408,7 +407,7 @@ def test_mla_attention_quant_pattern( dist_init, disable_vllm_compile_cache, ): - """Test MLA AttentionQuantPattern fusion pass""" + """Test MLA AttentionQuantPattern fusion pass.""" if ( model_class is TestMLAAttentionNvfp4QuantPatternModel and not is_nvfp4_supported() diff --git a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py index ee94960844..10dc401a26 100644 --- a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for manual fusion via maybe_fused_act_quant. +"""Tests for manual fusion via maybe_fused_act_quant. Tests all fusion paths in _FUSED_ACT_QUANT: - kFp8StaticTensorSym: all platforms diff --git a/tests/compile/silly_attention.py b/tests/compile/silly_attention.py index 29c02f6e6a..0d33135129 100644 --- a/tests/compile/silly_attention.py +++ b/tests/compile/silly_attention.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Shared PyTorch custom silly attention for compilation tests. +"""Shared PyTorch custom silly attention for compilation tests. Centralizes custom operation definitions to avoid duplicate registrations. """ @@ -21,12 +20,12 @@ _global_counter = 0 def get_global_counter(): - """Get the current global counter value""" + """Get the current global counter value.""" return _global_counter def reset_global_counter(): - """Reset the global counter to 0""" + """Reset the global counter to 0.""" global _global_counter _global_counter = 0 @@ -34,8 +33,7 @@ def reset_global_counter(): def silly_attention( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, out: torch.Tensor ) -> None: - """ - Unified attention implementation that depends on + """Unified attention implementation that depends on all inputs and affects the output. Always increments a global counter that tests can use or ignore. """ @@ -51,7 +49,7 @@ def silly_attention( def silly_attention_fake( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, out: torch.Tensor ) -> None: - """Fake implementation for testing""" + """Fake implementation for testing.""" return diff --git a/tests/compile/test_aot_compile.py b/tests/compile/test_aot_compile.py index 90c132bee1..e5c11b8acc 100644 --- a/tests/compile/test_aot_compile.py +++ b/tests/compile/test_aot_compile.py @@ -268,8 +268,7 @@ def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch): def test_cache_load_returns_tuple_consistency_tuple_output( monkeypatch: pytest.MonkeyPatch, ): - """ - Test that cache loading correctly handles models that return tuples. + """Test that cache loading correctly handles models that return tuples. This verifies that when a model returns a tuple of tensors, the output type is preserved as a tuple between fresh compilation and cache load. @@ -335,8 +334,7 @@ def test_cache_load_returns_tuple_consistency_tuple_output( @pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10") def test_shape_env(monkeypatch: pytest.MonkeyPatch): - """ - Test that the shape environment is correctly serialized and preserved + """Test that the shape environment is correctly serialized and preserved when loading from cache. """ with monkeypatch.context() as m: @@ -374,8 +372,7 @@ def test_shape_env(monkeypatch: pytest.MonkeyPatch): def test_partition_wrapper_applied_on_aot_load( monkeypatch: pytest.MonkeyPatch, vllm_tmp_cache: Path, mocker ): - """ - Test that partition wrappers are applied when loading AOT cached functions. + """Test that partition wrappers are applied when loading AOT cached functions. This test verifies the fix for GitHub issue #31439 where AOT compile caused 2x latency regression when use_inductor_graph_partition=True. @@ -498,14 +495,12 @@ def test_standalone_compile_correctness(): @pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10") @create_new_process_for_each_test("spawn") def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch): - """ - Test that compiling gpt2 twice results in a cache hit. + """Test that compiling gpt2 twice results in a cache hit. Counter values are read from the EngineCore subprocess via ``LLM.collective_rpc`` so the test works under default V1 multiprocessing (no shared memory between test and engine). """ - from vllm import LLM def _snap(self): @@ -641,7 +636,7 @@ class TestStandaloneCompiledArtifacts: @patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize") def test_load_all_success(self, mock_deserialize): - """Test successful loading of all artifacts""" + """Test successful loading of all artifacts.""" cache = StandaloneCompiledArtifacts() mock_artifact1 = Mock() @@ -658,7 +653,7 @@ class TestStandaloneCompiledArtifacts: @patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize") def test_load_all_already_loaded(self, mock_deserialize): - """Test that load_all skips if already loaded""" + """Test that load_all skips if already loaded.""" cache = StandaloneCompiledArtifacts() mock_artifact = Mock() @@ -671,7 +666,7 @@ class TestStandaloneCompiledArtifacts: @patch("torch._inductor.standalone_compile.AOTCompiledArtifact.deserialize") def test_get_loaded_artifact(self, mock_deserialize): - """Test retrieving loaded artifacts""" + """Test retrieving loaded artifacts.""" cache = StandaloneCompiledArtifacts() mock_artifact = Mock() diff --git a/tests/compile/test_dynamic_shapes_compilation.py b/tests/compile/test_dynamic_shapes_compilation.py index 71f66fce40..ea2ccafbe7 100644 --- a/tests/compile/test_dynamic_shapes_compilation.py +++ b/tests/compile/test_dynamic_shapes_compilation.py @@ -22,7 +22,7 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer def get_test_models(): - """Get list of models to test based on PyTorch version""" + """Get list of models to test based on PyTorch version.""" models = [ "Qwen/Qwen3-0.6B", "openai-community/gpt2", @@ -172,7 +172,6 @@ def test_model_specialization_with_evaluate_guards( """Test that evaluate_guards correctly detects shape specialization violations. """ - if ( use_aot_compile and dynamic_shapes_type == DynamicShapesType.BACKED diff --git a/tests/compile/test_graph_partition.py b/tests/compile/test_graph_partition.py index bb2a6f2aee..f5aaffe00e 100644 --- a/tests/compile/test_graph_partition.py +++ b/tests/compile/test_graph_partition.py @@ -24,8 +24,7 @@ DEVICE_TYPE = current_platform.device_type def test_getitem_moved_to_producer_subgraph(): - """ - Test that getitem operations are moved to the same subgraph as their input, + """Test that getitem operations are moved to the same subgraph as their input, preventing tuple inputs to submodules. """ @@ -79,8 +78,7 @@ def test_getitem_moved_to_producer_subgraph(): def test_no_tuple_inputs_with_multiple_consumers(): - """ - Test that when a tuple is consumed by multiple split operations, + """Test that when a tuple is consumed by multiple split operations, getitem operations are properly moved to avoid tuple inputs. """ @@ -138,13 +136,10 @@ def test_no_tuple_inputs_with_multiple_consumers(): def test_consecutive_ops_in_split(): - """ - Test that consecutive splitting operations are grouped into the same subgraph - """ + """Test that consecutive splitting operations are grouped into the same subgraph.""" def model_fn(x: torch.Tensor) -> torch.Tensor: - """ - Define a simple model where consecutive operations create opportunities + """Define a simple model where consecutive operations create opportunities for splitting subgraphs. """ # Apply silly attention followed by consecutive operations @@ -211,8 +206,7 @@ def _subgraphs_with_empty_nodes(split_items, *, is_splitting_graph): def test_empty_only_partition_stays_separate_after_splitting_predecessor(): - """ - Empty-only subgraphs should not be merged when the only predecessor is + """Empty-only subgraphs should not be merged when the only predecessor is a splitting-op subgraph. """ @@ -248,8 +242,7 @@ def test_empty_only_partition_stays_separate_after_splitting_predecessor(): def test_empty_only_partition_is_merged(): - """ - Empty-only subgraphs should still be merged when a non-splitting predecessor + """Empty-only subgraphs should still be merged when a non-splitting predecessor exists. The merged empty node must remain outside splitting-op subgraphs. """ @@ -294,8 +287,7 @@ def test_empty_only_partition_is_merged(): def test_builtin_empty_only_partition_is_merged(): - """ - In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets + """In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets (not aten OpOverload). Ensure empty-only partitions are still merged. """ @@ -340,8 +332,7 @@ def test_builtin_empty_only_partition_is_merged(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_sym_size_whole_shape_boundary(): - """ - Test that using x.size() (whole shape) across a split boundary can be + """Test that using x.size() (whole shape) across a split boundary can be compiled by standalone_compile. The dynamo graph looks like: @@ -400,8 +391,7 @@ def test_sym_size_whole_shape_boundary(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_symint_crosses_split_boundary(): - """ - Test that SymInt placeholders from torch.compile + mark_dynamic + """Test that SymInt placeholders from torch.compile + mark_dynamic cross split boundaries safely via split_module's natural threading. SymInt values are threaded through subgraphs by split_module and @@ -459,8 +449,7 @@ def test_symint_crosses_split_boundary(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_shape_boundary_standalone_compile(): - """ - Repro for the original production bug: + """Repro for the original production bug: AssertionError: out_spec mismatch TreeSpec(tuple, None, [*, *, TreeSpec(Size, None, [*, *]), *]) @@ -529,8 +518,7 @@ def test_shape_boundary_standalone_compile(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_size_used_in_multiple_consumer_subgraphs(): - """ - Validates that x.size() (whole shape) used by multiple downstream subgraphs + """Validates that x.size() (whole shape) used by multiple downstream subgraphs does not cause torch.Size to cross split boundaries. Model: @@ -584,8 +572,7 @@ def test_size_used_in_multiple_consumer_subgraphs(): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_sym_size_metadata_propagated(): - """ - Validates that new sym_size.int nodes created by the pre-pass have + """Validates that new sym_size.int nodes created by the pre-pass have example_value metadata set. Without it, placeholder metadata in consumer subgraphs would be None, breaking any code that dynamically builds example inputs from metadata (e.g. standalone_compile per-submodule). @@ -634,8 +621,7 @@ def test_sym_size_metadata_propagated(): def test_decompose_size_with_getitem_user(): - """ - Regression test: _decompose_size_nodes must handle getitem users of size() + """Regression test: _decompose_size_nodes must handle getitem users of size() correctly. When a graph contains x.shape[i], it can appear as: @@ -704,8 +690,7 @@ def test_decompose_size_with_getitem_user(): def test_decompose_size_leaves_scalar_size_with_dim(): - """ - Regression test: _decompose_size_nodes must leave x.size(dim) alone. + """Regression test: _decompose_size_nodes must leave x.size(dim) alone. x.size() returns a torch.Size tuple that can't cross split boundaries and must be decomposed. x.size(dim), however, already returns a scalar diff --git a/tests/compile/test_structured_logging.py b/tests/compile/test_structured_logging.py index 10b0ed139c..98f092bfce 100644 --- a/tests/compile/test_structured_logging.py +++ b/tests/compile/test_structured_logging.py @@ -60,6 +60,7 @@ class TraceStructuredCapture: Args: event_type: The event type to filter by (e.g., "artifact", "graph_dump") name_pattern: Regex pattern to match against the artifact name + """ regex = re.compile(name_pattern) return [ diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index 590e97c1eb..1d0f4d915b 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -40,7 +40,7 @@ class DummyLogprobsMode(Enum): def test_hash_factors_deterministic(): - """Test that hash_factors produces consistent SHA-256 hashes""" + """Test that hash_factors produces consistent SHA-256 hashes.""" factors = {"a": 1, "b": "test"} hash1 = hash_factors(factors) hash2 = hash_factors(factors) diff --git a/tests/config/test_mp_reducer.py b/tests/config/test_mp_reducer.py index 56dc542f1c..56e4e47474 100644 --- a/tests/config/test_mp_reducer.py +++ b/tests/config/test_mp_reducer.py @@ -9,12 +9,10 @@ from vllm.v1.engine.async_llm import AsyncLLM def test_mp_reducer(): - """ - Test that _reduce_config reducer is registered when AsyncLLM is instantiated + """Test that _reduce_config reducer is registered when AsyncLLM is instantiated without transformers_modules. This is a regression test for https://github.com/vllm-project/vllm/pull/18640. """ - # Ensure transformers_modules is not in sys.modules if "transformers_modules" in sys.modules: del sys.modules["transformers_modules"] diff --git a/tests/cuda/test_cuda_context.py b/tests/cuda/test_cuda_context.py index aca490d156..cb2ba1b199 100644 --- a/tests/cuda/test_cuda_context.py +++ b/tests/cuda/test_cuda_context.py @@ -11,7 +11,7 @@ from vllm.platforms import current_platform def check_cuda_context(): - """Check CUDA driver context status""" + """Check CUDA driver context status.""" try: cuda = ctypes.CDLL("libcuda.so") device = ctypes.c_int() @@ -22,7 +22,7 @@ def check_cuda_context(): def run_cuda_test_in_thread(device_input, expected_device_id): - """Run CUDA context test in separate thread for isolation""" + """Run CUDA context test in separate thread for isolation.""" try: # New thread should have no CUDA context initially valid_before, device_before = check_cuda_context() diff --git a/tests/cuda/test_platform_no_cuda_init.py b/tests/cuda/test_platform_no_cuda_init.py index 697d2cfec8..6f50ce606b 100644 --- a/tests/cuda/test_platform_no_cuda_init.py +++ b/tests/cuda/test_platform_no_cuda_init.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test that platform imports do not prematurely initialize CUDA. +"""Test that platform imports do not prematurely initialize CUDA. This is critical for Ray-based multi-GPU setups where workers need to set CUDA_VISIBLE_DEVICES after importing vLLM but before CUDA is initialized. diff --git a/tests/detokenizer/test_stop_string_while_stop_model_terminates.py b/tests/detokenizer/test_stop_string_while_stop_model_terminates.py index 44215cb72a..3b72d612dd 100644 --- a/tests/detokenizer/test_stop_string_while_stop_model_terminates.py +++ b/tests/detokenizer/test_stop_string_while_stop_model_terminates.py @@ -44,8 +44,7 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0): def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: bool): - """ - This test verifies that the detokenizer correctly handles the case where + """This test verifies that the detokenizer correctly handles the case where the generated token sequence contains both: - a stop token - an token @@ -67,7 +66,6 @@ def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: boo where stop-string evaluation occurs before the early-return on stop_terminated. """ - # Generate text "abcdeZ" and tokenize it. generated_text = "abcde" eos_token = "Z" diff --git a/tests/distributed/conftest.py b/tests/distributed/conftest.py index 48df856f5e..85a9d73aa5 100644 --- a/tests/distributed/conftest.py +++ b/tests/distributed/conftest.py @@ -18,13 +18,13 @@ DP_RANK = 0 @pytest.fixture def random_port(): - """Generate a random port number for testing""" + """Generate a random port number for testing.""" return random.randint(10000, 59900) @pytest.fixture def publisher_config(random_port, request): - """Create a publisher config with inproc transport""" + """Create a publisher config with inproc transport.""" how = request.param if hasattr(request, "param") else "inproc" if how == "inproc": @@ -47,7 +47,7 @@ def publisher_config(random_port, request): @pytest.fixture def publisher(publisher_config): - """Create and return a publisher instance""" + """Create and return a publisher instance.""" pub = EventPublisherFactory.create(publisher_config, DP_RANK) yield pub pub.shutdown() @@ -55,7 +55,7 @@ def publisher(publisher_config): @pytest.fixture def subscriber(publisher_config): - """Create and return a subscriber for testing""" + """Create and return a subscriber for testing.""" endpoint = publisher_config.endpoint replay_endpoint = publisher_config.replay_endpoint @@ -74,7 +74,7 @@ def subscriber(publisher_config): class MockSubscriber: - """Helper class to receive and verify published events""" + """Helper class to receive and verify published events.""" def __init__( self, @@ -113,7 +113,7 @@ class MockSubscriber: self.decoder = msgspec.msgpack.Decoder(type=decode_type) def receive_one(self, timeout=1000) -> tuple[int, SampleBatch] | None: - """Receive a single message with timeout""" + """Receive a single message with timeout.""" if not self.sub.poll(timeout): return None @@ -127,7 +127,7 @@ class MockSubscriber: return seq, data def request_replay(self, start_seq: int, socket_idx: int = 0) -> None: - """Request replay of messages starting from start_seq""" + """Request replay of messages starting from start_seq.""" if not self.replay_sockets: raise ValueError("Replay sockets not initialized") if socket_idx >= len(self.replay_sockets): @@ -138,7 +138,7 @@ class MockSubscriber: ) def receive_replay(self, socket_idx: int = 0) -> list[tuple[int, SampleBatch]]: - """Receive replayed messages from a specific replay socket""" + """Receive replayed messages from a specific replay socket.""" if not self.replay_sockets: raise ValueError("Replay sockets not initialized") if socket_idx >= len(self.replay_sockets): @@ -170,7 +170,7 @@ class MockSubscriber: return replayed def close(self): - """Clean up resources""" + """Clean up resources.""" self.sub.close() for replay in self.replay_sockets: replay.close() diff --git a/tests/distributed/test_context_parallel.py b/tests/distributed/test_context_parallel.py index 007948aeba..6d5d454b4a 100644 --- a/tests/distributed/test_context_parallel.py +++ b/tests/distributed/test_context_parallel.py @@ -1,11 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -WARNING: This test runs in both single-node (4 GPUs) and multi-node - (2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is - important to set the distributed backend to "mp" to avoid Ray scheduling - all workers in a node other than the head node, which can cause the test - to fail. +"""WARNING: This test runs in both single-node (4 GPUs) and multi-node +(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is +important to set the distributed backend to "mp" to avoid Ray scheduling +all workers in a node other than the head node, which can cause the test +to fail. """ import json diff --git a/tests/distributed/test_eplb_algo.py b/tests/distributed/test_eplb_algo.py index 20360c8c61..aaa987b9fc 100644 --- a/tests/distributed/test_eplb_algo.py +++ b/tests/distributed/test_eplb_algo.py @@ -29,7 +29,7 @@ def test_eplb_load_stats_reduce_across_ranks(): def test_basic_rebalance(): - """Test basic rebalancing functionality""" + """Test basic rebalancing functionality.""" # Example from https://github.com/deepseek-ai/eplb weight = torch.tensor( [ @@ -92,7 +92,7 @@ def test_basic_rebalance(): def test_single_gpu_case(): - """Test single GPU case""" + """Test single GPU case.""" weight = torch.tensor([[10, 20, 30, 40]]) num_replicas = 4 num_groups = 1 @@ -115,7 +115,7 @@ def test_single_gpu_case(): def test_equal_weights(): - """Test case with equal weights""" + """Test case with equal weights.""" weight = torch.tensor([[50, 50, 50, 50, 50, 50, 50, 50]]) num_replicas = 8 num_groups = 2 @@ -139,7 +139,7 @@ def test_equal_weights(): def test_extreme_weight_imbalance(): - """Test extreme weight imbalance case""" + """Test extreme weight imbalance case.""" weight = torch.tensor([[1000, 1, 1, 1, 1, 1, 1, 1]]) num_replicas = 12 num_groups = 2 @@ -162,7 +162,7 @@ def test_extreme_weight_imbalance(): def test_multiple_layers(): - """Test multiple layers case""" + """Test multiple layers case.""" weight = torch.tensor( [ [10, 20, 30, 40, 50, 60], # First layer @@ -195,7 +195,7 @@ def test_multiple_layers(): def test_parameter_validation(): - """Test parameter validation""" + """Test parameter validation.""" weight = torch.tensor([[10, 20, 30, 40]]) # Test non-divisible case - this should handle normally without throwing @@ -213,7 +213,7 @@ def test_parameter_validation(): def test_small_scale_hierarchical(): - """Test small-scale hierarchical load balancing""" + """Test small-scale hierarchical load balancing.""" weight = torch.tensor( [ [100, 50, 200, 75, 150, 25, 300, 80], # 8 experts @@ -243,7 +243,7 @@ def test_small_scale_hierarchical(): def test_global_load_balance_fallback(): - """Test global load balancing fallback case""" + """Test global load balancing fallback case.""" # When num_groups % num_nodes != 0, should fall back to global load # balancing weight = torch.tensor([[10, 20, 30, 40, 50, 60]]) @@ -265,7 +265,7 @@ def test_global_load_balance_fallback(): @pytest.mark.parametrize("device", ["cpu", "cuda"]) def test_device_compatibility(device): - """Test device compatibility""" + """Test device compatibility.""" if device == "cuda" and not torch.cuda.is_available(): pytest.skip("CUDA not available") @@ -287,8 +287,7 @@ def test_device_compatibility(device): def test_additional_cases(): - """Test more edge cases and different parameter combinations""" - + """Test more edge cases and different parameter combinations.""" # Test case 1: Large-scale distributed setup weight1 = torch.tensor( [[50, 100, 75, 120, 90, 60, 80, 110, 40, 70, 95, 85, 65, 55, 45, 35]] @@ -320,8 +319,7 @@ def test_additional_cases(): def test_compute_logical_maps_with_negative_indices(): - """ - Test that compute_logical_maps correctly handles physical slots containing + """Test that compute_logical_maps correctly handles physical slots containing -1 (unused slots). """ # 2 layers, 6 physical slots, 4 logical experts. diff --git a/tests/distributed/test_eplb_events.py b/tests/distributed/test_eplb_events.py index 40323dc898..963e3cbb77 100644 --- a/tests/distributed/test_eplb_events.py +++ b/tests/distributed/test_eplb_events.py @@ -57,9 +57,8 @@ def test_reuse_across_multiple_cycles(): def test_producer_consumer(): - """ - This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared GPU - tensor on multiple CPU threads. + """This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared + GPU tensor on multiple CPU threads. """ worker_stream = torch.cuda.Stream() # Create a single element counter that will be shared between two threads diff --git a/tests/distributed/test_eplb_execute.py b/tests/distributed/test_eplb_execute.py index 1ff41f5df2..aed0decc6d 100644 --- a/tests/distributed/test_eplb_execute.py +++ b/tests/distributed/test_eplb_execute.py @@ -56,8 +56,7 @@ def create_expert_indices_with_redundancy( total_physical_experts: int, redundancy_config: list[int], # redundancy for each logical expert ) -> torch.Tensor: - """ - Create expert indices with redundancy. + """Create expert indices with redundancy. Args: num_layers: number of layers @@ -67,6 +66,7 @@ def create_expert_indices_with_redundancy( Returns: indices: Shape (num_layers, total_physical_experts) + """ assert sum(redundancy_config) == total_physical_experts assert len(redundancy_config) == num_logical_experts @@ -95,8 +95,7 @@ def create_expert_weights( device: torch.device, physical_to_logical_mapping: torch.Tensor, ) -> list[list[torch.Tensor]]: - """ - Create fake expert weights tensor for testing. + """Create fake expert weights tensor for testing. Use `arange` to generate predictable weights values, based on logical expert ID. @@ -105,6 +104,7 @@ def create_expert_weights( Args: physical_to_logical_mapping: Shape (num_layers, num_local_experts) mapping[layer, physical_pos] = logical_expert_id + """ expert_weights = [] @@ -209,9 +209,7 @@ def verify_redundant_experts_have_same_weights( world_size: int, num_local_experts: int, ) -> bool: - """ - Verify that all replicas of the same logical expert have the same weights. - """ + """Verify that all replicas of the same logical expert have the same weights.""" num_layers = len(expert_weights) total_physical_experts = world_size * num_local_experts @@ -568,7 +566,6 @@ def test_rearrange_expert_weights_with_redundancy( eplb_communicator, ): """Test the functionality of rearranging expert weights with redundancy.""" - if eplb_communicator == "nixl" and not has_nixl(): pytest.skip("NIXL is not available") if torch.accelerator.device_count() < world_size: @@ -678,7 +675,6 @@ def test_async_transfer_layer_without_mtp( eplb_communicator: str, ): """Exercise async EPLB transfer path without MTP/spec decode.""" - if eplb_communicator == "nixl" and not has_nixl(): pytest.skip("NIXL is not available") if torch.accelerator.device_count() < world_size: @@ -696,11 +692,9 @@ def test_async_transfer_layer_without_mtp( @pytest.mark.parametrize("world_size", [2, 4]) def test_rearrange_expert_weights_no_change(world_size): - """ - Test that when the indices do not change, the weights should remain + """Test that when the indices do not change, the weights should remain unchanged. """ - if torch.accelerator.device_count() < world_size: pytest.skip(f"Need at least {world_size} GPUs to run the test") distributed_run( @@ -800,7 +794,6 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None: @pytest.mark.parametrize("world_size", [2, 4]) def test_rearrange_expert_weights_profile_mode(world_size): """Test profile mode (should not copy actual weights)""" - if torch.accelerator.device_count() < world_size: pytest.skip(f"Need at least {world_size} GPUs to run the test") distributed_run( diff --git a/tests/distributed/test_eplb_spec_decode.py b/tests/distributed/test_eplb_spec_decode.py index b4211c74c1..ced206a305 100644 --- a/tests/distributed/test_eplb_spec_decode.py +++ b/tests/distributed/test_eplb_spec_decode.py @@ -75,8 +75,7 @@ def test_eplb_spec_decode( monkeypatch: pytest.MonkeyPatch, model_setup: tuple[str, str, str, int, float], ): - """ - Test the correctness of EPLB speculative decoding with GSM8K dataset. + """Test the correctness of EPLB speculative decoding with GSM8K dataset. Applicable to MoE models with mtp or eagle spec decode. """ method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup @@ -109,10 +108,7 @@ def test_eplb_spec_decode( @large_gpu_mark(min_gb=80) def test_eplb_spec_decode_qwen3_next_mtp_async() -> None: - """ - Ensure async EPLB works with MTP speculative decoding for Qwen3-Next. - """ - + """Ensure async EPLB works with MTP speculative decoding for Qwen3-Next.""" TASK = "gsm8k" FILTER = "exact_match,strict-match" RTOL = 0.03 diff --git a/tests/distributed/test_eplb_utils.py b/tests/distributed/test_eplb_utils.py index a6cfb0d29d..dd2bb2760b 100644 --- a/tests/distributed/test_eplb_utils.py +++ b/tests/distributed/test_eplb_utils.py @@ -30,8 +30,7 @@ def _make_model_state( def test_commit_eplb_maps_shape_change(): - """ - When the number of physical experts changes, resize the active map within its + """When the number of physical experts changes, resize the active map within its preallocated storage. """ num_layers, num_logical, num_physical = 2, 4, 6 @@ -65,8 +64,7 @@ def test_commit_eplb_maps_shape_change(): def test_commit_eplb_maps_for_layer_logical_padding(): - """ - Test that logical_to_physical_map is padded with -1 to fill the + """Test that logical_to_physical_map is padded with -1 to fill the pre-allocated slots when the new map has fewer replicas than the max. """ num_layers, num_logical, num_physical = 2, 4, 6 diff --git a/tests/distributed/test_events.py b/tests/distributed/test_events.py index 894f3f7f8b..aae80d9cce 100644 --- a/tests/distributed/test_events.py +++ b/tests/distributed/test_events.py @@ -20,27 +20,26 @@ class EventSample( tag=True, # type: ignore array_like=True, # type: ignore ): - """Test event for publisher testing""" + """Test event for publisher testing.""" id: int value: str class SampleBatch(EventBatch): - """Test event batch for publisher testing""" + """Test event batch for publisher testing.""" events: list[EventSample] def create_test_events(count: int) -> SampleBatch: - """Create a batch of test events""" + """Create a batch of test events.""" events = [EventSample(id=i, value=f"test-{i}") for i in range(count)] return SampleBatch(ts=time.time(), events=events) def test_basic_publishing(publisher, subscriber): - """Test basic event publishing works""" - + """Test basic event publishing works.""" test_batch = create_test_events(5) publisher.publish(test_batch) @@ -58,7 +57,7 @@ def test_basic_publishing(publisher, subscriber): def test_multiple_events(publisher, subscriber): - """Test publishing and receiving multiple event batches""" + """Test publishing and receiving multiple event batches.""" for _ in range(10): batch = create_test_events(2) publisher.publish(batch) @@ -75,7 +74,7 @@ def test_multiple_events(publisher, subscriber): def test_replay_mechanism(publisher, subscriber): - """Test the replay mechanism works correctly""" + """Test the replay mechanism works correctly.""" for _ in range(19): batch = create_test_events(1) publisher.publish(batch) @@ -96,7 +95,7 @@ def test_replay_mechanism(publisher, subscriber): def test_replay_includes_topic(publisher, subscriber, publisher_config): - """Test that replay responses include the topic, matching PUB format""" + """Test that replay responses include the topic, matching PUB format.""" for _ in range(5): publisher.publish(create_test_events(1)) @@ -115,7 +114,7 @@ def test_replay_includes_topic(publisher, subscriber, publisher_config): def test_buffer_limit(publisher, subscriber, publisher_config): - """Test buffer limit behavior""" + """Test buffer limit behavior.""" buffer_size = publisher_config.buffer_steps # Publish more events than the buffer can hold @@ -139,9 +138,7 @@ def test_buffer_limit(publisher, subscriber, publisher_config): def test_topic_filtering(publisher_config): - """ - Test that a subscriber only receives messages matching its topic filter - """ + """Test that a subscriber only receives messages matching its topic filter.""" publisher_config.replay_endpoint = None publisher_config.topic = "foo" @@ -174,7 +171,7 @@ def test_topic_filtering(publisher_config): def test_high_volume(publisher, subscriber): - """Test publishing and receiving a high volume of events""" + """Test publishing and receiving a high volume of events.""" num_batches = 10_000 events_per_batch = 100 @@ -210,7 +207,7 @@ def test_high_volume(publisher, subscriber): def test_null_publisher(): - """Test that NullEventPublisher can be used without errors""" + """Test that NullEventPublisher can be used without errors.""" publisher = NullEventPublisher(DP_RANK) # This should not raise any errors @@ -220,8 +217,7 @@ def test_null_publisher(): def test_data_parallel_rank_tagging(publisher_config): - """Test that events are properly tagged with their data parallel rank""" - + """Test that events are properly tagged with their data parallel rank.""" publisher_config.topic = "foo" pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK) pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1) @@ -285,7 +281,7 @@ def test_data_parallel_rank_tagging(publisher_config): def test_event_publisher_factory(random_port): - """Test event publisher factory creation behavior under different configurations""" + """Test event publisher factory creation behavior under different configurations.""" from vllm.config.kv_events import KVEventsConfig from vllm.distributed.kv_events import ZmqEventPublisher diff --git a/tests/distributed/test_expert_placement.py b/tests/distributed/test_expert_placement.py index 46f63408f4..4174b5c6cc 100644 --- a/tests/distributed/test_expert_placement.py +++ b/tests/distributed/test_expert_placement.py @@ -49,7 +49,6 @@ def verify_round_robin_pattern(expert_map, ep_rank, ep_size, global_num_experts) @pytest.mark.parametrize("world_size", [2, 4]) def test_expert_placement_various_sizes(expert_placement_strategy, world_size): """Test round_robin expert placement with various expert counts.""" - # Test with different global_num_experts values # Include both divisible and non-divisible cases if world_size == 2: @@ -116,7 +115,6 @@ def test_expert_placement_various_sizes(expert_placement_strategy, world_size): @pytest.mark.parametrize("world_size", [2, 4]) def test_expert_placement_edge_cases(expert_placement_strategy, world_size): """Test edge cases for round_robin expert placement.""" - # Test case 1: ep_size = 1 (should return None for expert_map) local_num_experts, expert_map, _ = determine_expert_map( ep_size=1, @@ -139,7 +137,6 @@ def test_expert_placement_edge_cases(expert_placement_strategy, world_size): def test_determine_expert_map_comprehensive(): """Test of determine_expert_map function with various configurations.""" - # Test cases: (ep_size, ep_rank, global_num_experts, # expert_placement_strategy, expected_local, expected_map_pattern) test_cases = [ diff --git a/tests/distributed/test_mnnvl_alltoall.py b/tests/distributed/test_mnnvl_alltoall.py index 636f7cb525..1cd6752460 100644 --- a/tests/distributed/test_mnnvl_alltoall.py +++ b/tests/distributed/test_mnnvl_alltoall.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for MNNVL AllToAll operations. +"""Tests for MNNVL AllToAll operations. Requires: docker run ... --cap-add=SYS_PTRACE ... Run: pytest tests/distributed/test_mnnvl_alltoall.py -v @@ -87,6 +86,7 @@ def _run_worker(rank, world_size, port, worker_fn, dp_size, dp_port, err_queue): Otherwise use tp=world_size (default for EP-based tests). dp_port: Separate port for the DP master (only used when dp_size is set). err_queue: Queue for propagating tracebacks to the parent process. + """ try: os.environ.pop("CUDA_VISIBLE_DEVICES", None) @@ -115,6 +115,7 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port): Args: port: Port for torch.distributed init. dp_port: Separate port for the DP master group init. + """ from vllm.config import VllmConfig, set_current_vllm_config from vllm.config.parallel import ParallelConfig diff --git a/tests/distributed/test_mq_connect_ip.py b/tests/distributed/test_mq_connect_ip.py index 4b0cdda3ad..05617bbb60 100644 --- a/tests/distributed/test_mq_connect_ip.py +++ b/tests/distributed/test_mq_connect_ip.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test that MessageQueue uses the local node's IP for binding, +"""Test that MessageQueue uses the local node's IP for binding, not a remote master_addr. This validates the fix for cross-node data-parallel where each DP group leader must bind to its own IP. diff --git a/tests/distributed/test_multiproc_executor.py b/tests/distributed/test_multiproc_executor.py index 599025d1c8..d0a417a788 100644 --- a/tests/distributed/test_multiproc_executor.py +++ b/tests/distributed/test_multiproc_executor.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Integration tests for MultiprocExecutor at the executor level. +"""Integration tests for MultiprocExecutor at the executor level. This test directly tests the executor without going through the LLM interface, focusing on executor initialization, RPC calls, and distributed execution. """ @@ -377,8 +376,7 @@ def test_multiproc_executor_properties(): @multi_gpu_test(num_gpus=4) def test_multiproc_executor_multi_node(): - """ - Test MultiprocExecutor with multi-node configuration. + """Test MultiprocExecutor with multi-node configuration. This simulates 2 nodes with TP=4: - Node 0 (rank 0): Uses GPUs 0,1 (CUDA_VISIBLE_DEVICES=0,1) with TP=2 - Node 1 (rank 1): Uses GPUs 2,3 (CUDA_VISIBLE_DEVICES=2,3) with TP=2 diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py index a464a61683..8ba93df4f5 100644 --- a/tests/distributed/test_pipeline_parallel.py +++ b/tests/distributed/test_pipeline_parallel.py @@ -1,11 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -WARNING: This test runs in both single-node (4 GPUs) and multi-node - (2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is - important to set the distributed backend to "mp" to avoid Ray scheduling - all workers in a node other than the head node, which can cause the test - to fail. +"""WARNING: This test runs in both single-node (4 GPUs) and multi-node +(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is +important to set the distributed backend to "mp" to avoid Ray scheduling +all workers in a node other than the head node, which can cause the test +to fail. """ import json diff --git a/tests/distributed/test_quick_all_reduce.py b/tests/distributed/test_quick_all_reduce.py index fb2b04abf0..f9228fbd68 100644 --- a/tests/distributed/test_quick_all_reduce.py +++ b/tests/distributed/test_quick_all_reduce.py @@ -396,8 +396,7 @@ def test_custom_quick_allreduce_bf16_cast(monkeypatch: pytest.MonkeyPatch): def qr_variable_input(rank, world_size): - """ - When the tensor parallelism is set to 4 or 8, frequent changes + """When the tensor parallelism is set to 4 or 8, frequent changes in the input shape can cause QuickReduce to hang (this issue has been observed with the gpt_oss model). """ diff --git a/tests/distributed/test_ray_v2_executor.py b/tests/distributed/test_ray_v2_executor.py index bd4a253b96..f9b9e57fc5 100644 --- a/tests/distributed/test_ray_v2_executor.py +++ b/tests/distributed/test_ray_v2_executor.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Integration tests for RayExecutorV2 at the executor level. +"""Integration tests for RayExecutorV2 at the executor level. Validates executor initialization, placement group support, RPC calls, and distributed execution with various TP/PP configurations. """ @@ -299,7 +298,6 @@ def test_ray_v2_run_refs_stored_for_monitoring(executor): @pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)]) def test_ray_v2_single_node_generation(tp_size, pp_size): """End-to-end LLM generation with RayExecutorV2.""" - llm = LLM( model=MODEL, tensor_parallel_size=tp_size, diff --git a/tests/distributed/test_ray_v2_executor_e2e.py b/tests/distributed/test_ray_v2_executor_e2e.py index fb58301326..335324c371 100644 --- a/tests/distributed/test_ray_v2_executor_e2e.py +++ b/tests/distributed/test_ray_v2_executor_e2e.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Orchestration-level integration tests for RayExecutorV2. -""" +"""Orchestration-level integration tests for RayExecutorV2.""" import gc import os @@ -158,8 +156,7 @@ def test_multi_replicas_with_bundle_indices(ray_init): def test_env_var_and_runtime_env_propagation(): - """ - Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env + """Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env propagate to RayWorkerProc actors. """ sentinel_vars = { diff --git a/tests/distributed/test_sharded_rdt_plan.py b/tests/distributed/test_sharded_rdt_plan.py index 7a75f0b7fc..7923c413fc 100644 --- a/tests/distributed/test_sharded_rdt_plan.py +++ b/tests/distributed/test_sharded_rdt_plan.py @@ -845,7 +845,7 @@ class TestChunkModuleScatters: ] def test_scatters_carry_their_own_dtype(self): - """dtype rides the record from the bake, where it is the fake's dtype + """Dtype rides the record from the bake, where it is the fake's dtype AFTER its op chain — not a plan-time lookup of the source name, which would be wrong for any chain that reinterprets dtype.""" layer = _FakeLayer("l") @@ -1110,7 +1110,7 @@ class TestSignalCompleteness: signaled parks a producer credit and hangs end_sync.""" def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2): - """pre / n_layers MoE layers / post. Each layer: one norm (its own + """Pre / n_layers MoE layers / post. Each layer: one norm (its own module, stamp -1) + n_experts expert names stamped ``e // n_local``. The worker's bake covers only ``worker_experts`` (its placement); foreign expert names never copied => dropped entirely, as in the real @@ -1173,7 +1173,7 @@ class TestSignalCompleteness: assert self._signals(plan) == list(range(len(group_lens))) def test_placement_changes_only_the_chunk_count_never_the_signals(self): - """linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6 + """Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6 hits all 4. More chunks per group, identical signal set.""" plans = {} for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])): diff --git a/tests/distributed/test_sharded_rdt_producer.py b/tests/distributed/test_sharded_rdt_producer.py index 5cfab1957d..8147841d9a 100644 --- a/tests/distributed/test_sharded_rdt_producer.py +++ b/tests/distributed/test_sharded_rdt_producer.py @@ -503,7 +503,7 @@ class TestGatherCredit: def test_the_residency_bound_scales_with_the_lookahead( self, server_factory, gather_engine ): - """lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing + """Lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing consumer sees exactly 3 groups gathered before the loop parks.""" server = server_factory(gather_lookahead=2) engine = gather_engine(server, 8, lookahead=2) diff --git a/tests/distributed/test_shm_broadcast.py b/tests/distributed/test_shm_broadcast.py index ef7f523b78..7debed4cbb 100644 --- a/tests/distributed/test_shm_broadcast.py +++ b/tests/distributed/test_shm_broadcast.py @@ -43,6 +43,7 @@ def distributed_run(fn, world_size, timeout=60): fn: Function to run in each process world_size: Number of processes to spawn timeout: Maximum time in seconds to wait for processes (default: 60) + """ number_of_processes = world_size processes = [] @@ -634,11 +635,9 @@ def test_acquire_read_releases_slot_when_reader_raises(): def test_warning_logs(caplog_vllm): - """ - Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals + """Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals when indefinite=False, and are not emitted when indefinite=True. """ - # Patch the warning log interval to every 1 ms during reads with mock.patch( "vllm.distributed.device_communicators.shm_broadcast.VLLM_RINGBUFFER_WARNING_INTERVAL", diff --git a/tests/distributed/test_shm_buffer.py b/tests/distributed/test_shm_buffer.py index e56c71b1c2..6f61c28322 100644 --- a/tests/distributed/test_shm_buffer.py +++ b/tests/distributed/test_shm_buffer.py @@ -12,20 +12,20 @@ from vllm.distributed.device_communicators.shm_object_storage import ( class TestSingleWriterShmRingBuffer(unittest.TestCase): - """Test suite for the ring buffer implementation""" + """Test suite for the ring buffer implementation.""" def setUp(self): - """Set up test fixtures""" + """Set up test fixtures.""" self.buffer_size = 4096 self.ring_buffer = None def tearDown(self): - """Clean up after tests""" + """Clean up after tests.""" if self.ring_buffer: self.ring_buffer.close() def test_buffer_opening(self): - """Test opening an existing buffer""" + """Test opening an existing buffer.""" # First create a buffer self.ring_buffer = SingleWriterShmRingBuffer( data_buffer_size=self.buffer_size, create=True @@ -39,7 +39,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): ) def test_buffer_access(self): - """Test accessing allocated buffers""" + """Test accessing allocated buffers.""" self.ring_buffer = SingleWriterShmRingBuffer( data_buffer_size=self.buffer_size, create=True ) @@ -61,7 +61,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): self.assertEqual(read_id, monotonic_id) def test_memory_error_on_full_buffer(self): - """Test that MemoryError is raised when buffer is full""" + """Test that MemoryError is raised when buffer is full.""" small_buffer_size = 200 self.ring_buffer = SingleWriterShmRingBuffer( data_buffer_size=small_buffer_size, create=True @@ -76,7 +76,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): self.ring_buffer.allocate_buf(1) # Would exceed buffer capacity def test_allocation_and_free(self): - """Test allocation and freeing of buffers""" + """Test allocation and freeing of buffers.""" small_buffer_size = 200 self.ring_buffer = SingleWriterShmRingBuffer( data_buffer_size=small_buffer_size, create=True @@ -96,7 +96,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): self.assertEqual(freed_ids[0], i) def test_clear_buffer(self): - """Test clearing the buffer""" + """Test clearing the buffer.""" self.ring_buffer = SingleWriterShmRingBuffer( data_buffer_size=self.buffer_size, create=True ) @@ -180,7 +180,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): def main(): - """Main function demonstrating usage and running tests""" + """Main function demonstrating usage and running tests.""" print("=== SingleWriterShmRingBuffer Test Suite ===\n") # Run unit tests diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index d6ce5b36b5..68fe250677 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -254,10 +254,13 @@ def test_get_type_hints(type_hint, expected): assert get_type_hints(type_hint) == expected -def test_get_kwargs(): - kwargs = get_kwargs(DummyConfig) - print(kwargs) +@pytest.fixture +def dummy_config_kwargs(): + return get_kwargs(DummyConfig) + +def test_get_kwargs(dummy_config_kwargs): + kwargs = dummy_config_kwargs # bools should not have their type set assert kwargs["regular_bool"].get("type") is None assert kwargs["optional_bool"].get("type") is None @@ -267,7 +270,7 @@ def test_get_kwargs(): assert kwargs["optional_bool_or_str"]["const"] is True assert "action" not in kwargs["optional_bool_or_str"] # optional literals should have None as a choice - assert kwargs["optional_literal"]["choices"] == ["x", "y", "None"] + assert kwargs["optional_literal"]["choices"] == ["x", "y", None] # tuples should have the correct nargs assert kwargs["tuple_n"]["nargs"] == "+" assert kwargs["tuple_2"]["nargs"] == 2 @@ -293,6 +296,22 @@ def test_get_kwargs(): assert kwargs["nested_config"]["type"]('{"field": 2}') == NestedConfig(2) # type: ignore[call-arg] +@pytest.mark.parametrize( + ("args", "expected"), + [ + (["--optional-literal", "None"], None), + (["--optional-literal", ""], None), + (["--optional-literal", "x"], "x"), + ], +) +def test_optional_handling(args, expected, dummy_config_kwargs): + parser = FlexibleArgumentParser() + parser.add_argument("--optional-literal", **dummy_config_kwargs["optional_literal"]) + + assert parser.parse_args(args).optional_literal is expected + assert "None" in parser.format_help() + + def test_jit_monitor_verbose_arg(): parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) args = parser.parse_args(["--jit-monitor-verbose"]) @@ -375,8 +394,7 @@ def test_media_io_kwargs_parser(arg, expected): ], ) def test_optimization_level(args, expected): - """ - Test space-separated optimization levels (-O 1, -O 2, -O 3) map to + """Test space-separated optimization levels (-O 1, -O 2, -O 3) map to optimization_level. """ parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) @@ -395,9 +413,7 @@ def test_optimization_level(args, expected): ], ) def test_mode_parser(args, expected): - """ - Test compilation config modes (-cc.mode=int) map to compilation_config. - """ + """Test compilation config modes (-cc.mode=int) map to compilation_config.""" parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) parsed_args = parser.parse_args(args) assert parsed_args.compilation_config.mode == expected diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 0c986f1291..226908d673 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -480,7 +480,7 @@ class TestThinkingBlockConversion: """ def test_thinking_plus_text_in_assistant_message(self): - """thinking + text → reasoning field + plain-string content.""" + """Thinking + text → reasoning field + plain-string content.""" request = _make_request( [ {"role": "user", "content": "Write me some code."}, @@ -542,7 +542,7 @@ class TestThinkingBlockConversion: assert asst.get("content") is None def test_thinking_plus_tool_use_in_assistant_message(self): - """thinking + tool_use: reasoning field set, tool_calls populated.""" + """Thinking + tool_use: reasoning field set, tool_calls populated.""" request = _make_request( [ {"role": "user", "content": "What is 2+2?"}, diff --git a/tests/entrypoints/launchers/test_cli_args.py b/tests/entrypoints/launchers/test_cli_args.py index e84abe5d26..0bcca8cb41 100644 --- a/tests/entrypoints/launchers/test_cli_args.py +++ b/tests/entrypoints/launchers/test_cli_args.py @@ -152,7 +152,7 @@ def test_multiple_valid_inputs(serve_parser): ### Tests for serve argument validation that run prior to loading def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser): - """Ensure validation fails if tool choice is enabled with no call parser""" + """Ensure validation fails if tool choice is enabled with no call parser.""" # If we enable-auto-tool-choice, explode with no tool-call-parser args = serve_parser.parse_args(args=["--enable-auto-tool-choice"]) with pytest.raises(TypeError): @@ -160,7 +160,7 @@ def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser): def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser): - """Ensure validation passes with tool choice enabled with a call parser""" + """Ensure validation passes with tool choice enabled with a call parser.""" args = serve_parser.parse_args( args=[ "--enable-auto-tool-choice", @@ -172,7 +172,7 @@ def test_enable_auto_choice_passes_with_tool_call_parser(serve_parser): def test_enable_auto_choice_fails_with_enable_reasoning(serve_parser): - """Ensure validation fails if reasoning is enabled with auto tool choice""" + """Ensure validation fails if reasoning is enabled with auto tool choice.""" args = serve_parser.parse_args( args=[ "--enable-auto-tool-choice", @@ -197,7 +197,7 @@ def test_passes_with_reasoning_parser(serve_parser): def test_chat_template_validation_for_happy_paths(serve_parser): - """Ensure validation passes if the chat template exists""" + """Ensure validation passes if the chat template exists.""" args = serve_parser.parse_args( args=["--chat-template", CHATML_JINJA_PATH.absolute().as_posix()] ) @@ -205,7 +205,7 @@ def test_chat_template_validation_for_happy_paths(serve_parser): def test_chat_template_validation_for_sad_paths(serve_parser): - """Ensure validation fails if the chat template doesn't exist""" + """Ensure validation fails if the chat template doesn't exist.""" args = serve_parser.parse_args(args=["--chat-template", "does/not/exist"]) with pytest.raises(VLLMValidationError): validate_parsed_serve_args(args) @@ -304,13 +304,13 @@ def test_sse_keep_alive_interval_non_integer(serve_parser, value): ], ) def test_middleware(serve_parser, cli_args, expected_middleware): - """Ensure multiple middleware args are parsed properly""" + """Ensure multiple middleware args are parsed properly.""" args = serve_parser.parse_args(args=cli_args) assert args.middleware == expected_middleware def test_default_chat_template_kwargs_parsing(serve_parser): - """Ensure default_chat_template_kwargs JSON is parsed correctly""" + """Ensure default_chat_template_kwargs JSON is parsed correctly.""" args = serve_parser.parse_args( args=["--default-chat-template-kwargs", '{"enable_thinking": false}'] ) @@ -318,7 +318,7 @@ def test_default_chat_template_kwargs_parsing(serve_parser): def test_default_chat_template_kwargs_complex(serve_parser): - """Ensure complex default_chat_template_kwargs JSON is parsed correctly""" + """Ensure complex default_chat_template_kwargs JSON is parsed correctly.""" kwargs_json = '{"enable_thinking": false, "custom_param": "value", "num": 42}' args = serve_parser.parse_args(args=["--default-chat-template-kwargs", kwargs_json]) assert args.default_chat_template_kwargs == { @@ -329,7 +329,7 @@ def test_default_chat_template_kwargs_complex(serve_parser): def test_default_chat_template_kwargs_default_none(serve_parser): - """Ensure default_chat_template_kwargs defaults to None""" + """Ensure default_chat_template_kwargs defaults to None.""" args = serve_parser.parse_args(args=[]) assert args.default_chat_template_kwargs is None @@ -355,7 +355,7 @@ def test_resolve_default_chat_template_kwargs(default_kwargs, cohere_format, exp def test_default_chat_template_kwargs_invalid_json(serve_parser): - """Ensure invalid JSON raises an error""" + """Ensure invalid JSON raises an error.""" with pytest.raises(SystemExit): serve_parser.parse_args( args=["--default-chat-template-kwargs", "not valid json"] @@ -402,7 +402,7 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises): ### Tests for LoRA target modules parsing def test_lora_target_modules_single(serve_parser): - """Test parsing single lora-target-modules argument""" + """Test parsing single lora-target-modules argument.""" args = serve_parser.parse_args( args=["--enable-lora", "--lora-target-modules", "o_proj"] ) @@ -410,7 +410,7 @@ def test_lora_target_modules_single(serve_parser): def test_lora_target_modules_multiple(serve_parser): - """Test parsing multiple lora-target-modules arguments""" + """Test parsing multiple lora-target-modules arguments.""" args = serve_parser.parse_args( args=[ "--enable-lora", @@ -424,6 +424,6 @@ def test_lora_target_modules_multiple(serve_parser): def test_lora_target_modules_default_none(serve_parser): - """Test that lora-target-modules defaults to None""" + """Test that lora-target-modules defaults to None.""" args = serve_parser.parse_args(args=[]) assert args.lora_target_modules is None diff --git a/tests/entrypoints/launchers/test_dp_supervisor.py b/tests/entrypoints/launchers/test_dp_supervisor.py index eb5bf0a088..f96b020d10 100644 --- a/tests/entrypoints/launchers/test_dp_supervisor.py +++ b/tests/entrypoints/launchers/test_dp_supervisor.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for DPSupervisor: unit tests and lifecycle integration tests. +"""Tests for DPSupervisor: unit tests and lifecycle integration tests. Lifecycle integration tests replace child vLLM servers with lightweight aiohttp "fake" servers controlled by the test, so the suite runs without GPUs. @@ -463,8 +462,7 @@ async def test_start_server_no_log_config_when_no_filter(monkeypatch): class MockVLLMServer: - """ - Minimal FastAPI server that mimics one vLLM replica. + """Minimal FastAPI server that mimics one vLLM replica. GET /health returns 200 when healthy, 503 otherwise. Health state is toggled by the test via set_healthy(). """ @@ -582,8 +580,7 @@ def launch_mock_vllm_with_drain( async def _poll_supervisor_health(expected_status: int, use_ssl: bool = False) -> bool: - """ - GET /health on the supervisor once and check for expected_status. + """GET /health on the supervisor once and check for expected_status. Pass expected_status=-1 to assert the supervisor is not listening yet (a connection error is expected). The supervisor only starts its HTTP @@ -705,8 +702,7 @@ async def _run_supervisor( @pytest.mark.asyncio async def test_basic_lifecycle(monkeypatch): - """ - A) Supervisor is not listening while children are unhealthy. + """A) Supervisor is not listening while children are unhealthy. B) /health returns 200 once every child reports healthy. C) SIGTERM and shutdown """ @@ -777,8 +773,7 @@ async def test_basic_lifecycle_with_ssl(monkeypatch): @pytest.mark.asyncio async def test_failed_startup(monkeypatch): - """ - A) One of the vLLM servers crashes during startup. + """A) One of the vLLM servers crashes during startup. B) DPSupervisor detects this, and cleans up resources. """ args = _make_args() @@ -801,8 +796,7 @@ async def test_failed_startup(monkeypatch): @pytest.mark.asyncio async def test_becomes_unhealthy(monkeypatch): - """ - A) Supervisor is not listening while children are unhealthy. + """A) Supervisor is not listening while children are unhealthy. B) /health returns 200 once every child reports healthy. C) Child process becomes unhealthy. D) Detected and shutdown. @@ -842,8 +836,7 @@ async def test_becomes_unhealthy(monkeypatch): @pytest.mark.asyncio async def test_dp_server_fails(monkeypatch): - """ - A) Supervisor is not listening while children are unhealthy. + """A) Supervisor is not listening while children are unhealthy. B) /health returns 200 once every child reports healthy. C) Child process fails. D) Detected and shutdown. @@ -886,8 +879,7 @@ async def test_dp_server_fails(monkeypatch): @pytest.mark.asyncio async def test_shutdown_timeout(monkeypatch: pytest.MonkeyPatch): - """ - Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight + """Child mock servers delay shutdown by 10s on SIGTERM (simulating in-flight request drain). The supervisor is configured with shutdown_timeout=10, so its total wait budget is 10 + CHILD_EXIT_GRACE_S seconds. The children exit naturally within that window, so no force-kill should occur diff --git a/tests/entrypoints/launchers/test_run_batch.py b/tests/entrypoints/launchers/test_run_batch.py index 7e6ba3d78d..f7a2505807 100644 --- a/tests/entrypoints/launchers/test_run_batch.py +++ b/tests/entrypoints/launchers/test_run_batch.py @@ -509,9 +509,7 @@ def test_completions(): def test_completions_invalid_input(): - """ - Ensure that we fail when the input doesn't conform to the openai api. - """ + """Ensure that we fail when the input doesn't conform to the openai api.""" with ( tempfile.NamedTemporaryFile("w") as input_file, tempfile.NamedTemporaryFile("r") as output_file, @@ -602,9 +600,7 @@ def test_score(input_batch): def test_reasoning_parser(): - """ - Test that reasoning_parser parameter works correctly in run_batch. - """ + """Test that reasoning_parser parameter works correctly in run_batch.""" with ( tempfile.NamedTemporaryFile("w") as input_file, tempfile.NamedTemporaryFile("r") as output_file, @@ -767,8 +763,7 @@ def test_translation(): def test_tool_calling(): - """ - Test that tool calling works correctly in run_batch. + """Test that tool calling works correctly in run_batch. Verifies that requests with tools return tool_calls in the response. """ with ( diff --git a/tests/entrypoints/launchers/test_shutdown.py b/tests/entrypoints/launchers/test_shutdown.py index 643eff7594..6d18d825eb 100644 --- a/tests/entrypoints/launchers/test_shutdown.py +++ b/tests/entrypoints/launchers/test_shutdown.py @@ -134,7 +134,6 @@ async def test_shutdown_on_engine_failure(tmp_path: Path): Starts a vLLM server, kills it to simulate a crash, then verifies that subsequent API calls fail appropriately. """ - port = get_open_port() # Redirect server output to a file rather than a pipe: nothing drains the diff --git a/tests/entrypoints/llm/offline_mode/test_offline_mode.py b/tests/entrypoints/llm/offline_mode/test_offline_mode.py index 555e8b9b7d..ab7edd019b 100644 --- a/tests/entrypoints/llm/offline_mode/test_offline_mode.py +++ b/tests/entrypoints/llm/offline_mode/test_offline_mode.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for HF_HUB_OFFLINE mode""" +"""Tests for HF_HUB_OFFLINE mode.""" import importlib import sys diff --git a/tests/entrypoints/llm/test_accuracy.py b/tests/entrypoints/llm/test_accuracy.py index af607720c8..49d83b9cf6 100644 --- a/tests/entrypoints/llm/test_accuracy.py +++ b/tests/entrypoints/llm/test_accuracy.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file test accuracy of the vLLM server via LMEval. +"""This file test accuracy of the vLLM server via LMEval. It uses local-completions, which interacts with vLLM through the OAI API with N concurrent connections. This simulates real work usage of the API and makes @@ -33,7 +32,6 @@ EXPECTED_VALUES = { def run_test(model_name, more_args=None): """Run the end to end accuracy test.""" - model_args = f"pretrained={model_name},max_model_len=4096" if more_args is not None: @@ -64,7 +62,6 @@ TPU_TP_TEST_STR = "" # "tensor_parallel_size=4" @pytest.mark.parametrize("model", MODEL_NAMES) def test_lm_eval_accuracy_v1_engine(model): """Run with the V1 Engine.""" - more_args = None if current_platform.is_tpu(): # Limit compilation time for TPU V1 @@ -81,7 +78,6 @@ def test_lm_eval_accuracy_v1_engine(model): @pytest.mark.parametrize("model", FP8_KV_MODEL_NAMES) def test_lm_eval_accuracy_v1_engine_fp8_kv_cache(model): """Run with the V1 Engine.""" - more_args = None if current_platform.is_tpu(): # Limit compilation time for TPU V1 diff --git a/tests/entrypoints/llm/test_chat.py b/tests/entrypoints/llm/test_chat.py index 6ee8d34ce1..276426e692 100644 --- a/tests/entrypoints/llm/test_chat.py +++ b/tests/entrypoints/llm/test_chat.py @@ -20,8 +20,7 @@ def text_llm(vllm_runner): @pytest.fixture(scope="function") def llm_for_failure_test(vllm_runner): - """ - Fixture for testing issue #26081. + """Fixture for testing issue #26081. Uses a small max_model_len to easily trigger length errors. """ with vllm_runner( @@ -67,8 +66,7 @@ def test_multi_chat(text_llm): def test_llm_chat_tokenization_no_double_bos(text_llm): - """ - LLM.chat() should not add special tokens when using chat templates. + """LLM.chat() should not add special tokens when using chat templates. Check we get a single BOS token for llama chat. """ messages = [ @@ -127,8 +125,7 @@ def test_chat_extra_kwargs(thinking_llm, enable_thinking): def test_chat_batch_failure_cleanup(llm_for_failure_test): - """ - Tests that if a batch call to llm.chat() fails mid-way + """Tests that if a batch call to llm.chat() fails mid-way (e.g., due to one invalid prompt), the requests that were already enqueued are properly aborted and do not pollute the queue for subsequent calls. diff --git a/tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py b/tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py index 5c378de35d..a3511b9528 100644 --- a/tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py +++ b/tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects +"""Test that ``InputProcessor.inject_into_mm_cache()`` correctly injects pre-processed mm_kwargs into the processor cache and reports MM cache hit rate metrics accurately. diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py index 14ac7ce043..5031cbea1a 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py @@ -47,8 +47,7 @@ async def client(server): async def test_online_audio_in_video( client: openai.AsyncOpenAI, video_assets: VideoTestAssets ): - """Test video input with `audio_in_video=True`""" - + """Test video input with `audio_in_video=True`.""" # we don't use video_urls above because they missed audio stream. video_path = video_assets[0].video_path with open(video_path, "rb") as f: @@ -97,8 +96,7 @@ async def test_online_audio_in_video( async def test_online_audio_in_video_multi_videos( client: openai.AsyncOpenAI, video_assets: VideoTestAssets ): - """Test multi-video input with `audio_in_video=True`""" - + """Test multi-video input with `audio_in_video=True`.""" # we don't use video_urls above because they missed audio stream. video_path = video_assets[0].video_path with open(video_path, "rb") as f: @@ -151,8 +149,7 @@ async def test_online_audio_in_video_multi_videos( async def test_online_audio_in_video_interleaved( client: openai.AsyncOpenAI, video_assets: VideoTestAssets ): - """Test interleaved video/audio input with `audio_in_video=True`""" - + """Test interleaved video/audio input with `audio_in_video=True`.""" # we don't use video_urls above because they missed audio stream. video_path = video_assets[0].video_path with open(video_path, "rb") as f: diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_vision.py b/tests/entrypoints/multimodal/openai/chat_completion/test_vision.py index e6b8dc23e9..56a7cf6a13 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_vision.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_vision.py @@ -34,8 +34,7 @@ REQUIRED_BEAM_SEARCH_TERMS = [ def check_output_matches_terms(content: str, term_groups: list[list[str]]) -> bool: - """ - Check if content matches all required term groups. + """Check if content matches all required term groups. Each term group requires at least one of its terms to be present. All term groups must be satisfied. """ diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index 20218ac6c8..467e23e7d3 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -259,7 +259,7 @@ async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput @pytest.mark.asyncio @pytest.mark.skip_global_cleanup async def test_batched_echo_does_not_prepend_user_prompt() -> None: - """echo must only echo the assistant turn, never the user's prompt. + """Echo must only echo the assistant turn, never the user's prompt. With `add_generation_prompt` (the default) the response role is `assistant` while the last conversation message is the user's, so there is nothing to diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index aafb899f00..b83341cc50 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -121,7 +121,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat: @pytest.mark.asyncio async def test_chat_error_non_stream(): - """test finish_reason='error' returns 500 InternalServerError (non-streaming)""" + """Test finish_reason='error' returns 500 InternalServerError (non-streaming).""" mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig() @@ -246,7 +246,7 @@ async def test_renderer_only_chat_request_skips_mm_cache(): @pytest.mark.asyncio async def test_chat_error_stream(): - """test finish_reason='error' returns 500 InternalServerError (streaming)""" + """Test finish_reason='error' returns 500 InternalServerError (streaming).""" mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig() diff --git a/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py b/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py index 3e59e7402e..61fd2f1576 100644 --- a/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py +++ b/tests/entrypoints/openai/chat_completion/test_completion_with_function_calling.py @@ -451,7 +451,7 @@ async def test_inconsistent_tool_choice_and_tools( async def test_max_tokens_with_tool_choice_required( client: openai.AsyncOpenAI, tool_choice ): - """ """ + """Requesting a single token with a required tool choice must not crash.""" models = await client.models.list() model_name: str = models.data[0].id diff --git a/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py b/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py index 8c03fc1daa..f4036c4374 100644 --- a/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py +++ b/tests/entrypoints/openai/chat_completion/test_extra_content_fields.py @@ -52,7 +52,7 @@ def stop_sign_image_url(): @pytest.mark.asyncio async def test_translategemma_extra_lang_code_fields(client: openai.AsyncOpenAI): - """en -> es translation through TranslateGemma's bundled chat template's + """En -> es translation through TranslateGemma's bundled chat template's ``text`` content branch, which depends on ``source_lang_code`` / ``target_lang_code`` extra fields being preserved on the content part.""" completion = await client.chat.completions.create( @@ -92,7 +92,7 @@ async def test_translategemma_image_extra_lang_code_fields( client: openai.AsyncOpenAI, stop_sign_image_url: str, ): - """en -> es OCR-translation through TranslateGemma's bundled chat + """En -> es OCR-translation through TranslateGemma's bundled chat template's ``image`` content branch. Exercises the multimodal branch of ``_collect_extra_fields`` in ``_parse_chat_message_content_part``: the ``source_lang_code`` / ``target_lang_code`` extras must survive parsing diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index e7ea15877c..2757320362 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -1232,7 +1232,6 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated(): """Regression test: when the Mistral tokenizer path returns token IDs directly, we must still apply input length + max_tokens validation. """ - mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True) @@ -1271,7 +1270,6 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected(): """Regression test: MistralTokenizer token-id prompts must still enforce the max context length for the input itself (token_num >= max_model_len). """ - mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True) @@ -1512,8 +1510,7 @@ async def _render_chat_prompt_token_ids( class TestServingChatWithHarmony: - """ - These tests ensure Chat Completion requests are being properly converted into + """These tests ensure Chat Completion requests are being properly converted into Harmony messages and Harmony response messages back into Chat Completion responses. These tests are not exhaustive, but each one was created to cover a specific case that we got wrong but is now fixed. diff --git a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py index ec7a977f1a..ecff3f8ee6 100644 --- a/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py +++ b/tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py @@ -33,7 +33,6 @@ def _count_reasoning_decode_token_ids_between_markers( reasoning_end_ids: list[int], ) -> int | None: """Count decode tokens in the thinking span (after last start, before first end).""" - if not reasoning_start_ids or not reasoning_end_ids: raise ValueError("reasoning marker token id lists must be non-empty") @@ -153,7 +152,6 @@ async def client(request, server, server_with_auto_reasoning_config): async def test_thinking_token_budget_mixed_requests(client: openai.AsyncOpenAI): """Test that mixed requests (some with thinking_token_budget, some without) complete successfully without errors.""" - response_with_budget = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES, @@ -182,7 +180,6 @@ async def test_thinking_token_budget_limits_reasoning(client: openai.AsyncOpenAI grouped into streamed chunks (a single chunk can carry several tokens under async scheduling / stream_interval > 1). Counting chunks under-counts. """ - tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME) start_ids = list(tokenizer.encode(REASONING_START_STR, add_special_tokens=False)) end_ids = list(tokenizer.encode(REASONING_END_STR, add_special_tokens=False)) @@ -227,7 +224,6 @@ async def test_thinking_token_budget_qwen35_fp8_mtp_concurrent_mixed_budget_and_ Qwen3.5 FP8 + MTP (TP=2) server. Budgeted calls are checked with ``_count_reasoning_decode_token_ids_between_markers`` on full token ids. """ - _batch_spec: list[tuple[Literal["budget"], int] | tuple[Literal["plain"], None]] = [ ("budget", 1), ("budget", 12), diff --git a/tests/entrypoints/openai/completion/test_completion.py b/tests/entrypoints/openai/completion/test_completion.py index 1bc6260cde..2876642241 100644 --- a/tests/entrypoints/openai/completion/test_completion.py +++ b/tests/entrypoints/openai/completion/test_completion.py @@ -350,7 +350,6 @@ async def test_parallel_no_streaming(client: openai.AsyncOpenAI, model_name: str """Parallel sampling without streaming. A single request output contains a list of completions. """ - prompt = "What is an LLM?" n = 3 max_tokens = 50 @@ -396,7 +395,6 @@ async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str): The tokens from multiple samples, are flattened into a single stream, with an index to indicate which sample the token belongs to. """ - prompt = "What is an LLM?" n = 3 max_tokens = 50 diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 513bde465d..46f4fe452c 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -303,7 +303,7 @@ def test_completion_metrics_carries_both_timing_and_spec_decode(): @pytest.mark.asyncio async def test_completion_error_non_stream(): - """test finish_reason='error' returns 500 InternalServerError (non-streaming)""" + """Test finish_reason='error' returns 500 InternalServerError (non-streaming).""" mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig() @@ -439,7 +439,7 @@ async def test_renderer_only_completion_request_skips_mm_cache(): @pytest.mark.asyncio async def test_completion_error_stream(): - """test finish_reason='error' returns 500 InternalServerError (streaming)""" + """Test finish_reason='error' returns 500 InternalServerError (streaming).""" mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False mock_engine.model_config = MockModelConfig() diff --git a/tests/entrypoints/openai/completion/test_token_in_token_out.py b/tests/entrypoints/openai/completion/test_token_in_token_out.py index 8882ae6244..45024d8277 100644 --- a/tests/entrypoints/openai/completion/test_token_in_token_out.py +++ b/tests/entrypoints/openai/completion/test_token_in_token_out.py @@ -39,8 +39,7 @@ def server(): @pytest.mark.asyncio async def test_token_in_token_out_and_logprobs(server): - """ - Test token-in-token-out and token_ids align with prompt_logprobs + """Test token-in-token-out and token_ids align with prompt_logprobs & logprobs when return_tokens_as_token_ids is enabled. """ tokenizer = get_tokenizer(tokenizer_name=MODEL_NAME) diff --git a/tests/entrypoints/openai/correctness/test_lmeval.py b/tests/entrypoints/openai/correctness/test_lmeval.py index aad1b5e062..c6ce3ebc2b 100644 --- a/tests/entrypoints/openai/correctness/test_lmeval.py +++ b/tests/entrypoints/openai/correctness/test_lmeval.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file test accuracy of the vLLM server via LMEval. +"""This file test accuracy of the vLLM server via LMEval. It uses local-completions, which interacts with vLLM through the OAI API with N concurrent connections. This simulates real work usage of the API and makes @@ -37,7 +36,6 @@ if current_platform.is_tpu(): def run_test(more_args): """Run the end to end accuracy test.""" - args = list(DEFAULT_ARGS) args.extend(more_args) print(f"Running with: {args}") @@ -68,7 +66,6 @@ def run_test(more_args): def test_lm_eval_accuracy_v1_engine(): """Run with the V1 Engine.""" - more_args = [] # Limit compilation time for V1 on TPU diff --git a/tests/entrypoints/openai/parser/test_harmony_utils.py b/tests/entrypoints/openai/parser/test_harmony_utils.py index 1b19714ac2..4c1dfe52e0 100644 --- a/tests/entrypoints/openai/parser/test_harmony_utils.py +++ b/tests/entrypoints/openai/parser/test_harmony_utils.py @@ -170,7 +170,7 @@ class TestIsFunctionRecipientWithAllowedNames: """Tests for is_function_recipient with allowed_function_tool_names.""" def test_prefixed_always_accepted(self): - """functions. prefix is always accepted regardless of allowed names.""" + """Functions. prefix is always accepted regardless of allowed names.""" fn_names = frozenset({"other_tool"}) assert is_function_recipient("functions.get_weather", fn_names) is True @@ -252,8 +252,7 @@ class TestExtractFunctionFromRecipient: class TestCommonParseInputToHarmonyMessage: - """ - Tests for scenarios that are common to both Chat Completion + """Tests for scenarios that are common to both Chat Completion parse_chat_input_to_harmony_message and Responses API response_previous_input_to_harmony functions. """ @@ -448,8 +447,7 @@ class TestCommonParseInputToHarmonyMessage: class TestParseChatInputToHarmonyMessage: - """ - Tests for scenarios that are specific to the Chat Completion API + """Tests for scenarios that are specific to the Chat Completion API parse_chat_input_to_harmony_message function. """ diff --git a/tests/entrypoints/openai/responses/test_errors.py b/tests/entrypoints/openai/responses/test_errors.py index 0ea7833da9..1e0182ba6b 100644 --- a/tests/entrypoints/openai/responses/test_errors.py +++ b/tests/entrypoints/openai/responses/test_errors.py @@ -14,7 +14,7 @@ from vllm.exceptions import GenerationError @pytest.mark.asyncio async def test_raise_if_error_raises_generation_error(): - """test _raise_if_error raises GenerationError""" + """Test _raise_if_error raises GenerationError.""" # create a minimal GenerateBaseServing instance mock_engine = MagicMock() mock_engine.model_config = MagicMock() @@ -42,7 +42,7 @@ async def test_raise_if_error_raises_generation_error(): @pytest.mark.asyncio async def test_convert_generation_error_to_streaming_response(): - """test _convert_generation_error_to_streaming_response output""" + """Test _convert_generation_error_to_streaming_response output.""" mock_engine = MagicMock() mock_engine.model_config = MagicMock() mock_engine.model_config.max_model_len = 100 diff --git a/tests/entrypoints/openai/responses/test_function_call.py b/tests/entrypoints/openai/responses/test_function_call.py index 9dcbd74c89..006ee0f6f2 100644 --- a/tests/entrypoints/openai/responses/test_function_call.py +++ b/tests/entrypoints/openai/responses/test_function_call.py @@ -168,8 +168,7 @@ async def test_max_tokens_with_tool_choice_required( @pytest.mark.asyncio async def test_named_tool_use(client: openai.AsyncOpenAI): def get_weather(latitude: float, longitude: float) -> str: - """ - Mock function to simulate getting weather data. + """Mock function to simulate getting weather data. In a real application, this would call an external weather API. """ return f"Current temperature at ({latitude}, {longitude}) is 20°C." diff --git a/tests/entrypoints/openai/responses/test_function_call_parsing.py b/tests/entrypoints/openai/responses/test_function_call_parsing.py index f90a641db7..5b70c7819a 100644 --- a/tests/entrypoints/openai/responses/test_function_call_parsing.py +++ b/tests/entrypoints/openai/responses/test_function_call_parsing.py @@ -57,7 +57,6 @@ def test_direct_function_call_object_preservation(): def test_mixed_input_types_with_function_calls(): """Test parsing with mixed input types including function calls.""" - request_data = { "model": "gpt-oss", "input": [ @@ -177,7 +176,6 @@ def test_empty_list_input(): def test_function_call_output_not_affected(): """Test that FunctionCallOutput is not affected by the function_call parsing.""" - # Test with FunctionCallOutput as dict (should not be parsed) request_data = { "model": "gpt-oss", @@ -265,7 +263,6 @@ def test_function_call_validation_failure_logs_debug(caplog): def test_validator_handles_iterator_input(): """Test that validator can handle ValidatorIterator input (Pydantic internal).""" - # This test simulates when Pydantic passes a ValidatorIterator instead of a list # This happened with complex nested structures containing reasoning + function_call diff --git a/tests/entrypoints/openai/responses/test_harmony_utils.py b/tests/entrypoints/openai/responses/test_harmony_utils.py index 54c034de44..b524842467 100644 --- a/tests/entrypoints/openai/responses/test_harmony_utils.py +++ b/tests/entrypoints/openai/responses/test_harmony_utils.py @@ -20,8 +20,7 @@ from vllm.exceptions import VLLMValidationError class TestResponsePreviousInputToHarmony: - """ - Tests for scenarios that are specific to the Responses API + """Tests for scenarios that are specific to the Responses API response_previous_input_to_harmony function. """ diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py index 01293f62b2..85bf6f2020 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py +++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py @@ -36,9 +36,6 @@ class _NoOpParser(DelegatingParser): def is_reasoning_end(self, input_ids: list[int]) -> bool: return False - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - return input_ids - def extract_reasoning(self, model_output, request): return None, model_output @@ -63,9 +60,6 @@ class _ReasoningOnlyParser(DelegatingParser): def is_reasoning_end(self, input_ids: list[int]) -> bool: return False - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - return input_ids - def extract_reasoning(self, model_output, request): if "" in model_output and "" in model_output: start = model_output.index("") + len("") @@ -131,9 +125,6 @@ class _ToolCallingParser(DelegatingParser): def is_reasoning_end(self, input_ids: list[int]) -> bool: return False - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - return input_ids - def extract_reasoning(self, model_output, request): return None, model_output diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index bd41b8cda2..1a315902e2 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -18,6 +18,9 @@ from openai.types.responses import ( ResponseTextConfig, ResponseTextDeltaEvent, ) +from openai.types.responses import ( + ResponseUsage as OpenAIResponseUsage, +) from openai.types.responses.response_format_text_json_schema_config import ( ResponseFormatTextJSONSchemaConfig, ) @@ -45,6 +48,7 @@ from vllm.entrypoints.openai.responses.context import ( SimpleContext, ) from vllm.entrypoints.openai.responses.protocol import ( + ResponseCompletedEvent, ResponseCreatedEvent, ResponseRawMessageAndToken, ResponsesRequest, @@ -68,6 +72,7 @@ from vllm.renderers.online_renderer import ( _extract_allowed_tools_from_mcp_requests, ) from vllm.sampling_params import SamplingParams +from vllm.v1.metrics.stats import RequestStateStats pytestmark = pytest.mark.skip_global_cleanup @@ -81,7 +86,7 @@ def _new_online_renderer() -> OnlineRenderer: class MockConversationContext(ConversationContext): - """Mock conversation context for testing""" + """Mock conversation context for testing.""" def __init__(self): self.init_tool_sessions_called = False @@ -123,7 +128,7 @@ def test_serialize_message_pydantic_model_returns_dict() -> None: @pytest.fixture def mock_serving_responses(): - """Create a mock OpenAIServingResponses instance""" + """Create a mock OpenAIServingResponses instance.""" serving_responses = MagicMock(spec=OpenAIServingResponses) serving_responses.tool_server = MagicMock(spec=ToolServer) return serving_responses @@ -131,13 +136,13 @@ def mock_serving_responses(): @pytest.fixture def mock_context(): - """Create a mock conversation context""" + """Create a mock conversation context.""" return MockConversationContext() @pytest.fixture def mock_exit_stack(): - """Create a mock async exit stack""" + """Create a mock async exit stack.""" return MagicMock(spec=AsyncExitStack) @@ -550,7 +555,7 @@ def test_responses_render_result_rejects_non_single_prompt(engine_inputs): class TestInitializeToolSessions: - """Test class for _initialize_tool_sessions method""" + """Test class for _initialize_tool_sessions method.""" @pytest.fixture(params=[ParsableContext, HarmonyContext]) def tool_context(self, request): @@ -635,7 +640,7 @@ class TestInitializeToolSessions: @pytest_asyncio.fixture async def serving_responses_instance(self): - """Create a real OpenAIServingResponses instance for testing""" + """Create a real OpenAIServingResponses instance for testing.""" # Create minimal mocks for required dependencies engine_client = MagicMock() @@ -792,6 +797,7 @@ class TestInitializeToolSessions: pass assert serving_responses_instance.engine_client.generate.call_count == 2 + assert context.request_metrics_cover_all_generation_turns is False followup_engine_input = ( serving_responses_instance.engine_client.generate.call_args_list[1].args[0] ) @@ -802,8 +808,7 @@ class TestInitializeToolSessions: async def test_initialize_tool_sessions( self, serving_responses_instance, mock_context, mock_exit_stack ): - """Test that method works correctly with only MCP tools""" - + """Test that method works correctly with only MCP tools.""" request = ResponsesRequest(input="test input", tools=[]) # Call the method @@ -852,11 +857,11 @@ class TestInitializeToolSessions: class TestValidateGeneratorInput: - """Test class for _validate_generator_input method""" + """Test class for _validate_generator_input method.""" @pytest_asyncio.fixture async def serving_responses_instance(self): - """Create a real OpenAIServingResponses instance for testing""" + """Create a real OpenAIServingResponses instance for testing.""" # Create minimal mocks for required dependencies engine_client = MagicMock() @@ -884,7 +889,7 @@ class TestValidateGeneratorInput: return instance def test_validate_generator_input(self, serving_responses_instance): - """Test _validate_generator_input with valid prompt length""" + """Test _validate_generator_input with valid prompt length.""" # Create an engine prompt with valid length (less than max_model_len) valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len engine_input = tokens_input(valid_prompt_token_ids) @@ -976,6 +981,7 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): outputs=[completion], finished=True, num_cached_tokens=0, + num_cache_creation_tokens=3, ) context.append_output(req_output) @@ -996,10 +1002,13 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch): ) assert response.usage.output_tokens_details.reasoning_tokens == 1 + assert response.usage.input_tokens_details.cache_write_tokens == 3 + + OpenAIResponseUsage.model_validate(response.usage.model_dump(mode="json")) class TestExtractAllowedToolsFromMcpRequests: - """Test class for _extract_allowed_tools_from_mcp_requests function""" + """Test class for _extract_allowed_tools_from_mcp_requests function.""" def test_extract_allowed_tools_basic_formats(self): """Test extraction with list format, object format, and None.""" @@ -1113,7 +1122,7 @@ class TestHarmonyPreambleStreaming: return item def test_preamble_delta_emits_text_events(self) -> None: - """commentary + recipient=None should emit output_text.delta events.""" + """Commentary + recipient=None should emit output_text.delta events.""" from vllm.entrypoints.openai.responses.streaming_events import ( emit_content_delta_events, ) @@ -1146,7 +1155,7 @@ class TestHarmonyPreambleStreaming: assert "response.output_item.added" not in type_names def test_commentary_with_function_recipient_not_preamble(self) -> None: - """commentary + recipient='functions.X' must NOT use preamble path.""" + """Commentary + recipient='functions.X' must NOT use preamble path.""" from vllm.entrypoints.openai.responses.streaming_events import ( emit_content_delta_events, ) @@ -1184,7 +1193,7 @@ class TestHarmonyPreambleStreaming: assert "response.output_item.done" in type_names def test_commentary_with_recipient_no_preamble_done(self) -> None: - """commentary + recipient='functions.X' should route to function call + """Commentary + recipient='functions.X' should route to function call done, not preamble done.""" from vllm.entrypoints.openai.responses.streaming_events import ( emit_previous_item_done_events, @@ -1242,9 +1251,19 @@ class TestHarmonyPreambleStreaming: assert "response.output_item.done" in type_names -def _make_simple_context_with_output(text, token_ids, response_parser=None): - """Create a SimpleContext with a RequestOutput containing the given text.""" - ctx = SimpleContext(response_parser=response_parser) +_PER_REQUEST_STATS = RequestStateStats( + queued_ts=1.0, + scheduled_ts=1.5, + first_token_ts=2.0, + last_token_ts=3.0, +) + + +def _make_request_output( + text, + token_ids, + metrics: RequestStateStats | None = None, +): completion = CompletionOutput( index=0, text=text, @@ -1254,7 +1273,7 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None): finish_reason=None, stop_reason=None, ) - req_output = RequestOutput( + return RequestOutput( request_id="req", prompt="hi", prompt_token_ids=[7, 8], @@ -1262,13 +1281,29 @@ def _make_simple_context_with_output(text, token_ids, response_parser=None): outputs=[completion], finished=False, num_cached_tokens=0, + metrics=metrics, ) + + +def _make_simple_context_with_output( + text, + token_ids, + response_parser=None, + metrics: RequestStateStats | None = None, +): + """Create a SimpleContext with a RequestOutput containing the given text.""" + ctx = SimpleContext(response_parser=response_parser) + req_output = _make_request_output(text, token_ids, metrics) ctx.append_output(req_output) + ctx.request_metrics = metrics return ctx -def _make_serving_instance_with_reasoning(): - """Create an OpenAIServingResponses with a mocked reasoning parser.""" +def _make_serving_instance( + *, + reasoning_parser: str = "", + enable_per_request_metrics: bool = False, +) -> OpenAIServingResponses: engine_client = MagicMock() model_config = MagicMock() model_config.max_model_len = 100 @@ -1279,18 +1314,112 @@ def _make_serving_instance_with_reasoning(): engine_client.input_processor = MagicMock() engine_client.renderer = MagicMock() - models = MagicMock() - - serving = OpenAIServingResponses( + return OpenAIServingResponses( engine_client=engine_client, - models=models, + models=MagicMock(), online_renderer=MagicMock(), request_logger=None, chat_template=None, chat_template_content_format="auto", - reasoning_parser="qwen3", + reasoning_parser=reasoning_parser, + enable_per_request_metrics=enable_per_request_metrics, ) - return serving + + +async def _empty_context_generator(): + if False: + yield + + +async def _make_full_metrics_response( + enable_per_request_metrics: bool, + request_metrics_cover_all_generation_turns: bool = True, +): + serving = _make_serving_instance( + enable_per_request_metrics=enable_per_request_metrics + ) + request = ResponsesRequest(input="hi", tools=[], stream=False, store=False) + sampling_params = SamplingParams(max_tokens=16) + context = SimpleContext() + context.request_metrics_cover_all_generation_turns = ( + request_metrics_cover_all_generation_turns + ) + + async def generate(*args, **kwargs): + yield _make_request_output("hello", [10, 20], _PER_REQUEST_STATS) + + serving.engine_client.generate.side_effect = generate + result_generator = serving._generate_with_builtin_tools( + request_id=request.request_id, + engine_input=tokens_input([7, 8]), + sampling_params=sampling_params, + context=context, + ) + response = await serving.responses_full_generator( + request=request, + sampling_params=sampling_params, + result_generator=result_generator, + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + assert isinstance(response, ResponsesResponse) + return response + + +@pytest.mark.asyncio +async def test_responses_per_request_metrics_follow_server_flag(): + disabled_response = await _make_full_metrics_response(False) + assert disabled_response.metrics is None + assert "metrics" not in disabled_response.model_dump(mode="json") + + enabled_response = await _make_full_metrics_response(True) + assert enabled_response.metrics is not None + assert enabled_response.metrics.time_to_first_token_ms == pytest.approx(500.0) + + +@pytest.mark.asyncio +async def test_responses_streaming_metrics_only_on_completed_event(): + serving = _make_serving_instance(enable_per_request_metrics=True) + request = ResponsesRequest(input="hi", tools=[], stream=True, store=False) + context = _make_simple_context_with_output( + "hello", [10, 20], metrics=_PER_REQUEST_STATS + ) + + events = [ + event + async for event in serving.responses_stream_generator( + request=request, + sampling_params=SamplingParams(max_tokens=16), + result_generator=_empty_context_generator(), + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + ] + + for event in events[:-1]: + assert "metrics" not in event.response.model_dump(mode="json") + assert isinstance(events[-1], ResponseCompletedEvent) + assert events[-1].response.metrics is not None + assert events[-1].response.metrics.time_to_first_token_ms == pytest.approx(500.0) + + +@pytest.mark.asyncio +async def test_responses_metrics_suppressed_for_multiple_generation_turns(): + response = await _make_full_metrics_response( + True, request_metrics_cover_all_generation_turns=False + ) + + assert response.metrics is None + assert "metrics" not in response.model_dump(mode="json") + + +def _make_serving_instance_with_reasoning(): + """Create an OpenAIServingResponses with a mocked reasoning parser.""" + return _make_serving_instance(reasoning_parser="qwen3") def _identity_increment(event): @@ -1342,7 +1471,6 @@ class TestStreamingReasoningToContentTransition: chunk), the trailing reasoning text must be emitted as a ResponseReasoningTextDeltaEvent and included in the ResponseReasoningTextDoneEvent text.""" - monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) serving = _make_serving_instance_with_reasoning() @@ -1412,7 +1540,6 @@ class TestStreamingReasoningToContentTransition: ): """When the transition from reasoning to content is clean (no mixed delta), no extra reasoning delta event should be emitted.""" - monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) serving = _make_serving_instance_with_reasoning() @@ -1474,7 +1601,6 @@ class TestStreamingReasoningToContentTransition: """When the stream has only reasoning deltas and no content, the reasoning done event should be emitted at finalization with the full accumulated text, and no text delta events should appear.""" - monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) serving = _make_serving_instance_with_reasoning() diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 6d3fc2f447..18b01b85a8 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -83,8 +83,7 @@ def before_generate_case(context: schemathesis.HookContext, strategy): assert op is not None def no_invalid_types(case: schemathesis.Case): - """ - Skips tool_calls with `"type": "custom"` which schemathesis incorrectly + """Skips tool_calls with `"type": "custom"` which schemathesis incorrectly generates instead of the valid `"type": "function"`. Example test case that is skipped: diff --git a/tests/entrypoints/openai/test_return_token_ids.py b/tests/entrypoints/openai/test_return_token_ids.py index 8537082e3f..9fc8c102d4 100644 --- a/tests/entrypoints/openai/test_return_token_ids.py +++ b/tests/entrypoints/openai/test_return_token_ids.py @@ -189,8 +189,7 @@ async def test_chat_completion_with_tool_use(server): @pytest.mark.asyncio async def test_comparison_with_prompt_logprobs_and_logprobs(server): - """ - Test that token_ids align with prompt_logprobs and + """Test that token_ids align with prompt_logprobs and logprobs when return_tokens_as_token_ids is enabled. """ async with server.get_async_client() as client: diff --git a/tests/entrypoints/openai/test_stop_token_ids.py b/tests/entrypoints/openai/test_stop_token_ids.py index 5e9d79f686..d6ac2cf190 100644 --- a/tests/entrypoints/openai/test_stop_token_ids.py +++ b/tests/entrypoints/openai/test_stop_token_ids.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for stop_token_ids propagation from default_sampling_params +"""Unit tests for stop_token_ids propagation from default_sampling_params to SamplingParams in ChatCompletionRequest and CompletionRequest. Regression test for https://github.com/vllm-project/vllm/issues/22519 diff --git a/tests/entrypoints/openai/test_tool_choice_content_none.py b/tests/entrypoints/openai/test_tool_choice_content_none.py index 7db815c45f..a0a8f38e9f 100644 --- a/tests/entrypoints/openai/test_tool_choice_content_none.py +++ b/tests/entrypoints/openai/test_tool_choice_content_none.py @@ -31,9 +31,6 @@ class _DummyDelegatingParser(DelegatingParser): def is_reasoning_end(self, input_ids: list[int]) -> bool: return False - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - return input_ids - def extract_reasoning(self, model_output: str, request): return None, model_output diff --git a/tests/entrypoints/openai/utils.py b/tests/entrypoints/openai/utils.py index 4cc4800bf8..02a4c51f94 100644 --- a/tests/entrypoints/openai/utils.py +++ b/tests/entrypoints/openai/utils.py @@ -16,8 +16,7 @@ from vllm.entrypoints.serve.engine.protocol import UsageInfo async def accumulate_streaming_response( stream_generator: AsyncGenerator[str, None], ) -> ChatCompletionResponse: - """ - Accumulate streaming SSE chunks into a complete ChatCompletionResponse. + """Accumulate streaming SSE chunks into a complete ChatCompletionResponse. This helper parses the SSE format and builds up the complete response by combining all the delta chunks. diff --git a/tests/entrypoints/pooling/basic/test_tiling_engine.py b/tests/entrypoints/pooling/basic/test_tiling_engine.py index 88c9b434b3..c620816dc5 100644 --- a/tests/entrypoints/pooling/basic/test_tiling_engine.py +++ b/tests/entrypoints/pooling/basic/test_tiling_engine.py @@ -30,8 +30,7 @@ def llm(vllm_runner): @pytest.mark.skip_global_cleanup def test_tiling_engine_basic(llm): - """ - Basic test with a small number of prompts (less than max_num_seqs). + """Basic test with a small number of prompts (less than max_num_seqs). No tiling should be triggered, but the engine still processes correctly. """ prompts = ["Hello", "World"] @@ -41,8 +40,7 @@ def test_tiling_engine_basic(llm): @pytest.mark.skip_global_cleanup def test_tiling_engine_many_requests(llm): - """ - Test with a large number of prompts that exceeds max_num_seqs. + """Test with a large number of prompts that exceeds max_num_seqs. This verifies that _run_tiling_engine correctly chunks requests, processes all of them, and returns outputs in the correct order. """ @@ -54,8 +52,7 @@ def test_tiling_engine_many_requests(llm): @pytest.mark.skip_global_cleanup def test_tiling_engine_with_pooling_params(llm): - """ - Test the tiling engine when different PoolingParams are provided. + """Test the tiling engine when different PoolingParams are provided. The engine must handle a list of params that matches the number of prompts. """ num_prompts = 10 @@ -77,8 +74,7 @@ def test_tiling_engine_with_pooling_params(llm): @pytest.mark.skip_global_cleanup def test_tiling_engine_abort_on_exception(llm): - """ - Test that abort_request IS called with the correct arguments when an + """Test that abort_request IS called with the correct arguments when an exception occurs inside the engine's step() loop. """ prompts = ["Prompt 0", "Prompt 1", "Prompt 2"] diff --git a/tests/entrypoints/pooling/embed/test_online_dimensions.py b/tests/entrypoints/pooling/embed/test_online_dimensions.py index 7abec99ec3..67c6d7b134 100644 --- a/tests/entrypoints/pooling/embed/test_online_dimensions.py +++ b/tests/entrypoints/pooling/embed/test_online_dimensions.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Run `pytest tests/entrypoints/openai/test_embedding_dimensions.py`. -""" +"""Run `pytest tests/entrypoints/openai/test_embedding_dimensions.py`.""" import openai import pytest diff --git a/tests/entrypoints/pooling/embed/test_online_long_text.py b/tests/entrypoints/pooling/embed/test_online_long_text.py index 60cacb17f8..ab3633cd3d 100644 --- a/tests/entrypoints/pooling/embed/test_online_long_text.py +++ b/tests/entrypoints/pooling/embed/test_online_long_text.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test cases for long text embedding with automatic chunking mechanism. +"""Test cases for long text embedding with automatic chunking mechanism. This test suite validates vLLM's automatic chunking functionality for handling text inputs that exceed the model's maximum token length, specifically targeting @@ -242,7 +241,6 @@ async def test_long_text_embedding_1500_chars( ): """Test embedding processing for ~1500 character long text (~1028 tokens, exceeding 512 token limit).""" - # Verify text length # Verify text has sufficient word count (approximately 1500 words) word_count = len(LONG_TEXT_1500_WORDS.split()) @@ -291,7 +289,6 @@ async def test_long_text_embedding_2500_chars( ): """Test embedding processing for ~2500 character long text (~2048 tokens, requiring multiple chunks).""" - # Verify text length # Verify text has sufficient word count (approximately 2500 words) word_count = len(LONG_TEXT_2500_WORDS.split()) @@ -339,7 +336,6 @@ async def test_batch_long_text_embedding( client_with_chunked_processing: openai.AsyncOpenAI, model_name: str ): """Test batch long text embedding processing.""" - input_texts = [ LONG_TEXT_1500_WORDS, LONG_TEXT_2500_WORDS, @@ -385,7 +381,6 @@ async def test_chunked_vs_normal_consistency( ): """Test consistency between chunked and normal processing (using short text).""" - # Use a short text within the 512 token limit short_text = ( "Artificial intelligence technology is changing our world, " @@ -424,7 +419,6 @@ async def test_chunked_processing_response_format( client_with_chunked_processing: openai.AsyncOpenAI, model_name: str ): """Test response format and structure during chunked processing.""" - # Test with long text to trigger chunking embedding_response = await client_with_chunked_processing.embeddings.create( model=model_name, diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index a30e487450..d7e0298ac3 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -555,7 +555,7 @@ async def test_derender_chat_oversized_token_ids_rejected(client): @pytest.mark.asyncio async def test_derender_chat_too_many_choices_rejected(client): - """choices count exceeding VLLM_MAX_N_SEQUENCES returns 400.""" + """Choices count exceeding VLLM_MAX_N_SEQUENCES returns 400.""" # Default VLLM_MAX_N_SEQUENCES is 16384; use a larger count. oversized_choices = [ {"index": i, "token_ids": [42], "finish_reason": "stop"} for i in range(20_000) diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index b783f40ae3..df10b31d28 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -41,7 +41,6 @@ def _make_stream_chunk( usage: dict | None = None, ) -> GenerateStreamResponse: """Build a GenerateStreamResponse SSE chunk.""" - return GenerateStreamResponse( request_id=request_id, choices=[ @@ -61,7 +60,6 @@ def _make_usage_chunk( request_id: str = "test-req", ) -> GenerateStreamResponse: """Build a usage only final SSE chunk (empty choices).""" - return GenerateStreamResponse( request_id=request_id, choices=[], @@ -531,7 +529,6 @@ async def client(server): async def _render_chat(client) -> dict: """Render a minimal chat request and return the GenerateRequest dict.""" - resp = await client.post( "/v1/chat/completions/render", json={ diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index 6aaf119f16..e21343553d 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -337,7 +337,6 @@ async def test_chat_completion_render_multi_turn(client): @pytest.mark.asyncio async def test_chat_completion_render_with_stream_true(client): """Render accepts stream params but still returns JSON (non-streamed).""" - response = await client.post( "/v1/chat/completions/render", json={ diff --git a/tests/entrypoints/scale_out/render/test_render_multimodal.py b/tests/entrypoints/scale_out/render/test_render_multimodal.py index c6a6931948..eb45ce68ea 100644 --- a/tests/entrypoints/scale_out/render/test_render_multimodal.py +++ b/tests/entrypoints/scale_out/render/test_render_multimodal.py @@ -16,7 +16,6 @@ VISION_MODEL_NAME = "Qwen/Qwen3-VL-2B-Instruct" @pytest.fixture(scope="module") def vision_server(): """Vision-capable server used for multimodal /render tests.""" - args = [ "--enforce-eager", "--max-model-len", @@ -48,7 +47,6 @@ async def test_chat_completion_render_with_base64_image_url( local_asset_server, ): """Render a multimodal chat request and verify tokens are returned.""" - image = local_asset_server.get_image_asset("RGBA_comp.png") data_url = encode_image_url(image, format="PNG") @@ -112,7 +110,6 @@ async def test_tokenize_matches_render_for_multimodal_input( local_asset_server, ): """`/tokenize` should match `/v1/chat/completions/render` token output.""" - image = local_asset_server.get_image_asset("RGBA_comp.png") data_url = encode_image_url(image, format="PNG") diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index ad39958702..d9ae424cc8 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -12,6 +12,9 @@ from vllm.config.multimodal import MultiModalConfig from vllm.entrypoints.generate.base.protocol import StreamOptions from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( + placeholder_ranges_from_engine_input, +) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, GenerateResponse, @@ -19,6 +22,7 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens from vllm.exceptions import GenerationError from vllm.logprobs import Logprob +from vllm.multimodal.inputs import PlaceholderRange from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers import renderer_from_config from vllm.renderers.online_renderer import OnlineRenderer @@ -176,6 +180,29 @@ def _parse_sse_chunks(chunks: list[str]) -> list[Any]: return parsed +def test_placeholder_ranges_from_engine_input(): + engine_input: Any = { + "type": "multimodal", + "mm_placeholders": { + "image": [ + PlaceholderRange(offset=1, length=5), + PlaceholderRange(offset=8, length=2), + ] + }, + } + + placeholders = placeholder_ranges_from_engine_input(engine_input) + + assert placeholders is not None + assert [item.model_dump() for item in placeholders["image"]] == [ + {"offset": 1, "length": 5}, + {"offset": 8, "length": 2}, + ] + + text_only: Any = {"type": "token", "prompt_token_ids": [1, 2, 3]} + assert placeholder_ranges_from_engine_input(text_only) is None + + @pytest.mark.asyncio async def test_serve_tokens_skips_mm_cache_for_remote_engine_execution(): engine = _mock_engine() @@ -294,6 +321,336 @@ async def test_stream_basic(): assert data_chunks[2]["choices"][0]["finish_reason"] == "stop" +@pytest.mark.asyncio +async def test_non_stream_returns_final_prompt_token_ids_when_requested(): + engine = _mock_engine() + final_prompt_token_ids = [1, 2, 2, 2, 3] + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", + token_ids=[10], + finish_reason="stop", + finished=True, + prompt_token_ids=final_prompt_token_ids, + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + + assert isinstance(response, GenerateResponse) + assert response.prompt_token_ids == final_prompt_token_ids + + +@pytest.mark.asyncio +async def test_stream_returns_prompt_token_ids_only_once_when_requested(): + engine = _mock_engine() + final_prompt_token_ids = [1, 2, 2, 2, 3] + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[10], prompt_token_ids=final_prompt_token_ids + ) + yield _make_request_output( + "req-1", + token_ids=[20], + finish_reason="stop", + finished=True, + prompt_token_ids=final_prompt_token_ids, + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=2), + model=MODEL_NAME, + stream=True, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] + + assert data_chunks[0]["prompt_token_ids"] == final_prompt_token_ids + assert "prompt_token_ids" not in data_chunks[1] + + +MM_ENGINE_INPUT: Any = { + "type": "multimodal", + "prompt_token_ids": [1, 2, 2, 2, 3], + "mm_placeholders": {"image": [PlaceholderRange(offset=1, length=3)]}, +} +MM_PLACEHOLDERS = {"image": [{"offset": 1, "length": 3}]} + + +def _build_mm_serving_tokens(engine: AsyncLLM) -> ServingTokens: + serving = _build_serving_tokens(engine) + + async def _fake_preprocess(*args, **kwargs): + return [MM_ENGINE_INPUT] + + serving.online_renderer.preprocess_completion = AsyncMock( + side_effect=_fake_preprocess + ) + return serving + + +@pytest.mark.asyncio +@pytest.mark.parametrize("return_token_ids", [False, True]) +async def test_non_stream_prompt_metadata_requires_return_token_ids( + return_token_ids: bool, +): + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[10], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_mm_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + return_token_ids=return_token_ids, + ) + + response = await serving.serve_tokens(request) + + assert isinstance(response, GenerateResponse) + if return_token_ids: + assert response.prompt_token_ids == [1, 2, 3] + assert response.model_dump()["mm_placeholders"] == MM_PLACEHOLDERS + else: + assert response.prompt_token_ids is None + assert response.mm_placeholders is None + assert request._response_mm_placeholders is None + + +@pytest.mark.asyncio +async def test_non_stream_text_only_prompt_has_no_mm_placeholders(): + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[10], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + + assert isinstance(response, GenerateResponse) + assert response.prompt_token_ids == [1, 2, 3] + assert response.mm_placeholders is None + + +@pytest.mark.asyncio +async def test_non_stream_encoder_decoder_omits_mm_placeholders(): + engine = _mock_engine() + engine.model_config.is_encoder_decoder = True + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[10], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_mm_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + + assert isinstance(response, GenerateResponse) + assert response.prompt_token_ids == [1, 2, 3] + assert response.mm_placeholders is None + + +@pytest.mark.asyncio +async def test_stream_prompt_metadata_lands_on_first_emitted_chunk(): + """An empty first output is skipped; metadata goes on the first real chunk, + once, and never on the sibling chunk of the same iteration (n=2).""" + engine = _mock_engine() + + def _two_outputs(tokens0: list[int], tokens1: list[int], **kwargs): + res = _make_request_output("req-1", token_ids=tokens0, **kwargs) + res.outputs.append( + CompletionOutput( + index=1, + text="", + token_ids=tokens1, + cumulative_logprob=None, + logprobs=None, + finish_reason=kwargs.get("finish_reason"), + ) + ) + return res + + async def mock_generate(*args, **kwargs): + yield _two_outputs([], []) + yield _two_outputs([10], [11]) + yield _two_outputs([20], [21], finish_reason="stop", finished=True) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_mm_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=2, n=2), + model=MODEL_NAME, + stream=True, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] + + assert [c["choices"][0]["token_ids"] for c in data_chunks] == [ + [10], + [11], + [20], + [21], + ] + assert data_chunks[0]["prompt_token_ids"] == [1, 2, 3] + assert data_chunks[0]["mm_placeholders"] == MM_PLACEHOLDERS + for chunk in data_chunks[1:]: + assert "prompt_token_ids" not in chunk + assert "mm_placeholders" not in chunk + + +@pytest.mark.asyncio +async def test_stream_omits_prompt_metadata_by_default(): + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[10], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_mm_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + stream=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] + + assert data_chunks + for chunk in data_chunks: + assert "prompt_token_ids" not in chunk + assert "mm_placeholders" not in chunk + + +@pytest.mark.asyncio +async def test_stream_text_only_prompt_omits_mm_placeholders_key(): + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[10], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + stream=True, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] + + assert data_chunks[0]["prompt_token_ids"] == [1, 2, 3] + assert "mm_placeholders" not in data_chunks[0] + + +@pytest.mark.asyncio +async def test_stream_zero_token_completion_still_delivers_prompt_metadata(): + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_mm_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + stream=True, + return_token_ids=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + data_chunks = [c for c in parsed if isinstance(c, dict) and c.get("choices")] + + assert len(data_chunks) == 1 + assert data_chunks[0]["choices"][0]["token_ids"] == [] + assert data_chunks[0]["choices"][0]["finish_reason"] == "stop" + assert data_chunks[0]["prompt_token_ids"] == [1, 2, 3] + assert data_chunks[0]["mm_placeholders"] == MM_PLACEHOLDERS + + +@pytest.mark.asyncio +async def test_stream_zero_token_completion_emits_no_chunk_by_default(): + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", token_ids=[], finish_reason="stop", finished=True + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + stream=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + + assert not [c for c in parsed if isinstance(c, dict) and c.get("choices")] + assert parsed[-1] == "[DONE]" + + @pytest.mark.asyncio async def test_stream_error_mid_generation(): """finish_reason='error' mid-stream yields error chunk then [DONE].""" diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py index 764d684445..5cd60a959f 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Roundtrip tests for multimodal serde used by the +"""Roundtrip tests for multimodal serde used by the token_in_token_out generate endpoint. """ diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py index 892d3c8285..1d017bd3cf 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py @@ -154,11 +154,28 @@ async def test_render_to_generate_roundtrip(client, test_image): ) -@pytest.mark.asyncio -async def test_content_parts_generates_tokens(client, test_image): - """content_parts with raw media should produce output tokens.""" - data_url = encode_image_url(test_image, format="PNG") +def _collapse_placeholders( + token_ids: list[int], mm_placeholders: dict[str, list[dict[str, int]]] +) -> list[int]: + """Undo placeholder expansion: each expanded range becomes one token. + ``/generate`` with ``content_parts`` expects unexpanded placeholders and + expands them itself, so the roundtrip must start from a collapsed prompt. + """ + ranges = sorted( + (r["offset"], r["length"]) for rs in mm_placeholders.values() for r in rs + ) + collapsed: list[int] = [] + pos = 0 + for offset, length in ranges: + collapsed.extend(token_ids[pos:offset]) + collapsed.append(token_ids[offset]) + pos = offset + length + collapsed.extend(token_ids[pos:]) + return collapsed + + +async def _render_image_prompt(client, data_url: str) -> dict: render_resp = await client.post( RENDER_ENDPOINT, json={ @@ -175,14 +192,26 @@ async def test_content_parts_generates_tokens(client, test_image): }, ) render_resp.raise_for_status() - token_ids = render_resp.json()["token_ids"] + return render_resp.json() + + +@pytest.mark.asyncio +async def test_content_parts_generates_tokens(client, test_image): + """Unexpanded prompt + content_parts must expand exactly like /render.""" + data_url = encode_image_url(test_image, format="PNG") + render_data = await _render_image_prompt(client, data_url) + expanded = render_data["token_ids"] + mm_placeholders = render_data["features"]["mm_placeholders"] + unexpanded = _collapse_placeholders(expanded, mm_placeholders) + assert len(unexpanded) < len(expanded) gen_resp = await client.post( GEN_ENDPOINT, json={ - "token_ids": token_ids, + "token_ids": unexpanded, "content_parts": [{"type": "image_url", "url": data_url}], "sampling_params": {"max_tokens": 10, "temperature": 0.0}, + "return_token_ids": True, }, ) gen_resp.raise_for_status() @@ -192,6 +221,8 @@ async def test_content_parts_generates_tokens(client, test_image): choice = gen_data["choices"][0] assert "token_ids" in choice assert len(choice["token_ids"]) > 0 + assert gen_data["prompt_token_ids"] == expanded + assert gen_data["mm_placeholders"] == mm_placeholders @pytest.mark.asyncio @@ -200,33 +231,20 @@ async def test_content_parts_streaming(client, test_image): import json as json_mod data_url = encode_image_url(test_image, format="PNG") - - render_resp = await client.post( - RENDER_ENDPOINT, - json={ - "model": MODEL_NAME, - "messages": [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": data_url}}, - {"type": "text", "text": "Describe this."}, - ], - } - ], - }, - ) - render_resp.raise_for_status() - token_ids = render_resp.json()["token_ids"] + render_data = await _render_image_prompt(client, data_url) + expanded = render_data["token_ids"] + mm_placeholders = render_data["features"]["mm_placeholders"] + unexpanded = _collapse_placeholders(expanded, mm_placeholders) async with client.stream( "POST", GEN_ENDPOINT, json={ - "token_ids": token_ids, + "token_ids": unexpanded, "content_parts": [{"type": "image_url", "url": data_url}], "sampling_params": {"max_tokens": 8, "temperature": 0.0}, "stream": True, + "return_token_ids": True, }, ) as resp: resp.raise_for_status() @@ -236,3 +254,9 @@ async def test_content_parts_streaming(client, test_image): chunks.append(json_mod.loads(line[6:])) assert len(chunks) > 0 + prompt_metadata_chunks = [ + chunk for chunk in chunks if chunk.get("prompt_token_ids") is not None + ] + assert len(prompt_metadata_chunks) == 1 + assert prompt_metadata_chunks[0]["prompt_token_ids"] == expanded + assert prompt_metadata_chunks[0]["mm_placeholders"] == mm_placeholders diff --git a/tests/entrypoints/serve/dev/rlhf/conftest.py b/tests/entrypoints/serve/dev/rlhf/conftest.py index 1d2c598b19..c636b51572 100644 --- a/tests/entrypoints/serve/dev/rlhf/conftest.py +++ b/tests/entrypoints/serve/dev/rlhf/conftest.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Shared fixtures and helpers for the RL lifecycle test suite. +"""Shared fixtures and helpers for the RL lifecycle test suite. All test modules under this directory import from here to avoid duplication. @@ -92,6 +91,7 @@ def server( port: HTTP port to bind (caller is responsible for uniqueness). timeout: Seconds to wait for /health before giving up. dummy_weights: If True, use --load-format dummy (fast, no real weights). + """ env = {**os.environ, "VLLM_SERVER_DEV_MODE": "1"} base = _DUMMY_ARGS if dummy_weights else _BASE_ARGS diff --git a/tests/entrypoints/serve/dev/rpc/test_collective_rpc.py b/tests/entrypoints/serve/dev/rpc/test_collective_rpc.py index eb9aa7663c..7cf6f4490c 100644 --- a/tests/entrypoints/serve/dev/rpc/test_collective_rpc.py +++ b/tests/entrypoints/serve/dev/rpc/test_collective_rpc.py @@ -25,7 +25,7 @@ class TestWorkerExtension: ) def return_none(self, *args, **kwargs) -> None: - """Test method that does not return anything""" + """Test method that does not return anything.""" return @@ -48,7 +48,7 @@ def server(): def test_get_model_name(server): - """Test basic response""" + """Test basic response.""" response = requests.post( server.url_for("collective_rpc"), json={"method": "get_model_name"} ) @@ -59,7 +59,7 @@ def test_get_model_name(server): def test_return_none(server): - """Test return none""" + """Test return none.""" response = requests.post( server.url_for("collective_rpc"), json={"method": "return_none"} ) @@ -69,7 +69,7 @@ def test_return_none(server): def test_echo_args_kwargs(server): - """Test args, kwargs, and dict response""" + """Test args, kwargs, and dict response.""" args = ["arg1", "arg2"] kwargs = {"key1": "value1", "key2": "value2"} response = requests.post( diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/serve/dev/test_sleep.py index 260dcd00ba..5ebee50600 100644 --- a/tests/entrypoints/serve/dev/test_sleep.py +++ b/tests/entrypoints/serve/dev/test_sleep.py @@ -87,8 +87,7 @@ def test_sleep_mode(): def _get_sleep_metrics_from_api(response: requests.Response): - """Return (awake, weights_offloaded, discard_all)""" - + """Return (awake, weights_offloaded, discard_all).""" awake, weights_offloaded, discard_all = None, None, None for family in text_string_to_metric_families(response.text): diff --git a/tests/entrypoints/serve/instrumentator/test_basic.py b/tests/entrypoints/serve/instrumentator/test_basic.py index 73a97c4fa2..0711cbee95 100644 --- a/tests/entrypoints/serve/instrumentator/test_basic.py +++ b/tests/entrypoints/serve/instrumentator/test_basic.py @@ -21,7 +21,7 @@ MODEL_NAME = "Qwen/Qwen3-0.6B" @pytest.fixture(scope="module") def server_args(request: pytest.FixtureRequest) -> list[str]: - """Provide extra arguments to the server via indirect parametrization + """Provide extra arguments to the server via indirect parametrization. Usage: diff --git a/tests/entrypoints/serve/instrumentator/test_metrics.py b/tests/entrypoints/serve/instrumentator/test_metrics.py index 297fe233b5..ac4576fbb9 100644 --- a/tests/entrypoints/serve/instrumentator/test_metrics.py +++ b/tests/entrypoints/serve/instrumentator/test_metrics.py @@ -403,8 +403,7 @@ async def _poll_until( def _get_running_metrics_from_api(server: RemoteOpenAIServer): - """Return (running_count, waiting_count, kv_cache_usage)""" - + """Return (running_count, waiting_count, kv_cache_usage).""" response = requests.get(server.url_for("metrics")) assert response.status_code == HTTPStatus.OK diff --git a/tests/entrypoints/serve/lora/test_lora_adapters.py b/tests/entrypoints/serve/lora/test_lora_adapters.py index a22f0b3899..8a860dc625 100644 --- a/tests/entrypoints/serve/lora/test_lora_adapters.py +++ b/tests/entrypoints/serve/lora/test_lora_adapters.py @@ -26,7 +26,7 @@ BADREQUEST_CASES = [ ( "test_modules_to_save", {"modules_to_save": ["lm_head"]}, - "only supports modules_to_save being None", + "Unsupported modules_to_save", ), ] diff --git a/tests/entrypoints/serve/middleware/test_optional_middleware.py b/tests/entrypoints/serve/middleware/test_optional_middleware.py index 865bd445c8..160f1e248d 100644 --- a/tests/entrypoints/serve/middleware/test_optional_middleware.py +++ b/tests/entrypoints/serve/middleware/test_optional_middleware.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for middleware that's off by default and can be toggled through +"""Tests for middleware that's off by default and can be toggled through server arguments, mainly --api-key and --enable-request-id-headers. """ diff --git a/tests/entrypoints/serve/tokenize/test_tokenization_vlm.py b/tests/entrypoints/serve/tokenize/test_tokenization_vlm.py index 6b226c6999..c4317973cb 100644 --- a/tests/entrypoints/serve/tokenize/test_tokenization_vlm.py +++ b/tests/entrypoints/serve/tokenize/test_tokenization_vlm.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Regression test: ``/tokenize`` must expand image placeholders for VLM models. +"""Regression test: ``/tokenize`` must expand image placeholders for VLM models. Fixed by PR #34560 ("Move InputPreprocessor into Renderer (2/2)"). Before that change, ``/tokenize`` returned ~26 tokens for a message with an diff --git a/tests/entrypoints/serve/tokenize/test_tokenize_then_chat_vlm.py b/tests/entrypoints/serve/tokenize/test_tokenize_then_chat_vlm.py index 50083e3455..1d114bc8b8 100644 --- a/tests/entrypoints/serve/tokenize/test_tokenize_then_chat_vlm.py +++ b/tests/entrypoints/serve/tokenize/test_tokenize_then_chat_vlm.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Regression test: calling ``/tokenize`` with multimodal data followed by +"""Regression test: calling ``/tokenize`` with multimodal data followed by ``/v1/chat/completions`` with the same data must not cause an error. Ensures that the ``/tokenize`` endpoint does not pollute internal caches @@ -53,7 +52,6 @@ async def test_tokenize_then_chat_completion_with_image( ): """Tokenize a multimodal message, then send the same message to chat completions. The chat completion must succeed (not 500).""" - image_url = local_asset_server.url_for("stop_sign.jpg") messages = [ { diff --git a/tests/entrypoints/speech_to_text/conftest.py b/tests/entrypoints/speech_to_text/conftest.py index 94505bd054..bf65a72ab5 100644 --- a/tests/entrypoints/speech_to_text/conftest.py +++ b/tests/entrypoints/speech_to_text/conftest.py @@ -11,6 +11,7 @@ def add_attention_backend(server_args, attention_config): Args: server_args: List of server arguments to extend in-place. attention_config: Dict with 'backend' key, or None. + """ if attention_config and "backend" in attention_config: server_args.extend(["--attention-backend", attention_config["backend"]]) diff --git a/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py index 713fa48eee..855dafaaba 100644 --- a/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py +++ b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Evaluate Transcription API correctness by computing Word Error Rate (WER) +"""Evaluate Transcription API correctness by computing Word Error Rate (WER) on a given ASR dataset. When provided, it will also compare the WER against a baseline. This simulates real work usage of the API and makes sure that the frontend and diff --git a/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py b/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py index cd61507728..5534845326 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py +++ b/tests/entrypoints/speech_to_text/transcription/test_chunk_timestamp_offset.py @@ -18,9 +18,8 @@ _PATCH = "vllm.entrypoints.speech_to_text.base.serving" @pytest.mark.asyncio async def test_chunk_offsets_are_cumulative_not_nominal(): - """ - chunk_start_offsets must be cumulative actual chunk lengths, not the old approach of - 'idx * max_audio_clip_s'. When split_audio places a boundary before the + """chunk_start_offsets must be cumulative actual chunk lengths, not the old approach + of 'idx * max_audio_clip_s'. When split_audio places a boundary before the nominal 30 s mark, the old formula drifts; the fixed formula stays exact. """ # Chunks shorter than exactly 30 s, as split_audio produces when a quiet diff --git a/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py index 51599d359e..d5658e7cf0 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py +++ b/tests/entrypoints/speech_to_text/transcription/test_transcription_validation_whisper.py @@ -238,8 +238,7 @@ async def test_stream_options(winning_call, whisper_client): @pytest.mark.asyncio async def test_sampling_params(mary_had_lamb, whisper_client): - """ - Compare sampling with params and greedy sampling to assert results + """Compare sampling with params and greedy sampling to assert results are different when extreme sampling parameters values are picked. """ transcription = await whisper_client.audio.transcriptions.create( diff --git a/tests/entrypoints/tool_parsers/test_hermes_tool_parser.py b/tests/entrypoints/tool_parsers/test_hermes_tool_parser.py index f03b6e8a38..79b46bea51 100644 --- a/tests/entrypoints/tool_parsers/test_hermes_tool_parser.py +++ b/tests/entrypoints/tool_parsers/test_hermes_tool_parser.py @@ -150,7 +150,6 @@ async def test_non_streaming_tool_call( client: openai.AsyncOpenAI, server_config: ServerConfig ): """Test tool call in non-streaming mode.""" - response = await client.chat.completions.create( model=server_config["model_arg"], messages=MESSAGES, @@ -183,7 +182,6 @@ async def test_streaming_tool_call( client: openai.AsyncOpenAI, server_config: ServerConfig ): """Test tool call in streaming mode.""" - stream = await client.chat.completions.create( model=server_config["model_arg"], messages=MESSAGES, @@ -230,7 +228,6 @@ async def test_non_streaming_product_tool_call( client: openai.AsyncOpenAI, server_config: ServerConfig ): """Test tool call integer and boolean parameters in non-streaming mode.""" - response = await client.chat.completions.create( model=server_config["model_arg"], messages=PRODUCT_MESSAGES, @@ -272,7 +269,6 @@ async def test_streaming_product_tool_call( client: openai.AsyncOpenAI, server_config: ServerConfig ): """Test tool call integer and boolean parameters in streaming mode.""" - stream = await client.chat.completions.create( model=server_config["model_arg"], messages=PRODUCT_MESSAGES, diff --git a/tests/entrypoints/tool_parsers/test_openai_tool_parser.py b/tests/entrypoints/tool_parsers/test_openai_tool_parser.py index d99b66d9ac..de402b435c 100644 --- a/tests/entrypoints/tool_parsers/test_openai_tool_parser.py +++ b/tests/entrypoints/tool_parsers/test_openai_tool_parser.py @@ -127,8 +127,7 @@ FUNC_ARGS_TIME = '{"city": "New York"}' # Utility to extract reasoning and tool calls # ========================================================== def extract_reasoning_and_calls(chunks: list) -> tuple[str, list[str], list[str]]: - """ - Extract accumulated reasoning text and tool call arguments + """Extract accumulated reasoning text and tool call arguments from streaming chunks. """ reasoning: str = "" @@ -165,7 +164,6 @@ def extract_reasoning_and_calls(chunks: list) -> tuple[str, list[str], list[str] @pytest.mark.asyncio async def test_calculator_tool_call_and_argument_accuracy(client: openai.AsyncOpenAI): """Verify calculator tool call is made and arguments are accurate.""" - response = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES_CALC, @@ -205,7 +203,6 @@ async def test_calculator_tool_call_and_argument_accuracy(client: openai.AsyncOp @pytest.mark.asyncio async def test_streaming_tool_call_get_time_with_reasoning(client: openai.AsyncOpenAI): """Verify streamed reasoning and tool call behavior for get_time.""" - stream = await client.chat.completions.create( model=MODEL_NAME, messages=MESSAGES_GET_TIME, @@ -258,8 +255,7 @@ async def test_streaming_multiple_tools(client: openai.AsyncOpenAI): @pytest.mark.asyncio async def test_invalid_tool_call(client: openai.AsyncOpenAI): - """ - Verify that ambiguous instructions that should not trigger a tool + """Verify that ambiguous instructions that should not trigger a tool do not produce any tool calls. """ response = await client.chat.completions.create( @@ -283,8 +279,7 @@ async def test_invalid_tool_call(client: openai.AsyncOpenAI): @pytest.mark.asyncio async def test_tool_call_with_temperature(client: openai.AsyncOpenAI): - """ - Verify model produces valid tool or text output + """Verify model produces valid tool or text output under non-deterministic sampling. """ response = await client.chat.completions.create( diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index 630f641df5..ab57140c2b 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -940,7 +940,6 @@ def test_parse_chat_messages_audio_embeds_with_string( audio_embeds_model_config, ): """Test audio_embeds with base64 string embedding data.""" - import torch # Create a sample audio embedding tensor @@ -983,7 +982,6 @@ async def test_parse_chat_messages_audio_embeds_async( audio_embeds_model_config, ): """Test audio_embeds with async futures.""" - import torch # Create a sample audio embedding tensor @@ -2986,7 +2984,6 @@ def test_postprocess_messages_null_arguments_string(): @pytest.mark.asyncio async def test_resolve_items_runs_modalities_concurrently_and_preserves_order(): """Media fetches overlap while modality and item order are preserved.""" - active_fetches = 0 max_active_fetches = 0 diff --git a/tests/entrypoints/unit_tests/test_context.py b/tests/entrypoints/unit_tests/test_context.py index 1c1f6ed235..a5b1afe8c5 100644 --- a/tests/entrypoints/unit_tests/test_context.py +++ b/tests/entrypoints/unit_tests/test_context.py @@ -71,8 +71,8 @@ async def generate_mock_outputs( class FakeHarmonyParser(HarmonyParser): def __init__(self): # Skip HarmonyParser initialization and script outputs directly. - self.reasoning_parser = None - self.tool_parser = None + self._reasoning_parser = None + self._tool_parser = None self._chunk_results: list[ChunkResult] = [] self._flush_results: list[list[Segment]] = [] self.processed_chunks: list[list[int]] = [] @@ -544,7 +544,6 @@ async def test_streaming_multi_turn_token_counting(): @pytest.mark.asyncio async def test_streaming_message_synchronization(): """Completed messages from append-local and flush segments sync into context.""" - # Create a streaming context with some initial messages initial_messages = [ Message( @@ -683,6 +682,7 @@ def create_simple_context_output( prompt="Test prompt", prompt_token_ids=None, num_cached_tokens=0, + num_cache_creation_tokens=None, logprobs=None, finished=True, ): @@ -708,6 +708,7 @@ def create_simple_context_output( ], finished=finished, num_cached_tokens=num_cached_tokens, + num_cache_creation_tokens=num_cache_creation_tokens, ) @@ -828,6 +829,7 @@ def test_simple_context_token_counting(): token_ids=[10, 11], prompt_token_ids=[1, 2, 3, 4, 5], num_cached_tokens=2, + num_cache_creation_tokens=3, ) ) context.append_output( @@ -836,12 +838,14 @@ def test_simple_context_token_counting(): token_ids=[12], prompt_token_ids=[1, 2, 3, 4, 5], num_cached_tokens=2, + num_cache_creation_tokens=3, ) ) assert context.num_prompt_tokens == 5 assert context.num_output_tokens == 3 # 2 + 1 assert context.num_cached_tokens == 2 + assert context.num_cache_creation_tokens == 3 def test_simple_context_final_output(): diff --git a/tests/evals/gpt_oss/conftest.py b/tests/evals/gpt_oss/conftest.py index d35dec4831..28fbb88d52 100644 --- a/tests/evals/gpt_oss/conftest.py +++ b/tests/evals/gpt_oss/conftest.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Pytest configuration for GPT-OSS evaluation tests. -""" +"""Pytest configuration for GPT-OSS evaluation tests.""" from pathlib import Path diff --git a/tests/evals/gpt_oss/test_gpqa_correctness.py b/tests/evals/gpt_oss/test_gpqa_correctness.py index 63188ec407..99d799698a 100644 --- a/tests/evals/gpt_oss/test_gpqa_correctness.py +++ b/tests/evals/gpt_oss/test_gpqa_correctness.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GPQA evaluation using vLLM server and GPT-OSS evaluation package. +"""GPQA evaluation using vLLM server and GPT-OSS evaluation package. Usage: pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py \ @@ -48,7 +47,6 @@ def ensure_tiktoken_files(): def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> float: """Run GPQA evaluation using the gpt-oss evaluation package.""" - # Build the command to run the evaluation cmd = [ sys.executable, diff --git a/tests/evals/gsm8k/gsm8k_eval.py b/tests/evals/gsm8k/gsm8k_eval.py index 83e54afd9e..7e2534b55a 100644 --- a/tests/evals/gsm8k/gsm8k_eval.py +++ b/tests/evals/gsm8k/gsm8k_eval.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Isolated GSM8K evaluation script for vLLM serve endpoint. -""" +"""Isolated GSM8K evaluation script for vLLM serve endpoint.""" import argparse import ast @@ -45,7 +43,7 @@ def download_and_cache_file(url: str, filename: str | None = None) -> str: def load_gsm8k_data() -> tuple[list[dict], list[dict]]: - """Load GSM8K train and test data""" + """Load GSM8K train and test data.""" train_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/train.jsonl" test_url = f"{VLLM_S3_BUCKET_URL}/ci-datasets/gsm8k/test.jsonl" @@ -91,6 +89,7 @@ async def call_vllm_api( Returns: Tuple of (response_text, completion_tokens) + """ data = { "prompt": prompt, @@ -221,8 +220,7 @@ def evaluate_gsm8k( gen_prefix: str = "", max_concurrency: int | None = None, ) -> dict[str, float | int]: - """ - Evaluate GSM8K accuracy using vLLM serve endpoint. + """Evaluate GSM8K accuracy using vLLM serve endpoint. Returns dict with accuracy, invalid_rate, latency, etc. """ diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index 5a237cf947..477a69a16c 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GSM8K evaluation using vLLM server and isolated GSM8K script. +"""GSM8K evaluation using vLLM server and isolated GSM8K script. Replacement for lm-eval-harness with better performance and control. Usage: diff --git a/tests/evals/gsm8k/test_gsm8k_offloading.py b/tests/evals/gsm8k/test_gsm8k_offloading.py index c140a41e26..bcc001e2bd 100644 --- a/tests/evals/gsm8k/test_gsm8k_offloading.py +++ b/tests/evals/gsm8k/test_gsm8k_offloading.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GSM8K correctness test for CPU KV offloading connectors. +"""GSM8K correctness test for CPU KV offloading connectors. Regression guard for stride computation bugs in the offloading worker (e.g. https://github.com/vllm-project/vllm/pull/46888) and silent KV diff --git a/tests/evals/mrcr/test_mrcr_correctness.py b/tests/evals/mrcr/test_mrcr_correctness.py index 3adfd4dc87..f381a592dd 100644 --- a/tests/evals/mrcr/test_mrcr_correctness.py +++ b/tests/evals/mrcr/test_mrcr_correctness.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -MRCR long-context accuracy test. +"""MRCR long-context accuracy test. Usage: pytest -s -v tests/evals/mrcr/test_mrcr_correctness.py \ diff --git a/tests/fusion/test_quant_activation_contract.py b/tests/fusion/test_quant_activation_contract.py index ab12393692..15b9835d6c 100644 --- a/tests/fusion/test_quant_activation_contract.py +++ b/tests/fusion/test_quant_activation_contract.py @@ -24,6 +24,11 @@ from vllm.model_executor.kernels.linear.nvfp4.flashinfer import ( FlashInferCutlassNvFp4LinearKernel, FlashInferTrtllmNvFp4LinearKernel, ) +from vllm.model_executor.kernels.linear.scaled_mm.aiter import ( + AiterHipbMMPerTokenFp8ScaledMMLinearKernel, + AiterPerTokenFp8ScaledMMLinearKernel, + AiterPreshuffledPerTokenFp8ScaledMMLinearKernel, +) from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( CutlassFP8ScaledMMLinearKernel, ) @@ -56,6 +61,9 @@ SUPPORTING = { FlashInferFP8ScaledMMLinearKernel, FlashInferCutlassNvFp4LinearKernel, PerTensorTorchFP8ScaledMMLinearKernel, + AiterHipbMMPerTokenFp8ScaledMMLinearKernel, + AiterPreshuffledPerTokenFp8ScaledMMLinearKernel, + AiterPerTokenFp8ScaledMMLinearKernel, FlashInferCutedslMxfp8LinearKernel, FlashInferCutlassMxfp8LinearKernel, } diff --git a/tests/ir/ir_test_utils.py b/tests/ir/ir_test_utils.py index a82206b6f7..5cd05b9bcf 100644 --- a/tests/ir/ir_test_utils.py +++ b/tests/ir/ir_test_utils.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Shared test utilities for vLLM IR op correctness tests. -""" +"""Shared test utilities for vLLM IR op correctness tests.""" import torch diff --git a/tests/ir/test_op.py b/tests/ir/test_op.py index ca8185b2d3..5033b0ec10 100644 --- a/tests/ir/test_op.py +++ b/tests/ir/test_op.py @@ -34,7 +34,7 @@ def custom_add_op(fake_vllm_ir): @_custom_add.register_impl("impl_b", inplace=True) def impl_b(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - """Computes x+y+20""" + """Computes x+y+20.""" x.add_(y) x.add_(20) return x @@ -101,7 +101,7 @@ def test_no_kw_only_args(fake_vllm_ir): class TestIrOpCustomAdd: # Registration invariants def test_decorated_object(self, custom_add_op): - """Make sure that referring directly to an op is correct""" + """Make sure that referring directly to an op is correct.""" _custom_add = custom_add_op assert isinstance(_custom_add, IrOp) assert "_custom_add" in IrOp.registry diff --git a/tests/kernels/attention/test_flashinfer.py b/tests/kernels/attention/test_flashinfer.py index 109f59bc85..c6cba8d530 100644 --- a/tests/kernels/attention/test_flashinfer.py +++ b/tests/kernels/attention/test_flashinfer.py @@ -96,6 +96,7 @@ def _make_paged_kv_metadata( kv_indices – CUDA int32, shape [total_blocks] kv_last_page_lens – CPU int32, shape [num_seqs] block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq] + """ num_seqs = len(kv_lens) max_blocks = (max(kv_lens) + block_size - 1) // block_size diff --git a/tests/kernels/attention/test_flashinfer_mla_decode.py b/tests/kernels/attention/test_flashinfer_mla_decode.py index a49ae3ae1a..0d37919643 100644 --- a/tests/kernels/attention/test_flashinfer_mla_decode.py +++ b/tests/kernels/attention/test_flashinfer_mla_decode.py @@ -286,7 +286,7 @@ def test_flashinfer_sm90_fp8_mla_decode_without_rope(): @requires_sm10x def test_flashinfer_mla_decode_workspace_supports_autotune(): - """vLLM's FlashInfer MLA decode workspace must be int8 for autotuning. + """VLLM's FlashInfer MLA decode workspace must be int8 for autotuning. Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts diff --git a/tests/kernels/attention/test_lightning_attn.py b/tests/kernels/attention/test_lightning_attn.py index 61e1316680..d4b1a359d2 100644 --- a/tests/kernels/attention/test_lightning_attn.py +++ b/tests/kernels/attention/test_lightning_attn.py @@ -23,7 +23,7 @@ DTYPES = [torch.float32] def reference_lightning_attention(q, k, v, ed, block_size, kv_history): - """Reference implementation of lightning attention core algorithm + """Reference implementation of lightning attention core algorithm. The difference from the main implementation is that this processes each step sequentially, instead of using parallelized triton kernels @@ -72,7 +72,7 @@ def reference_lightning_attention(q, k, v, ed, block_size, kv_history): def reference_linear_decode(q, k, v, kv_caches, slope_rate, slot_idx): - """Reference implementation: linear attention decode function""" + """Reference implementation: linear attention decode function.""" B, H, _, D = q.shape output = torch.zeros(B, H * D, dtype=q.dtype, device=q.device) diff --git a/tests/kernels/attention/test_mha_attn.py b/tests/kernels/attention/test_mha_attn.py index b6578fa2f9..8fa4963a6a 100644 --- a/tests/kernels/attention/test_mha_attn.py +++ b/tests/kernels/attention/test_mha_attn.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test: +"""Test: * Tests for MMEncoderAttention layer """ @@ -49,9 +48,7 @@ if current_platform.is_rocm(): @pytest.mark.parametrize("device", devices) def test_mha_attn_platform(default_vllm_config, device: str): - """ - Test the attention selector between different platform and device. - """ + """Test the attention selector between different platform and device.""" torch.set_default_dtype(torch.float16) if device == "cpu": @@ -113,8 +110,7 @@ def ref_attention( value: torch.Tensor, scale: float, ) -> torch.Tensor: - """ - Native implementation of scaled dot product attention without mask: + """Native implementation of scaled dot product attention without mask: - query, key, value: [batch_size, seq_len, num_heads, head_size] - attn_mask: [batch_size, seq_len, seq_len] """ diff --git a/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py b/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py index 659211521a..0944be8937 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_op_registration.py @@ -35,7 +35,7 @@ def _require_aiter(): @torch.inference_mode() def test_mla_decode_fwd_op_schema() -> None: - """opcheck validates registration, schema, fake-tensor, and ``o`` aliasing. + """Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing. A single opcheck call covers that the op is registered/callable, that its fake implementation matches the real op (torch.compile tracing), and that diff --git a/tests/kernels/attention/test_triton_unified_attention_diffkv.py b/tests/kernels/attention/test_triton_unified_attention_diffkv.py index 513811a09c..04b7755853 100644 --- a/tests/kernels/attention/test_triton_unified_attention_diffkv.py +++ b/tests/kernels/attention/test_triton_unified_attention_diffkv.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for the Triton DiffKV unified-attention kernel. -""" +"""Unit tests for the Triton DiffKV unified-attention kernel.""" import pytest import torch diff --git a/tests/kernels/attention/test_trtllm_kvfp8_dequant.py b/tests/kernels/attention/test_trtllm_kvfp8_dequant.py index bd267c43be..8368869b65 100644 --- a/tests/kernels/attention/test_trtllm_kvfp8_dequant.py +++ b/tests/kernels/attention/test_trtllm_kvfp8_dequant.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant. +"""Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant. Tests both contiguous and non-contiguous (cross-layer unified) KV cache layouts against a pure-PyTorch reference implementation. @@ -43,8 +42,7 @@ def make_random_kv_cache( layout=KVCacheLayout.LBHNC, num_layers=None, ): - """ - Create a random fp8 KV cache mimicking the production allocation. + """Create a random fp8 KV cache mimicking the production allocation. Physical layout: [L, B, H, N, 2 * hs] permuted by ``layout.stride_order`` Returned view: (num_blocks, 2, num_kv_heads, block_size, head_size) @@ -382,8 +380,7 @@ def test_large_block_size(): @torch.inference_mode() def test_cross_layer_many_layers(): - """ - Non-contiguous with 36 layers -- matches real gpt-oss-120b. + """Non-contiguous with 36 layers -- matches real gpt-oss-120b. Cross-layer (BLHNC) strides are far from contiguous (factor of 36 in the gaps). """ from vllm.v1.attention.backends.flashinfer import ( diff --git a/tests/kernels/attention/test_xpu_mla_sparse.py b/tests/kernels/attention/test_xpu_mla_sparse.py index 965c6f735f..47139fc465 100644 --- a/tests/kernels/attention/test_xpu_mla_sparse.py +++ b/tests/kernels/attention/test_xpu_mla_sparse.py @@ -30,12 +30,12 @@ def reference_mla_sparse_prefill( topk_length: torch.Tensor | None = None, attn_sink: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Returns: + """Returns: - o: [s_q, h_q, dv] - o_fp32: [s_q, h_q, dv] - max_logits: [s_q, h_q] - lse: [s_q, h_q] + """ s_q, h_q, d_qk = q.shape s_kv, _, _ = kv.shape diff --git a/tests/kernels/core/test_apply_rotary_emb.py b/tests/kernels/core/test_apply_rotary_emb.py index 5c765bbb4a..fd92de16bb 100644 --- a/tests/kernels/core/test_apply_rotary_emb.py +++ b/tests/kernels/core/test_apply_rotary_emb.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for ApplyRotaryEmb CustomOp dispatch behavior. +"""Tests for ApplyRotaryEmb CustomOp dispatch behavior. This test ensures that RotaryEmbedding classes correctly call the appropriate ApplyRotaryEmb methods based on the calling context: @@ -179,8 +178,7 @@ def test_rotary_embedding_dispatch( test_case: RotaryEmbeddingTestCase, device: str, ): - """ - Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method. + """Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method. - forward_native methods should call ApplyRotaryEmb.forward_native() - forward_cuda/forward methods should call ApplyRotaryEmb.forward() diff --git a/tests/kernels/core/test_minimax_reduce_rms.py b/tests/kernels/core/test_minimax_reduce_rms.py index cd8efc14a3..b92a08a317 100644 --- a/tests/kernels/core/test_minimax_reduce_rms.py +++ b/tests/kernels/core/test_minimax_reduce_rms.py @@ -36,7 +36,6 @@ def _worker_forward_qk( eps, ): """Per-rank worker: compare NCCL allreduce path vs Lamport fused kernel.""" - if not hasattr(torch.ops._C, "minimax_allreduce_rms_qk"): cleanup_dist_env_and_memory() return diff --git a/tests/kernels/core/test_mrope.py b/tests/kernels/core/test_mrope.py index 81e0a916a3..0e355d7cd3 100644 --- a/tests/kernels/core/test_mrope.py +++ b/tests/kernels/core/test_mrope.py @@ -229,7 +229,7 @@ def test_mrope_torch_compile_tracing( # Create a wrapper that makes the in-place function appear functional def functional_forward_cuda(pos, q, k): - """Wrapper that converts in-place operation to functional style + """Wrapper that converts in-place operation to functional style. CUDA Graph does not support in-place operations. This wrapper creates working copies of the diff --git a/tests/kernels/core/test_opcheck.py b/tests/kernels/core/test_opcheck.py index 40ced08b93..45dd01d44e 100644 --- a/tests/kernels/core/test_opcheck.py +++ b/tests/kernels/core/test_opcheck.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for miscellaneous utilities -""" +"""Tests for miscellaneous utilities.""" import torch diff --git a/tests/kernels/core/test_rotary_embedding.py b/tests/kernels/core/test_rotary_embedding.py index 1cbb5dbd18..353aefa11a 100644 --- a/tests/kernels/core/test_rotary_embedding.py +++ b/tests/kernels/core/test_rotary_embedding.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for miscellaneous utilities -""" +"""Tests for miscellaneous utilities.""" import pytest import torch diff --git a/tests/kernels/core/test_rotary_embedding_mla_cache_fused.py b/tests/kernels/core/test_rotary_embedding_mla_cache_fused.py index 1ac17335b6..3f192ba2e7 100644 --- a/tests/kernels/core/test_rotary_embedding_mla_cache_fused.py +++ b/tests/kernels/core/test_rotary_embedding_mla_cache_fused.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for fused MLA KV-cache write and RoPE fused kernel -""" +"""Tests for fused MLA KV-cache write and RoPE fused kernel.""" import random diff --git a/tests/kernels/helion/test_config_manager.py b/tests/kernels/helion/test_config_manager.py index f8e5eae6f1..63f8130849 100644 --- a/tests/kernels/helion/test_config_manager.py +++ b/tests/kernels/helion/test_config_manager.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for Helion ConfigManager and ConfigSet. +"""Unit tests for Helion ConfigManager and ConfigSet. Tests the simplified configuration management system for Helion custom kernels. """ diff --git a/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py b/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py index 6f5a40fad8..14a4356e14 100644 --- a/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py +++ b/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel +"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel. Run `pytest tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py`. """ diff --git a/tests/kernels/helion/test_fused_qk_norm_rope.py b/tests/kernels/helion/test_fused_qk_norm_rope.py index 4ced522d91..a973533169 100644 --- a/tests/kernels/helion/test_fused_qk_norm_rope.py +++ b/tests/kernels/helion/test_fused_qk_norm_rope.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the fused_qk_norm_rope helion kernel +"""Tests for the fused_qk_norm_rope helion kernel. Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`. """ diff --git a/tests/kernels/helion/test_helion_available.py b/tests/kernels/helion/test_helion_available.py index 560a671f14..a4de68bd67 100644 --- a/tests/kernels/helion/test_helion_available.py +++ b/tests/kernels/helion/test_helion_available.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for Helion kernel availability and basic functionality. +"""Tests for Helion kernel availability and basic functionality. This module demonstrates the pattern for testing optional Helion kernels. Tests in this directory will be skipped if Helion is not installed. diff --git a/tests/kernels/helion/test_per_token_group_fp8_quant.py b/tests/kernels/helion/test_per_token_group_fp8_quant.py index d9d96f9287..d9dd8a8817 100644 --- a/tests/kernels/helion/test_per_token_group_fp8_quant.py +++ b/tests/kernels/helion/test_per_token_group_fp8_quant.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the per_token_group_fp8_quant helion kernel +"""Tests for the per_token_group_fp8_quant helion kernel. Run `pytest tests/kernels/helion/test_per_token_group_fp8_quant.py`. """ diff --git a/tests/kernels/helion/test_register.py b/tests/kernels/helion/test_register.py index 9876135056..dd4274bfaf 100644 --- a/tests/kernels/helion/test_register.py +++ b/tests/kernels/helion/test_register.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for Helion kernel registration. +"""Unit tests for Helion kernel registration. Tests ConfiguredHelionKernel, HelionKernelWrapper, and PresetConfigSearch including config picker registration and custom autotuner integration. @@ -1020,7 +1019,6 @@ class TestTorchCompileHOP: ) def test_inductor_backend_compiles_helion_hop(self): """Test torch.compile with inductor backend and Helion fusion enabled.""" - configs: dict[CaseKey, helion.Config] = { CaseKey.default(): helion.Config(block_sizes=[4, 4]) } diff --git a/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py b/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py index 50591ae34a..d0650dfc7a 100644 --- a/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py +++ b/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the rms_norm_dynamic_per_token_quant helion kernel +"""Tests for the rms_norm_dynamic_per_token_quant helion kernel. Run `pytest tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py`. """ diff --git a/tests/kernels/helion/test_rms_norm_per_block_quant.py b/tests/kernels/helion/test_rms_norm_per_block_quant.py index 4ffc759c63..e9c003f4e3 100644 --- a/tests/kernels/helion/test_rms_norm_per_block_quant.py +++ b/tests/kernels/helion/test_rms_norm_per_block_quant.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the rms_norm_per_block_quant helion kernel +"""Tests for the rms_norm_per_block_quant helion kernel. Run `pytest tests/kernels/helion/test_rms_norm_per_block_quant.py`. """ diff --git a/tests/kernels/helion/utils.py b/tests/kernels/helion/utils.py index 38893fc8fe..4a282f275d 100644 --- a/tests/kernels/helion/utils.py +++ b/tests/kernels/helion/utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Helion Kernel test utils""" +"""Helion Kernel test utils.""" import pytest import torch diff --git a/tests/kernels/ir/test_aiter_norm_dispatch.py b/tests/kernels/ir/test_aiter_norm_dispatch.py index 19cf5d3112..f84a7615b1 100644 --- a/tests/kernels/ir/test_aiter_norm_dispatch.py +++ b/tests/kernels/ir/test_aiter_norm_dispatch.py @@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif( def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim): - """q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does.""" + """Q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does.""" q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim qkv = torch.empty(num_tokens, q_size + 2 * kv_size) q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0] diff --git a/tests/kernels/ir/test_ir_ops.py b/tests/kernels/ir/test_ir_ops.py index 730db9833b..560d047c0f 100644 --- a/tests/kernels/ir/test_ir_ops.py +++ b/tests/kernels/ir/test_ir_ops.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Meta-tests for vLLM IR op infrastructure. +"""Meta-tests for vLLM IR op infrastructure. Ensures all registered ops have input generators defined. Per-op correctness tests live alongside their op definitions diff --git a/tests/kernels/mamba/test_causal_conv1d.py b/tests/kernels/mamba/test_causal_conv1d.py index d2a2981edf..d4f5fac59c 100644 --- a/tests/kernels/mamba/test_causal_conv1d.py +++ b/tests/kernels/mamba/test_causal_conv1d.py @@ -36,8 +36,7 @@ def causal_conv1d_ref( final_states_out: torch.Tensor | None = None, activation: str | None = "silu", ): - """ - x: (batch, dim, seqlen) + """x: (batch, dim, seqlen) weight: (dim, width) bias: (dim,) initial_states: (batch, dim, width - 1) @@ -72,8 +71,7 @@ def causal_conv1d_ref( def causal_conv1d_update_ref( x, conv_state, weight, bias=None, activation=None, cache_seqlens=None ): - """ - x: (batch, dim) or (batch, dim, seqlen) + """x: (batch, dim) or (batch, dim, seqlen) conv_state: (batch, dim, state_len), where state_len >= width - 1 weight: (dim, width) bias: (dim,) @@ -136,8 +134,7 @@ def causal_conv1d_opcheck_fn( activation: str | None = "silu", null_block_id: int = NULL_BLOCK_ID, ): - """ - x: (batch, dim, seqlen) + """x: (batch, dim, seqlen) weight: (dim, width) bias: (dim,) seq_idx: (batch, seqlen) diff --git a/tests/kernels/mamba/test_mamba_ssm.py b/tests/kernels/mamba/test_mamba_ssm.py index f8af3db56b..c24f4c0f87 100644 --- a/tests/kernels/mamba/test_mamba_ssm.py +++ b/tests/kernels/mamba/test_mamba_ssm.py @@ -51,8 +51,7 @@ def selective_scan_ref( prev_state=None, final_state_out=None, ): - """ - u: r(B D L) + """u: r(B D L) delta: r(B D L) A: c(D N) or r(D N) B: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L) @@ -135,7 +134,7 @@ def selective_scan_opcheck_fn( cu_chunk_seqlen=None, last_chunk_indices=None, ): - """if return_last_state is True, returns (out, last_state) + """If return_last_state is True, returns (out, last_state) last_state has shape (batch, dim, dstate). """ if u.stride(-1) != 1: diff --git a/tests/kernels/mamba/test_mamba_ssm_configs.py b/tests/kernels/mamba/test_mamba_ssm_configs.py index d35ce95746..7ea1529f2e 100644 --- a/tests/kernels/mamba/test_mamba_ssm_configs.py +++ b/tests/kernels/mamba/test_mamba_ssm_configs.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for the JSON-based config loader added to selective_state_update. +"""Unit tests for the JSON-based config loader added to selective_state_update. Tests cover: - Flat MoE-style filename generation diff --git a/tests/kernels/mamba/test_mamba_ssm_ssd.py b/tests/kernels/mamba/test_mamba_ssm_ssd.py index 1de25780ea..22abeab1ed 100644 --- a/tests/kernels/mamba/test_mamba_ssm_ssd.py +++ b/tests/kernels/mamba/test_mamba_ssm_ssd.py @@ -41,14 +41,15 @@ def segsum(x): def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None): - """ - Arguments: + """Arguments: X: (batch, length, n_heads, d_head) A: (batch, length, n_heads) B: (batch, length, n_heads, d_state) C: (batch, length, n_heads, d_state) + Return: Y: (batch, length, n_heads, d_head) + """ assert X.dtype == A.dtype == B.dtype == C.dtype assert X.shape[1] % block_len == 0 diff --git a/tests/kernels/mamba/utils.py b/tests/kernels/mamba/utils.py index b82c4d82c3..c7cf473427 100644 --- a/tests/kernels/mamba/utils.py +++ b/tests/kernels/mamba/utils.py @@ -9,8 +9,7 @@ from einops import rearrange, repeat def selective_state_update_ref( state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False ): - """ - Argument: + """Argument: state: (batch, dim, dstate) or (batch, nheads, dim, dstate) x: (batch, dim) or (batch, nheads, dim) dt: (batch, dim) or (batch, nheads, dim) @@ -20,8 +19,10 @@ def selective_state_update_ref( D: (dim,) or (nheads, dim) z: (batch, dim) or (batch, nheads, dim) dt_bias: (dim,) or (nheads, dim) + Return: out: (batch, dim) or (batch, nheads, dim) + """ has_heads = state.dim() > 3 if state.dim() == 3: diff --git a/tests/kernels/moe/modular_kernel_tools/common.py b/tests/kernels/moe/modular_kernel_tools/common.py index 8041db68d7..8adfa6588c 100644 --- a/tests/kernels/moe/modular_kernel_tools/common.py +++ b/tests/kernels/moe/modular_kernel_tools/common.py @@ -148,9 +148,7 @@ class Config: return self.E // self.world_size def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]: - """ - make env data for vllm launch. - """ + """Make env data for vllm launch.""" vllm_config = VllmConfig() vllm_config.model_config = SimpleNamespace( enforce_eager=True, @@ -444,9 +442,7 @@ class RankTensors: def make_hidden_states( config: Config, ) -> tuple[torch.Tensor, torch.Tensor | None]: - """ - Return hidden_states - """ + """Return hidden_states.""" m, k, dtype = (config.M, config.K, config.dtype) device = torch.accelerator.current_device_index() a = torch.randn((m, k), device=device, dtype=dtype) / 15.0 diff --git a/tests/kernels/moe/parallel_utils.py b/tests/kernels/moe/parallel_utils.py index f926b034b3..972ebe6512 100644 --- a/tests/kernels/moe/parallel_utils.py +++ b/tests/kernels/moe/parallel_utils.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -DeepEP test utilities -""" +"""DeepEP test utilities.""" import dataclasses import os diff --git a/tests/kernels/moe/test_batched_deepgemm.py b/tests/kernels/moe/test_batched_deepgemm.py index 3fca9190dc..4c4c82d54e 100644 --- a/tests/kernels/moe/test_batched_deepgemm.py +++ b/tests/kernels/moe/test_batched_deepgemm.py @@ -34,7 +34,6 @@ def test_batched_deepgemm_vs_triton( E: int, T: int, K: int, N: int, topk: int, monkeypatch, workspace_init ): """Compare BatchedDeepGemmExperts to BatchedTritonExperts.""" - monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1") device = "cuda" diff --git a/tests/kernels/moe/test_count_expert_num_tokens.py b/tests/kernels/moe/test_count_expert_num_tokens.py index 8014076192..b2b6abc5b4 100644 --- a/tests/kernels/moe/test_count_expert_num_tokens.py +++ b/tests/kernels/moe/test_count_expert_num_tokens.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests compute_expert_num_tokens kernels -""" +"""Tests compute_expert_num_tokens kernels.""" import dataclasses diff --git a/tests/kernels/moe/test_cpu_fused_moe.py b/tests/kernels/moe/test_cpu_fused_moe.py index dccda70c0b..cf83ef6fc0 100644 --- a/tests/kernels/moe/test_cpu_fused_moe.py +++ b/tests/kernels/moe/test_cpu_fused_moe.py @@ -501,7 +501,6 @@ def test_cpu_fused_moe_unaligned_intermediate_size( ): """CPU kernels handle unaligned intermediate sizes by zero-padding the weights before prepacking.""" - set_random_seed(0) batch_size = 64 intermediate_size = UNALIGNED_INTERMEDIATE_DIM diff --git a/tests/kernels/moe/test_cpu_int4_moe.py b/tests/kernels/moe/test_cpu_int4_moe.py index 04931e386f..2d5534ba08 100644 --- a/tests/kernels/moe/test_cpu_int4_moe.py +++ b/tests/kernels/moe/test_cpu_int4_moe.py @@ -62,6 +62,7 @@ def _make_int4_moe_weights( Returns: (w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias) where *_ref are the dequantized float reference weights + """ # Generate INT4 weights as int8 values in [-8, 7] w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8) diff --git a/tests/kernels/moe/test_cpu_quant_fused_moe.py b/tests/kernels/moe/test_cpu_quant_fused_moe.py index 84ff1ae266..99cae4534b 100644 --- a/tests/kernels/moe/test_cpu_quant_fused_moe.py +++ b/tests/kernels/moe/test_cpu_quant_fused_moe.py @@ -826,6 +826,7 @@ def _make_int4_moe_weights(E, N, K, group_size, quant_algo): w1_zeros, w2_zeros, w1_zeros_packed, w2_zeros_packed, w1_s, w2_s + """ w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32) w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32) diff --git a/tests/kernels/moe/test_cutedsl_moe.py b/tests/kernels/moe/test_cutedsl_moe.py index 3a371968fc..09a7256a5c 100644 --- a/tests/kernels/moe/test_cutedsl_moe.py +++ b/tests/kernels/moe/test_cutedsl_moe.py @@ -91,11 +91,9 @@ def break_fp4_bytes(a, dtype): def generate_balanced_routing( hidden_states: torch.Tensor, num_experts: int, top_k: int ): - """ - Generate routing weights and topk indices such that every expert is active. + """Generate routing weights and topk indices such that every expert is active. Returns routing_weights, topk_idx """ - num_tokens, hidden_dim = hidden_states.shape # num_tokens = batch_size * seq_len @@ -232,13 +230,13 @@ def grouped_gemm_ref( *, block_size: int = 16, ) -> torch.Tensor: - """ - Computes the reference grouped GEMM (fp4 quantized per-expert loop), + """Computes the reference grouped GEMM (fp4 quantized per-expert loop), computes flashinfer grouped GEMM (for scale consistency), and returns ONLY the repacked reference output: out_ref. Returns: out_ref: Tensor [num_experts, max_m, n_out] + """ device_hs = hidden_states_expanded.device device_w = weights.device diff --git a/tests/kernels/moe/test_deepep_deepgemm_moe.py b/tests/kernels/moe/test_deepep_deepgemm_moe.py index efb1e2f296..b755741380 100644 --- a/tests/kernels/moe/test_deepep_deepgemm_moe.py +++ b/tests/kernels/moe/test_deepep_deepgemm_moe.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test DeepEP + DeepGEMM integration +"""Test DeepEP + DeepGEMM integration DeepGEMM are gemm kernels specialized for the fp8 block-quantized case. """ @@ -92,9 +91,7 @@ def make_block_quant_fp8_weights( k: int, block_size: list[int], ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Return weights w1q, w2q, w1_scale, w2_scale - """ + """Return weights w1q, w2q, w1_scale, w2_scale.""" (_, w1q, w1_scale, _), (_, w2q, w2_scale, _) = make_test_weights( e, n, k, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size ) @@ -459,10 +456,7 @@ def test_ht_deepep_deepgemm_moe( disable_deepgemm_ue8m0, workspace_init, ): - """ - Tests for High-Throughput DeepEP + DeepGemm integration. - """ - + """Tests for High-Throughput DeepEP + DeepGemm integration.""" m, n, k = mnk set_random_seed(7) @@ -533,9 +527,7 @@ def test_ll_deepep_deepgemm_moe( disable_deepgemm_ue8m0, workspace_init, ): - """ - Tests for Low-Latency DeepEP + DeepGemm integration. - """ + """Tests for Low-Latency DeepEP + DeepGemm integration.""" assert not is_deep_gemm_e8m0_used() m, n, k = mnk diff --git a/tests/kernels/moe/test_deepep_moe.py b/tests/kernels/moe/test_deepep_moe.py index ce9e4f343d..ea2113d64b 100644 --- a/tests/kernels/moe/test_deepep_moe.py +++ b/tests/kernels/moe/test_deepep_moe.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test deepep dispatch-combine logic -""" +"""Test deepep dispatch-combine logic.""" import dataclasses @@ -56,9 +54,7 @@ MAX_TOKENS_PER_RANK = 64 def make_weights( e, n, k, dtype ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Return weights w1, w2, w1_scale, w2_scale - """ + """Return weights w1, w2, w1_scale, w2_scale.""" if dtype in [torch.float16, torch.bfloat16]: w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10 w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10 diff --git a/tests/kernels/moe/test_deepep_v2_moe.py b/tests/kernels/moe/test_deepep_v2_moe.py index f1e34d5bd7..2fbde2f86e 100644 --- a/tests/kernels/moe/test_deepep_v2_moe.py +++ b/tests/kernels/moe/test_deepep_v2_moe.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test DeepEP v2 (ElasticBuffer) dispatch-combine logic. +"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic. Compares against a pure-PyTorch reference MoE implementation. """ diff --git a/tests/kernels/moe/test_deepgemm.py b/tests/kernels/moe/test_deepgemm.py index ae6c9685a4..f6df64c51b 100644 --- a/tests/kernels/moe/test_deepgemm.py +++ b/tests/kernels/moe/test_deepgemm.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP). +"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP). Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts. """ @@ -79,8 +78,7 @@ def make_block_quant_fp8_weights( k: int, block_size: list[int], ): - """ - Generate (w1, w2) expert weights and their per-block scale tensors + """Generate (w1, w2) expert weights and their per-block scale tensors in FP8 block-quantized format. w1 shape: (E, 2N, K) @@ -121,8 +119,7 @@ def make_block_quant_fp8_weights( def run_single_case(m, n, k, topk, num_experts, block_size): - """ - Run one (M,N,K) configuration on a single GPU and assert DeepGEMM == + """Run one (M,N,K) configuration on a single GPU and assert DeepGEMM == Triton baseline within tolerance. """ tokens_bf16 = ( @@ -248,8 +245,7 @@ def make_mxfp4_weights( n: int, k: int, ): - """ - Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales, + """Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales, plus BF16 reference weights for validation. w1 shape: (E, 2N, K//2) uint8 — packed FP4 @@ -310,8 +306,7 @@ def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids): def run_single_fp4_case(m, n, k, topk, num_experts): - """ - Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert + """Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert DeepGEMM FP4 == BF16 reference within tolerance. """ tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5) diff --git a/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py b/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py index d8e449ec0a..3d637258d3 100644 --- a/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py +++ b/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py @@ -83,7 +83,7 @@ def _reference_activation( beta: float | None, limit: float | None, ): - """vLLM's own op for this activation, so the reference is not re-derived.""" + """VLLM's own op for this activation, so the reference is not re-derived.""" if activation == MoEActivation.RELU2_NO_MUL: return lambda x: torch.square(torch.relu(x)) if activation == MoEActivation.SWIGLUOAI: diff --git a/tests/kernels/moe/test_fused_topk.py b/tests/kernels/moe/test_fused_topk.py index 9c196a884d..18d1d9b4df 100644 --- a/tests/kernels/moe/test_fused_topk.py +++ b/tests/kernels/moe/test_fused_topk.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the MoE fused topk kernel +"""Tests for the MoE fused topk kernel. Run `pytest tests/kernels/moe/test_fused_topk.py`. """ diff --git a/tests/kernels/moe/test_grouped_topk.py b/tests/kernels/moe/test_grouped_topk.py index 1ab8550d64..c65cd381db 100644 --- a/tests/kernels/moe/test_grouped_topk.py +++ b/tests/kernels/moe/test_grouped_topk.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the MoE grouped topk kernel +"""Tests for the MoE grouped topk kernel. Run `pytest tests/kernels/moe/test_grouped_topk.py`. """ diff --git a/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py b/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py index 6756d479fe..97041e24b9 100644 --- a/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py +++ b/tests/kernels/moe/test_marlin_vs_trtllm_mxint4.py @@ -31,6 +31,7 @@ def mxint4_quantize( Returns: - uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7] - scales in BF16: [..., k//sf_vec_size] + """ x_reshaped = x.reshape(-1, sf_vec_size) x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32) @@ -61,6 +62,7 @@ def mxint4_quantize_moe_weights( Returns: - weights_mxint4: Quantized weights [e, n, k//2] uint8 - scales_mxint4: Quantization scales [e, n, k//group_size] bf16 + """ e = weights_bf16.shape[0] weight_list = [] @@ -101,6 +103,7 @@ def marlin_quantize_moe_weights( Returns: - weights_marlin: Marlin quantized weights [e, k//8, n] int32 - scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16 + """ from vllm.model_executor.layers.quantization.utils.marlin_utils_test import ( marlin_quantize, diff --git a/tests/kernels/moe/test_modular_oai_triton_moe.py b/tests/kernels/moe/test_modular_oai_triton_moe.py index d0fd4df23c..47be82dd2a 100644 --- a/tests/kernels/moe/test_modular_oai_triton_moe.py +++ b/tests/kernels/moe/test_modular_oai_triton_moe.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test modular OAI Triton MoE -""" +"""Test modular OAI Triton MoE.""" from __future__ import annotations diff --git a/tests/kernels/moe/test_moe.py b/tests/kernels/moe/test_moe.py index ea732d6887..8b4efaf913 100644 --- a/tests/kernels/moe/test_moe.py +++ b/tests/kernels/moe/test_moe.py @@ -119,8 +119,7 @@ def iterative_moe( expert_map: torch.Tensor = None, renormalize: bool = False, ) -> torch.Tensor: - """ - Baseline implementation of fused moe. + """Baseline implementation of fused moe. Args: hidden_states: [*, hidden_size] @@ -128,6 +127,7 @@ def iterative_moe( w2: [num_experts, hidden_size, intermediate_size] gating_output: [*, num_experts] expert_map: [num_experts] + """ orig_shape = hidden_states.shape hidden_size = hidden_states.shape[-1] @@ -1956,8 +1956,7 @@ def test_batched_fused_marlin_moe( ) def is_valid(self): - """ - Return True only if the input can be represented in a Batched + """Return True only if the input can be represented in a Batched format. """ return torch.all(self.expert_num_tokens_cpu <= self.max_tokens_per_batch) @@ -2078,9 +2077,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend( dtype: torch.dtype, workspace_init, ): - """ - Test BF16 unquantized MoE with FlashInfer TRTLLM backend. - """ + """Test BF16 unquantized MoE with FlashInfer TRTLLM backend.""" set_random_seed(7) from vllm.model_executor.layers.fused_moe.config import ( diff --git a/tests/kernels/moe/test_moe_align_block_size.py b/tests/kernels/moe/test_moe_align_block_size.py index 8c16690785..a1599f5ad3 100644 --- a/tests/kernels/moe/test_moe_align_block_size.py +++ b/tests/kernels/moe/test_moe_align_block_size.py @@ -54,8 +54,7 @@ def _verify_expert_level_sorting( valid_length: int, total_tokens: int, ): - """ - Verify that actual_sorted_ids follows the correct expert-level sorting. + """Verify that actual_sorted_ids follows the correct expert-level sorting. The kerne limplementation may or may not preserve original token order in topk_ids in the final sorted_ids however this does not impact quality. """ @@ -96,8 +95,7 @@ def torch_moe_align_block_size( expert_map: torch.Tensor | None = None, pad_sorted_ids: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Golden torch implementation of moe_align_block_size. + """Golden torch implementation of moe_align_block_size. This function aligns the token distribution across experts to be compatible with block size for matrix multiplication by sorting tokens by expert and @@ -189,7 +187,7 @@ def torch_moe_align_block_size( def test_moe_align_block_size( m: int, topk: int, num_experts: int, block_size: int, pad_sorted_ids: bool ): - """Test moe_align_block_size without expert mapping""" + """Test moe_align_block_size without expert mapping.""" topk_ids = torch.zeros((m, topk), device="cuda", dtype=torch.int32) for i in range(m): experts = torch.randperm(num_experts, device="cuda")[:topk] @@ -254,7 +252,7 @@ def test_moe_align_block_size_with_expert_map( block_size: int, mask_inactive_experts: bool, ): - """Test moe_align_block_size with expert mapping (EP scenario)""" + """Test moe_align_block_size with expert mapping (EP scenario).""" expert_map = torch.full((num_experts,), -1, device="cuda", dtype=torch.int32) local_experts = list(range(0, num_experts, 2)) for i, expert_id in enumerate(local_experts): diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index e1a152a233..5b1af59731 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -240,8 +240,7 @@ def maybe_roundup_layer_hidden_size( act_dtype: torch.dtype, backend: str | None, ) -> int: - """ - Given layer hidden size and MoE configurations, round up hidden_size + """Given layer hidden size and MoE configurations, round up hidden_size if necessary. Args: @@ -253,6 +252,7 @@ def maybe_roundup_layer_hidden_size( Rounded up hidden_size if rounding up is required based on the configs and all2all backend. Original hidden size otherwise. + """ if backend == "deepep_high_throughput": from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import ( @@ -945,6 +945,7 @@ def create_shared_experts_from_config( Returns: TestMLP instance or None if config is None + """ if shared_experts_config is None: return None @@ -991,6 +992,7 @@ def setup_moe_test_data( Returns: MoETestData containing all test data and transforms + """ # For latent MoE: latent_size = k // 2 latent_size = k // 2 @@ -1710,7 +1712,6 @@ def test_moe_layer_no_parallel( monkeypatch, ): """Test MoE layer without parallelism (dp_size=1, tp_size=1, use_ep=False).""" - if os.environ.get("VLLM_LOGGING_LEVEL") is None: monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR") diff --git a/tests/kernels/moe/test_moe_permute_unpermute.py b/tests/kernels/moe/test_moe_permute_unpermute.py index 4089bbfe84..76d1e57f58 100644 --- a/tests/kernels/moe/test_moe_permute_unpermute.py +++ b/tests/kernels/moe/test_moe_permute_unpermute.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the MOE permute/unpermute kernel +"""Tests for the MOE permute/unpermute kernel. Run `pytest tests/kernels/test_moe_permute_unpermute.py`. """ diff --git a/tests/kernels/moe/test_mxfp4_moe.py b/tests/kernels/moe/test_mxfp4_moe.py index 16b233b935..7c7ca10885 100644 --- a/tests/kernels/moe/test_mxfp4_moe.py +++ b/tests/kernels/moe/test_mxfp4_moe.py @@ -67,8 +67,7 @@ def compute_ref_output( @pytest.mark.parametrize("num_experts", [8, 16, 32]) @pytest.mark.parametrize("out_dtype", [torch.bfloat16]) def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype): - """ - Test the MXFP4 grouped GEMM kernel by: + """Test the MXFP4 grouped GEMM kernel by: 1. Creating random per-expert inputs and weights 2. Quantizing both to MXFP4 using the CUDA kernel 3. Running the CUTLASS grouped GEMM @@ -205,9 +204,7 @@ def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype): reason="mxfp4_experts_quant requires CUDA SM100", ) def test_mxfp4_experts_quant_basic(): - """ - Basic smoke test for the MXFP4 experts quantization kernel. - """ + """Basic smoke test for the MXFP4 experts quantization kernel.""" device = "cuda" num_experts = 4 k = 256 @@ -295,8 +292,7 @@ def compute_reference_e8m0_scale(block_max: float) -> int: @pytest.mark.parametrize("k", [256, 7168]) @pytest.mark.parametrize("m", [16, 64]) def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k): - """ - Test that mxfp4_experts_quant computes E8M0 block scales correctly + """Test that mxfp4_experts_quant computes E8M0 block scales correctly per OCP MX spec (not the NVFP4 formula). The old buggy kernel used: floor(log2(max/6)) + 127 @@ -416,8 +412,7 @@ def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k): reason="mxfp4_experts_quant requires CUDA SM100", ) def test_mxfp4_experts_quant_no_saturation(): - """ - Test that the E8M0 scale is large enough to avoid unexpected saturation. + """Test that the E8M0 scale is large enough to avoid unexpected saturation. With the buggy NVFP4 formula, the scale was too small causing most values to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that diff --git a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py index 830ff48734..51aff4ea3b 100644 --- a/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py +++ b/tests/kernels/moe/test_mxfp8_aiter_backend_selection.py @@ -161,7 +161,7 @@ def test_gfx950_picks_aiter(): def test_gfx942_picks_emulation(): - """flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton + """Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton dot_scaled backend wins instead.""" with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False): backend, experts_cls = select_mxfp8_moe_backend(_config()) diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 9157d4eaa2..d513a14f77 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -214,8 +214,7 @@ def reference_moe( activation: str = "swiglu", use_interleaved_layout: bool = False, ): - """ - Reference MoE implementation for accuracy testing. + """Reference MoE implementation for accuracy testing. Args: activation: One of "swiglu", "silu", "relu2". Controls the activation @@ -224,6 +223,7 @@ def reference_moe( (gate=x[..., ::2], up=x[..., 1::2]) as used by SWIGLUOAI. If False, uses chunked layout (gate, up = chunk(x, 2)) as used by standard swiglu/silu. + """ # renormalize routing experts = torch.topk(roouting_logits, k=topk, dim=-1, sorted=True) @@ -1441,8 +1441,7 @@ def test_rocm_mxfp4_moe_oracle( intermediate_size: int, monkeypatch: pytest.MonkeyPatch, ): - """ - Test ROCm MXFP4 MoE using oracle functions. + """Test ROCm MXFP4 MoE using oracle functions. This test validates that the oracle functions work end-to-end: - select_mxfp4_moe_backend() selects a valid backend @@ -1825,8 +1824,7 @@ def test_mxfp4_emulation_rounds_up_to_block_size( def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons( monkeypatch: pytest.MonkeyPatch, ): - """ - select_mxfp4_moe_backend() must raise NotImplementedError, with the + """select_mxfp4_moe_backend() must raise NotImplementedError, with the collected per-backend unsupported reasons in the message, when no backend supports the requested deployment configuration. """ diff --git a/tests/kernels/moe/test_routing.py b/tests/kernels/moe/test_routing.py index 0eded09b1a..174950e06c 100644 --- a/tests/kernels/moe/test_routing.py +++ b/tests/kernels/moe/test_routing.py @@ -214,8 +214,7 @@ def assert_routing_results_close( rtol: float = 1e-3, atol: float = 1e-3, ): - """ - Compare routing results, sorting by expert ID first to handle non-deterministic + """Compare routing results, sorting by expert ID first to handle non-deterministic ordering from sorted=False in topk. """ # Sort both results by expert IDs for consistent comparison @@ -294,8 +293,7 @@ def assert_aiter_routing_valid( def baseline_fused_topk( router_logits: torch.Tensor, top_k: int, renormalize: bool ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Baseline for standard fused top-k routing. + """Baseline for standard fused top-k routing. Algorithm: 1. Apply softmax to router logits @@ -320,8 +318,7 @@ def baseline_fused_topk_bias( e_score_correction_bias: torch.Tensor, routed_scaling_factor: float, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Baseline for fused top-k with bias correction. + """Baseline for fused top-k with bias correction. Algorithm: 1. Apply softmax to router logits @@ -364,8 +361,7 @@ def baseline_grouped_topk( e_score_correction_bias: torch.Tensor | None, routed_scaling_factor: float, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Baseline for grouped top-k routing (e.g., DeepSeek). + """Baseline for grouped top-k routing (e.g., DeepSeek). Algorithm: 1. Apply scoring function (softmax or sigmoid) @@ -436,8 +432,7 @@ def baseline_grouped_topk( def baseline_custom_llama4( router_logits: torch.Tensor, top_k: int ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Baseline for Llama4 custom routing. + """Baseline for Llama4 custom routing. Algorithm: 1. Select top-k expert indices (without softmax) diff --git a/tests/kernels/moe/test_routing_simulator.py b/tests/kernels/moe/test_routing_simulator.py index 35b1af618c..2cb122122f 100644 --- a/tests/kernels/moe/test_routing_simulator.py +++ b/tests/kernels/moe/test_routing_simulator.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Test script for the token-to-expert routing simulator. +"""Test script for the token-to-expert routing simulator. This script demonstrates how to use the routing simulator to test different routing strategies and analyze their performance, including diff --git a/tests/kernels/moe/test_shared_fused_moe_routed_transform.py b/tests/kernels/moe/test_shared_fused_moe_routed_transform.py index af9455b960..895e168b2d 100644 --- a/tests/kernels/moe/test_shared_fused_moe_routed_transform.py +++ b/tests/kernels/moe/test_shared_fused_moe_routed_transform.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for FusedMoEFactory with routed_input_transform. +"""Tests for FusedMoEFactory with routed_input_transform. Verifies that applying routed_input_transform inside FusedMoEFactory produces the same results as applying the transform manually outside. diff --git a/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py b/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py index 85b992ad65..0ca08c50b3 100644 --- a/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py +++ b/tests/kernels/moe/test_silu_mul_fp8_quant_deep_gemm.py @@ -142,9 +142,7 @@ def silu_mul_quant( def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor: - """ - pack float32 scales into a int32 tensor - """ + """Pack float32 scales into a int32 tensor.""" assert x.dtype == torch.float32 E, T, G = x.size() @@ -170,8 +168,7 @@ def ref_with_scale_fmt( up: torch.Tensor, scale_fmt: DeepGemmQuantScaleFMT, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - The precision types of the operations triggered by this function + """The precision types of the operations triggered by this function match closely with the kernel implementation so we compare more accurately. """ @@ -204,8 +201,7 @@ def ref_with_scale_fmt( def token_random(E, T, H2, tokens_per_expert): - """ - Initialize each token in a random range so we test a range of + """Initialize each token in a random range so we test a range of scale values. """ y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE) diff --git a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py index bfe763b44a..204954d1fe 100644 --- a/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py +++ b/tests/kernels/moe/test_silu_mul_per_token_group_quant_fp8_colmajor.py @@ -19,11 +19,9 @@ GROUP_SIZE = 128 def reference_quant(x: torch.Tensor, use_ue8m0: bool): - """ - Reference triton quant kernel from, + """Reference triton quant kernel from, vllm.model_executor.layers.quantization.utils.fp8_utils """ - x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE) # Allocate the scale tensor in column-major format. diff --git a/tests/kernels/moe/test_triton_moe_ptpc_fp8.py b/tests/kernels/moe/test_triton_moe_ptpc_fp8.py index 01246325c8..8b10ff75e6 100644 --- a/tests/kernels/moe/test_triton_moe_ptpc_fp8.py +++ b/tests/kernels/moe/test_triton_moe_ptpc_fp8.py @@ -57,7 +57,6 @@ def fp8_mask(a, mask): def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk): """This function performs fused moe with per-column int8 quantization using native torch.""" - B, D = a.shape # Perform per-token quantization a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True) diff --git a/tests/kernels/moe/test_trtllm_bf16_moe.py b/tests/kernels/moe/test_trtllm_bf16_moe.py index 7b6b7d81ff..9e50b4fc3b 100644 --- a/tests/kernels/moe/test_trtllm_bf16_moe.py +++ b/tests/kernels/moe/test_trtllm_bf16_moe.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for the FlashInfer TRTLLM BF16 MoE backend +"""Tests for the FlashInfer TRTLLM BF16 MoE backend (`TrtLlmBf16ExpertsModular`). This mirrors the TRTLLM NvFP4 modular test shape: construct the modular diff --git a/tests/kernels/moe/test_trtllm_nvfp4_moe.py b/tests/kernels/moe/test_trtllm_nvfp4_moe.py index 59eaa4c139..b9684396b2 100644 --- a/tests/kernels/moe/test_trtllm_nvfp4_moe.py +++ b/tests/kernels/moe/test_trtllm_nvfp4_moe.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for the FlashInfer TRTLLM NvFP4 MoE backend +"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend (`TrtLlmNvFp4ExpertsModular`). Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated), diff --git a/tests/kernels/moe/test_zero_expert_moe.py b/tests/kernels/moe/test_zero_expert_moe.py index c2084800f9..0202b380c5 100644 --- a/tests/kernels/moe/test_zero_expert_moe.py +++ b/tests/kernels/moe/test_zero_expert_moe.py @@ -83,8 +83,7 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token @pytest.mark.parametrize("num_tokens", [1, 32]) def test_zero_expert_moe_forward(zero_expert_moe, num_tokens): - """ - Run a forward pass through FusedMoEFactory with zero experts + """Run a forward pass through FusedMoEFactory with zero experts and verify output shape. """ layer, vllm_config = zero_expert_moe diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index e91fa58f3e..e762472950 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -64,8 +64,7 @@ def make_dummy_moe_config( max_num_tokens: int = 512, activation: MoEActivation = MoEActivation.SILU, ) -> FusedMoEConfig: - """ - This is a dummy config for the mk constructor interface + """This is a dummy config for the mk constructor interface as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN do not actually use this config. diff --git a/tests/kernels/quantization/nvfp4_utils.py b/tests/kernels/quantization/nvfp4_utils.py index e0120301f7..04f7943ae6 100644 --- a/tests/kernels/quantization/nvfp4_utils.py +++ b/tests/kernels/quantization/nvfp4_utils.py @@ -118,6 +118,7 @@ def dequant_nvfp4_kv_cache( Returns: [..., num_heads, block_size, head_size] float32. + """ data_dim = head_size // 2 scale_dim = head_size // 16 diff --git a/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py b/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py index b70a1a8543..e4dadb53bd 100644 --- a/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py +++ b/tests/kernels/quantization/test_cpu_fp8_scaled_mm.py @@ -45,6 +45,7 @@ def quantize_weight_block_fp8( Returns: fp8_weight: [N, K] float8_e4m3fn scales: [n_tiles, k_tiles] float32 + """ N, K = weight.shape block_n, block_k = block_size diff --git a/tests/kernels/quantization/test_cutlass_scaled_mm.py b/tests/kernels/quantization/test_cutlass_scaled_mm.py index a522d31714..9e1d6d2cc7 100644 --- a/tests/kernels/quantization/test_cutlass_scaled_mm.py +++ b/tests/kernels/quantization/test_cutlass_scaled_mm.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for cutlass kernels +"""Tests for cutlass kernels. Run `pytest tests/kernels/quantization/test_cutlass_scaled_mm.py`. """ diff --git a/tests/kernels/quantization/test_cutlass_w4a8.py b/tests/kernels/quantization/test_cutlass_w4a8.py index 8cfc993fe8..33bb7bccc7 100644 --- a/tests/kernels/quantization/test_cutlass_w4a8.py +++ b/tests/kernels/quantization/test_cutlass_w4a8.py @@ -300,8 +300,7 @@ def test_w4a8_cuda_graph(): ) @pytest.mark.parametrize("shape", MNK_SHAPES) def test_convert_packed_uint4b8_to_signed_int4_inplace(shape): - """ - The W4A16 checkpoints encode the weights as int4b8 packed to int32. + """The W4A16 checkpoints encode the weights as int4b8 packed to int32. The CUTLASS kernels expect signed int4 packed to int32. This tests checks that the runtime int4b8 -> signed int4 conversion matches the offline conversion step exactly. diff --git a/tests/kernels/quantization/test_cutlass_w4a8_moe.py b/tests/kernels/quantization/test_cutlass_w4a8_moe.py index 5e6c170db6..10a76e553e 100644 --- a/tests/kernels/quantization/test_cutlass_w4a8_moe.py +++ b/tests/kernels/quantization/test_cutlass_w4a8_moe.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for the CUTLASS-based W4A8 grouped GEMM kernel and the full MoE layer. -""" +"""Tests for the CUTLASS-based W4A8 grouped GEMM kernel and the full MoE layer.""" import random from dataclasses import dataclass @@ -37,8 +35,7 @@ def cutlass_quantize( group_size: int | None, zero_points: bool = False, ): - """ - Quantize weights into W4 and compute reference dequantized weights. + """Quantize weights into W4 and compute reference dequantized weights. Encoding/reordering of weights and packing of scales is deferred until after all experts are combined. @@ -66,13 +63,13 @@ def cutlass_quantize( def cutlass_preprocess( w_q_experts: list[torch.Tensor], w_s_experts: list[torch.Tensor] ): - """ - Reorder/encode expert weights and pack scales. + """Reorder/encode expert weights and pack scales. Returns: w_q_packed: Packed/encoded int4 weights for all experts. w_s_packed: Packed fp8 scales for all experts. packed_layout: Layout/stride metadata for grouped GEMM. + """ w_s_packed = ops.cutlass_pack_scale_fp8(torch.stack(w_s_experts)) w_q_packed, packed_layout = ops.cutlass_encode_and_reorder_int4b_grouped( @@ -130,7 +127,6 @@ def make_moe_test_setup( random_zero: bool = False, ) -> MoETestSetup: """Create a full set of tensors for testing cutlass_w4a8_moe_mm.""" - assert K % GROUP_SIZE == 0 # Token counts per expert (multiples of `alignment`). Ms = [alignment * random.randint(1, max_blocks) for _ in range(num_experts)] @@ -276,9 +272,7 @@ def test_cutlass_w4a8_moe_mm_end_to_end(shape, random_zero): class W4A8MoELayer(torch.nn.Module): - """ - Minimal wrapper module to test cuda graphs - """ + """Minimal wrapper module to test cuda graphs.""" def __init__(self, setup: MoETestSetup): super().__init__() diff --git a/tests/kernels/quantization/test_fp8_min_max_helper.py b/tests/kernels/quantization/test_fp8_min_max_helper.py index 8cd68a3fef..4f1beb8576 100644 --- a/tests/kernels/quantization/test_fp8_min_max_helper.py +++ b/tests/kernels/quantization/test_fp8_min_max_helper.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for the get_fp8_min_max() helper function. +"""Unit tests for the get_fp8_min_max() helper function. These tests verify the FP8 min/max value logic for both standard and fnuz (ROCm MI300) dtype handling. diff --git a/tests/kernels/quantization/test_int8_kernel.py b/tests/kernels/quantization/test_int8_kernel.py index 704bb90f84..4858ea47ad 100644 --- a/tests/kernels/quantization/test_int8_kernel.py +++ b/tests/kernels/quantization/test_int8_kernel.py @@ -54,7 +54,6 @@ def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16): def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, topk, topk_weight, topk_ids): """This function performs fused moe with per-column int8 quantization using native torch.""" - B, D = a.shape # Perform per-token quantization a_q, a_s = per_token_quant_int8(a) diff --git a/tests/kernels/quantization/test_mxfp4_kernel_selection.py b/tests/kernels/quantization/test_mxfp4_kernel_selection.py index 28ce615ce6..4adfffa7c8 100644 --- a/tests/kernels/quantization/test_mxfp4_kernel_selection.py +++ b/tests/kernels/quantization/test_mxfp4_kernel_selection.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for MXFP4 linear kernel selection logic (CPU-only) +"""Tests for MXFP4 linear kernel selection logic (CPU-only). Run `pytest tests/kernels/quantization/test_mxfp4_kernel_selection.py`. """ diff --git a/tests/kernels/quantization/test_mxfp4_triton_ep.py b/tests/kernels/quantization/test_mxfp4_triton_ep.py index 93d30f4598..1edaa86cac 100644 --- a/tests/kernels/quantization/test_mxfp4_triton_ep.py +++ b/tests/kernels/quantization/test_mxfp4_triton_ep.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests that triton_kernel_moe_forward correctly applies expert_map +"""Tests that triton_kernel_moe_forward correctly applies expert_map remapping when expert parallelism (EP) is enabled. When expert_map is provided, global expert IDs are remapped to local IDs diff --git a/tests/kernels/quantization/test_mxfp6_kernel_selection.py b/tests/kernels/quantization/test_mxfp6_kernel_selection.py index 2c7b565bea..5c5b1b3b9e 100644 --- a/tests/kernels/quantization/test_mxfp6_kernel_selection.py +++ b/tests/kernels/quantization/test_mxfp6_kernel_selection.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for MXFP6 linear kernel selection logic (CPU-only) +"""Tests for MXFP6 linear kernel selection logic (CPU-only). Run `pytest tests/kernels/quantization/test_mxfp6_kernel_selection.py`. """ diff --git a/tests/kernels/quantization/test_nvfp4_emulation.py b/tests/kernels/quantization/test_nvfp4_emulation.py index eb057cebd9..33295b8784 100644 --- a/tests/kernels/quantization/test_nvfp4_emulation.py +++ b/tests/kernels/quantization/test_nvfp4_emulation.py @@ -64,8 +64,7 @@ def loaded_model_files(): class Nvfp4QuantizationEmulationTritonExpertsReference(TritonExperts): - """ - Extension of TritonExperts to support emulated NVFP4 MoE experts. + """Extension of TritonExperts to support emulated NVFP4 MoE experts. It may be used for NVFP4 models when the device does not have native support for this dtype. diff --git a/tests/kernels/quantization/test_nvfp4_quant.py b/tests/kernels/quantization/test_nvfp4_quant.py index 120dc2774a..b8a1cdda46 100644 --- a/tests/kernels/quantization/test_nvfp4_quant.py +++ b/tests/kernels/quantization/test_nvfp4_quant.py @@ -176,8 +176,7 @@ def test_python_util_matches_cpp_allocation( shape: tuple[int, int], is_sf_swizzled_layout: bool, ) -> None: - """ - Verify that the Python utility (create_fp4_output_tensors) allocates + """Verify that the Python utility (create_fp4_output_tensors) allocates tensors with the same shapes and dtypes as the C++ functional variant (scaled_fp4_quant_func). """ diff --git a/tests/kernels/quantization/test_per_token_group_quant.py b/tests/kernels/quantization/test_per_token_group_quant.py index f2326bfd25..6d3c91022e 100644 --- a/tests/kernels/quantization/test_per_token_group_quant.py +++ b/tests/kernels/quantization/test_per_token_group_quant.py @@ -119,7 +119,6 @@ def test_per_token_group_quant_fp8_packed( ): """Test the packed DeepGEMM quantization kernel against the Triton reference (row-major, UE8M0 scales).""" - device = "cuda" torch.manual_seed(42) @@ -206,7 +205,6 @@ def test_per_token_group_quant_fp8_packed_all_zero(): For all-zero input, eps/fp8_max < 1e-10, so the inner fmax clamps back to 1e-10, giving exp2(ceil(log2(1e-10))) = exp2(-33) => UE8M0 byte 0x5E (94). """ - device = "cuda" num_tokens, hidden_dim, group_size = 4, 7168, 128 x = torch.zeros((num_tokens, hidden_dim), device=device, dtype=torch.bfloat16) @@ -259,7 +257,6 @@ def test_per_token_group_quant_fp8_packed_mantissa_rounds_up(): """Inputs whose absmax/max_8bit produces a non-power-of-2 force the mantissa-rounding-up branch (exp_byte += 1). Locks down this behavior before optimization.""" - device = "cuda" num_tokens, hidden_dim, group_size = 4, 7168, 128 @@ -332,7 +329,6 @@ def test_per_token_group_quant_fp8_packed_zero_fills_padded_output_q( """When output_q is allocated with shape (tma_aligned_mn, k) instead of (mn, k), the kernel must overwrite the padded mn rows with zeros so callers can use ``torch.empty`` instead of ``torch.zeros``.""" - device = "cuda" group_size = 128 torch.manual_seed(42) @@ -396,7 +392,6 @@ def test_per_token_group_quant_fp8_packed_large_mn(): This is a differential test that compares fp8 output against Triton output reference when token size sits just above the gridDim.y 2^16 - 1 limit. """ - device = "cuda" group_size = 128 # hidden 2048 -> 2048/128 = 16 groups per row -> kx=16, ry=1: one grid row per mn diff --git a/tests/kernels/quantization/test_rocm_mxfp4.py b/tests/kernels/quantization/test_rocm_mxfp4.py index 43b5daa45a..0e14e2d258 100644 --- a/tests/kernels/quantization/test_rocm_mxfp4.py +++ b/tests/kernels/quantization/test_rocm_mxfp4.py @@ -504,7 +504,7 @@ def test_aiter_fp4_gemm_a4w4_determinism(): ], ) def test_aiter_hardware_fp4_dynamic_quant_format(shape): - """aiter hardware FP4 dynamic quant produces correct output format. + """Aiter hardware FP4 dynamic quant produces correct output format. Tests gfx950 hardware-accelerated FP4 quantization (OCP MXFP4 E2M1). Parity with B200 scaled_fp4_quant: block_size=32, packed uint8 output. diff --git a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py index 8275714744..d682ed05b0 100644 --- a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py +++ b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for ScaledMM kernel selection logic (CPU-only) +"""Tests for ScaledMM kernel selection logic (CPU-only). Run `pytest tests/kernels/quantization/test_scaled_mm_kernel_selection.py`. """ diff --git a/tests/kernels/quantization/test_triton_scaled_mm.py b/tests/kernels/quantization/test_triton_scaled_mm.py index e829c63a84..55d1f76b2d 100644 --- a/tests/kernels/quantization/test_triton_scaled_mm.py +++ b/tests/kernels/quantization/test_triton_scaled_mm.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the triton_scaled_mm kernel +"""Tests for the triton_scaled_mm kernel. Run `pytest tests/kernels/quantization/test_triton_scaled_mm.py`. """ diff --git a/tests/kernels/test_apply_repetition_penalties.py b/tests/kernels/test_apply_repetition_penalties.py index 8270cf885f..c1551e2e82 100644 --- a/tests/kernels/test_apply_repetition_penalties.py +++ b/tests/kernels/test_apply_repetition_penalties.py @@ -35,8 +35,7 @@ def test_apply_repetition_penalties( dtype: torch.dtype, seed: int, ) -> None: - """ - Test the apply_repetition_penalties custom op + """Test the apply_repetition_penalties custom op against a reference implementation. """ set_random_seed(seed) @@ -86,8 +85,7 @@ def test_apply_repetition_penalties( ) @torch.inference_mode() def test_apply_repetition_penalties_zero_seqs() -> None: - """ - Test the apply_repetition_penalties custom op with num_seqs=0 + """Test the apply_repetition_penalties custom op with num_seqs=0 against a reference implementation. """ num_seqs = 0 diff --git a/tests/kernels/test_awq_int4_to_int8.py b/tests/kernels/test_awq_int4_to_int8.py index f94214785d..dc7124e9ae 100644 --- a/tests/kernels/test_awq_int4_to_int8.py +++ b/tests/kernels/test_awq_int4_to_int8.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for AWQ INT4 W4A8 GEMM pipeline (SGLang kernel migration). +"""Unit tests for AWQ INT4 W4A8 GEMM pipeline (SGLang kernel migration). Part 1: Weight packing tests - convert_weight_packed_scale_zp correctness @@ -44,6 +43,7 @@ def make_awq_checkpoint_data(K, N, group_size, seed=42): float_ref: [K, N] float32, reference dequantized weights weight_int4_orig: [K, N] int32, original int4 values (0-15) zeros_int4_orig: [num_groups, N] int32, original zero points (0-15) + """ rng = np.random.RandomState(seed) num_groups = K // group_size diff --git a/tests/kernels/test_cache_kernels.py b/tests/kernels/test_cache_kernels.py index eff4508b0d..a3cb91242a 100644 --- a/tests/kernels/test_cache_kernels.py +++ b/tests/kernels/test_cache_kernels.py @@ -15,12 +15,10 @@ except ImportError: @pytest.mark.skipif(torch.accelerator.device_count() < 1, reason="Need CUDA device") def test_gather_cache_oob(): - """ - Tests for OOB read in gather_and_maybe_dequant_cache (Issue #27909). + """Tests for OOB read in gather_and_maybe_dequant_cache (Issue #27909). This test constructs a boundary case identified in the issue where seq_starts causes the block_table offset to read out of bounds. """ - block_size = 64 # The kernel only supports the MLA entry sizes. entry_size = 576 diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 73c2027a78..95ff6b3c05 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Round-trip tests for compressor → FP8 quant + KV cache insert → gather + dequant. +"""Round-trip tests for compressor → FP8 quant + KV cache insert → gather + dequant. These tests cover: A) DeepseekV4 Attention: head_dim=512 (448 FP8 nope + 64 bf16 rope), quant_block=64 @@ -1097,7 +1096,6 @@ def test_get_c128_boundary(starts, query_start_loc, expected): def test_deepseek_v4_attention_quant_cache_roundtrip(num_tokens: int, block_size: int): """compressed_kv → quantize_and_insert_k_cache → dequantize_and_gather_k_cache → compare against original.""" - HEAD_DIM = 512 NOPE_DIM = 448 HEAD_BYTES = 584 # 448 fp8 + 128 bf16 + 8 uint8 scale @@ -1308,9 +1306,8 @@ def test_dequantize_and_gather_k_cache( @pytest.mark.parametrize("num_tokens", [1, 4, 8, 17]) @pytest.mark.parametrize("block_size", [16, 64]) def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int): - """k → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache + """K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache → manual dequant → compare against original.""" - HEAD_DIM = 128 QUANT_BLOCK_SIZE = 128 # cache_stride = head_dim + (head_dim * 4 / quant_block_size) = 128 + 4 = 132 @@ -1374,7 +1371,6 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int): def test_indexer_gather_accepts_upper_bound_output(): """Gather only exact cu_seq_lens even when dst is over-allocated.""" - head_dim = 128 quant_block_size = 128 cache_stride = head_dim + head_dim * 4 // quant_block_size @@ -1474,7 +1470,6 @@ def test_indexer_gather_accepts_upper_bound_output(): def test_deepseek_v4_quant_magnitude_range(): """Test that quantization handles a range of magnitudes correctly.""" - HEAD_DIM = 512 NOPE_DIM = 448 HEAD_BYTES = 584 diff --git a/tests/kernels/test_cp_gather_fp8.py b/tests/kernels/test_cp_gather_fp8.py index fe539abb4f..40c5ef5fdb 100644 --- a/tests/kernels/test_cp_gather_fp8.py +++ b/tests/kernels/test_cp_gather_fp8.py @@ -29,6 +29,7 @@ def _build_test_case(seq_lens, block_size, seed=42): Returns: Tuple of (cache, block_table, workspace_starts_t, num_reqs, total_tokens, expected_output). + """ torch.manual_seed(seed) diff --git a/tests/kernels/test_flex_attention.py b/tests/kernels/test_flex_attention.py index 12d00ec405..6862f8ffb1 100644 --- a/tests/kernels/test_flex_attention.py +++ b/tests/kernels/test_flex_attention.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Integration tests for FlexAttention backend vs default backend""" +"""Integration tests for FlexAttention backend vs default backend.""" import pytest import torch diff --git a/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py b/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py index a0dcfc5bb7..ced96bc401 100644 --- a/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py +++ b/tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel: +"""Standalone unit test for the horizontally-fused DeepseekV4-MLA kernel: fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert - Q side: optional per-head RMSNorm + GPT-J RoPE on last 64 dims diff --git a/tests/kernels/test_fused_indexer_q_rope_quant.py b/tests/kernels/test_fused_indexer_q_rope_quant.py index 7464c90769..0ce3a91851 100644 --- a/tests/kernels/test_fused_indexer_q_rope_quant.py +++ b/tests/kernels/test_fused_indexer_q_rope_quant.py @@ -45,6 +45,7 @@ def quantize_to_mxfp4( Returns: packed: [..., head_dim//2] uint8 2 E2M1 nibbles/byte, low nibble = even index scales: [..., head_dim//32] uint8 1 ue8m0 byte + """ MXFP4_BLOCK_SIZE = 32 orig_shape = x.shape diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index d27144a38e..03d5b7921f 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Unit tests for the fused inverse RoPE + block-scaled FP8 quantization kernel. +"""Unit tests for the fused inverse RoPE + block-scaled FP8 quantization kernel. Tests compare the fused kernel against a reference implementation built from the existing separate operations (inverse RoPE via rotate_neox + FP8 quant @@ -132,6 +131,7 @@ def reference_inv_rope( Returns: o with inverse RoPE applied on the rope portion (bf16). + """ assert cos_sin_cache.dtype == torch.float32 cos_sin = cos_sin_cache[positions] # [T, rope_dim] fp32 @@ -164,6 +164,7 @@ def _ref_ue8m0_quant_block(x_f32: torch.Tensor) -> tuple[torch.Tensor, torch.Ten Returns: x_fp8: same shape, float8_e4m3fn scales: [...] float32, one scale per block + """ absmax = x_f32.abs().amax(dim=-1, keepdim=True).clamp(min=EPS) scale_raw = absmax * (1.0 / FP8_MAX) @@ -192,6 +193,7 @@ def reference_inv_rope_fp8_quant( Returns: o_fp8: [T, G, D] FP8 with strides (D, T*D, 1) o_scale: [T, G, S] FP32 with strides (S, T*S, 1) + """ assert cos_sin_cache.dtype == torch.float32 T, _H, head_dim = o.shape @@ -402,7 +404,6 @@ def test_output_strides(num_tokens, num_heads, n_groups): - FP8: logical [T, G, D] backed by contiguous [G, T, D]. - Scale: MN-major TMA-aligned (column-major: T-stride=1). """ - heads_per_group = num_heads // n_groups max_pos = 4096 device = "cuda" @@ -903,7 +904,6 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups): def test_with_real_deepseek_v4_rope(num_tokens, default_vllm_config): """Test with real DeepseekV4ScalingRotaryEmbedding (GPT-J style, mscale=0, YaRN scaling) matching the production config.""" - num_heads = 64 n_groups = 8 heads_per_group = num_heads // n_groups diff --git a/tests/kernels/test_mhc_kernels.py b/tests/kernels/test_mhc_kernels.py index 5fab150d9f..fafb37d97b 100644 --- a/tests/kernels/test_mhc_kernels.py +++ b/tests/kernels/test_mhc_kernels.py @@ -703,7 +703,7 @@ def mhc_pre_ref( hc_post_mult_value: float, sinkhorn_repeat: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """mHC pre reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_pre.py#L303""" + """MHC pre reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_pre.py#L303.""" hc_mult = residual.shape[-2] residual_flat = residual.flatten(-2, -1).float() @@ -742,7 +742,7 @@ def mhc_post_ref( post_layer_mix: torch.Tensor, comb_res_mix: torch.Tensor, ) -> torch.Tensor: - """mHC post reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_post.py#L68""" + """MHC post reference kernel from tilelang repo: https://github.com/tile-ai/tilelang/blob/d135bd1cd2d2eee74fbb41dd0a0831a427194c86/examples/deepseek_mhc/example_mhc_post.py#L68.""" term2 = torch.bmm(comb_res_mix.mT, residual.float()) return (x.float().unsqueeze(-2) * post_layer_mix + term2).bfloat16() diff --git a/tests/kernels/test_shuffle_rows.py b/tests/kernels/test_shuffle_rows.py index 08d5883a1e..4f32949eb7 100644 --- a/tests/kernels/test_shuffle_rows.py +++ b/tests/kernels/test_shuffle_rows.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the shuffle_rows function +"""Tests for the shuffle_rows function. Run `pytest tests/kernels/test_shuffle_rows.py`. """ diff --git a/tests/kernels/test_top_k_per_row.py b/tests/kernels/test_top_k_per_row.py index 2895394706..db49b8091b 100644 --- a/tests/kernels/test_top_k_per_row.py +++ b/tests/kernels/test_top_k_per_row.py @@ -157,8 +157,7 @@ def compare_top_k_results( top_k: int, tolerance: float = 1e-5, ) -> bool: - """ - Compare results from CUDA top_k_per_row with torch.topk. + """Compare results from CUDA top_k_per_row with torch.topk. Both results should be sorted and contain the same top-k elements. """ num_rows = cuda_indices.shape[0] @@ -213,8 +212,7 @@ def validate_topk_against_reference( top_k: int, kernel_name: str, ) -> None: - """ - Validate CUDA top-k results against PyTorch reference implementation. + """Validate CUDA top-k results against PyTorch reference implementation. Args: logits: Input logits tensor @@ -223,6 +221,7 @@ def validate_topk_against_reference( row_ends: Row end positions top_k: Number of top elements to select kernel_name: Name of the kernel being tested (for error messages) + """ num_rows = cuda_indices.shape[0] torch_indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda") @@ -248,9 +247,7 @@ def test_top_k_per_row( top_k: int, clean_logits: bool, ) -> None: - """ - Test top_k_per_row. - """ + """Test top_k_per_row.""" set_random_seed(0) torch.set_default_device("cuda:0") @@ -298,9 +295,7 @@ def _run_top_k_per_row_decode_test( clean_logits: bool, data_generation: str, ) -> None: - """ - Helper function to run top_k_per_row_decode test with given parameters. - """ + """Helper function to run top_k_per_row_decode test with given parameters.""" torch.set_default_device("cuda:0") # Create test data @@ -365,9 +360,7 @@ def test_top_k_per_row_decode( clean_logits: bool, data_generation: str, ) -> None: - """ - Test top_k_per_row with seq_lens tensor. - """ + """Test top_k_per_row with seq_lens tensor.""" set_random_seed(0) vocab_size = 20000 _run_top_k_per_row_decode_test( @@ -379,9 +372,7 @@ def test_top_k_per_row_decode( @pytest.mark.parametrize("clean_logits", [True, False]) @torch.inference_mode() def test_top_k_per_row_decode_large_vocab_size(clean_logits: bool) -> None: - """ - Test top_k_per_row_decode with large vocabulary size. - """ + """Test top_k_per_row_decode with large vocabulary size.""" set_random_seed(0) top_k = 2048 batch_size = 2 @@ -886,8 +877,7 @@ def test_deepseek_workspace_topk( next_n: int, backend: str, ) -> None: - """ - Test workspace top-k backends with varying sequence lengths and speculative + """Test workspace top-k backends with varying sequence lengths and speculative decoding. Supports speculative decoding with next_n > 1. """ @@ -1040,8 +1030,7 @@ def run_large_context_topk_test( seed: int = 42, backend: str = "cooperative_topk", ) -> None: - """ - Helper to run a top-k backend test with given parameters. + """Helper to run a top-k backend test with given parameters. Args: batch_size: Number of rows/sequences @@ -1050,6 +1039,7 @@ def run_large_context_topk_test( data_type: Type of test data to generate seed: Random seed for reproducibility backend: Top-k backend to test + """ torch.set_default_device("cuda:0") set_random_seed(seed) @@ -1274,8 +1264,7 @@ def test_cooperative_topk_cs2(top_k: int) -> None: @pytest.mark.parametrize("backend", WORKSPACE_TOPK_BACKENDS) @torch.inference_mode() def test_workspace_topk_correctness(test_config: dict, backend: str) -> None: - """ - Comprehensive correctness tests covering: + """Comprehensive correctness tests covering: - Sequence length edge cases (trivial, boundary, varied) - Very small sequences (< 100 elements) - Mixed sequence lengths in same batch @@ -1339,8 +1328,7 @@ def test_workspace_topk_correctness(test_config: dict, backend: str) -> None: @pytest.mark.parametrize("backend", WORKSPACE_TOPK_BACKENDS) @torch.inference_mode() def test_workspace_topk_algorithm_paths(test_config: dict, backend: str) -> None: - """ - Test different algorithm execution paths (capped at 163840 for DeepSeek V3.2): + """Test different algorithm execution paths (capped at 163840 for DeepSeek V3.2): - Batch size scalability (1, 4, 32, 256) - Single-CTA vs Multi-CTA execution - Extreme configurations (large batch, max context length) @@ -1357,8 +1345,7 @@ def test_workspace_topk_algorithm_paths(test_config: dict, backend: str) -> None @pytest.mark.parametrize("backend", WORKSPACE_TOPK_BACKENDS) @torch.inference_mode() def test_workspace_topk_stress(backend: str) -> None: - """ - Stress test with random configurations to catch edge cases. + """Stress test with random configurations to catch edge cases. Capped at 163840 (DeepSeek V3.2 max context) for realistic testing. """ torch.set_default_device("cuda:0") @@ -1505,8 +1492,7 @@ def test_cooperative_topk_512_tie_workspace_is_per_row() -> None: @pytest.mark.parametrize("backend", WORKSPACE_TOPK_BACKENDS) @torch.inference_mode() def test_workspace_topk(test_config: dict, top_k: int, backend: str) -> None: - """ - Tests specific to workspace top-k backends: + """Tests specific to workspace top-k backends: - Mixed medium/large rows in the same batch (dynamic per-row dispatch) - Boundary around LARGE_THRESHOLD (32K) - Trivial + medium + large rows in a single batch @@ -1567,8 +1553,7 @@ def test_persistent_topk_reused_group_after_short_row() -> None: @pytest.mark.parametrize("backend", WORKSPACE_TOPK_BACKENDS) @torch.inference_mode() def test_workspace_topk_padded_stride(top_k: int, backend: str) -> None: - """ - Test workspace top-k backends with padded logits (large stride, small seq_len) + """Test workspace top-k backends with padded logits (large stride, small seq_len) to simulate the e2e CUDAGraph scenario where fp8_paged_mqa_logits returns [B, max_model_len] with max_model_len=163840. """ diff --git a/tests/kernels/utils.py b/tests/kernels/utils.py index 8bd63872f3..cf68565e9d 100644 --- a/tests/kernels/utils.py +++ b/tests/kernels/utils.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Kernel test utils""" +"""Kernel test utils.""" import itertools import random @@ -94,16 +94,15 @@ def _assert_deterministic( class QKVInputs(NamedTuple): - """ - Data structure for representing unpacked attention inputs, + """Data structure for representing unpacked attention inputs, query/key/values and their sequence lengths. Attributes: - * {query,key,value}: unpacked (batch_size x padded_seq_len x num_heads x head_size) attention inputs * q_seq_lens: query sequence lengths list * kv_seq_lens: shared key/value sequence lengths list + """ query: torch.Tensor @@ -114,16 +113,15 @@ class QKVInputs(NamedTuple): class QKVO(NamedTuple): - """ - Data structure for representing unpacked attention inputs, + """Data structure for representing unpacked attention inputs, alongside unpacked known-correct attention output Attributes: - * qkv: unpacked (batch_size x padded_seq_len x num_heads x head_size) attention inputs * ideal_output: unpacked (batch_size x padded_seq_len x num_heads x head_size) known-correct attention output + """ qkv: QKVInputs @@ -131,11 +129,9 @@ class QKVO(NamedTuple): class PackedQKVInputs(NamedTuple): - """ - Data structure for representing packed attention inputs + """Data structure for representing packed attention inputs. Attributes: - * {query,key,value}: packed (number_of_tokens x num_heads x head_size) attention inputs * q_start_loc_list: list of query start locations within packed tensor @@ -143,6 +139,7 @@ class PackedQKVInputs(NamedTuple): packed tensor * q_seq_lens: query sequence lengths list * kv_seq_lens: shared key/value sequence lengths list + """ query: torch.Tensor @@ -155,16 +152,15 @@ class PackedQKVInputs(NamedTuple): class PackedQKVO(NamedTuple): - """ - Data structure for representing packed attention inputs, + """Data structure for representing packed attention inputs, alongside packed known-correct attention output Attributes: - * packed_qkv: packed (number_of_tokens x num_heads x head_size) attention inputs * ideal_output: packed (number_of_tokens x num_heads x head_size) known-correct attention output + """ packed_qkv: PackedQKVInputs | None @@ -172,13 +168,12 @@ class PackedQKVO(NamedTuple): class KVMemoryMap(NamedTuple): - """ - Data structure for encapsulating KV cache memory mapping. + """Data structure for encapsulating KV cache memory mapping. Attributes: - * block_tables: KV cache block tables * slot_mapping: mapping of sequence offset to physical address + """ block_tables: torch.Tensor @@ -186,18 +181,17 @@ class KVMemoryMap(NamedTuple): class PhaseTestParameters(NamedTuple): - """ - Data structure for encapsulating the test parameters + """Data structure for encapsulating the test parameters for a given test "phase" (prefill or decode phase) and attention scenario (encoder, decoder-self, encoder/decoder-cross) Attributes: - * packed_qkvo: packed (number_of_tokens x num_heads x head_size) attention inputs & known-correct output * kv_mmap: KV cache memory mapping, specific to this test phase & attention scenario + """ packed_qkvo: PackedQKVO @@ -208,13 +202,11 @@ def maybe_make_int_tensor( _list: list[int] | None, device: torch.device | str, ) -> torch.Tensor: - """ - Convert Python int list to a 1D int torch.Tensor on `device` + """Convert Python int list to a 1D int torch.Tensor on `device`. Returns: + If _list is not None: 1D int torch.Tensor on `device`, `None` otherwise. - * If _list is not None: 1D int torch.Tensor on `device` - * None otherwise """ return ( None if _list is None else torch.tensor(_list, dtype=torch.int, device=device) @@ -225,13 +217,11 @@ def maybe_make_long_tensor( _list: list[int] | None, device: torch.device | str, ) -> torch.Tensor: - """ - Convert Python int list to a 1D long torch.Tensor on `device` + """Convert Python int list to a 1D long torch.Tensor on `device`. Returns: + If _list is not None: 1D long torch.Tensor on `device`, `None` otherwise. - * If _list is not None: 1D long torch.Tensor on `device` - * None otherwise """ return ( None if _list is None else torch.tensor(_list, dtype=torch.long, device=device) @@ -239,11 +229,11 @@ def maybe_make_long_tensor( def maybe_max(_list: list | None) -> Number | None: - """ + """Compute the maximum of a list, if it exists. + Returns: + If _list is not None: max(_list), `None` otherwise. - * If _list is not None: max(_list) - * None otherwise """ return None if _list is None else max(_list) @@ -252,19 +242,16 @@ def make_causal_mask( q_max_seq_len: int, kv_max_seq_len: int, ) -> torch.Tensor: - """ - Create a q_max_seq_len x kv_max_seq_len causal mask + """Create a q_max_seq_len x kv_max_seq_len causal mask. Arguments: - - * q_max_seq_len: query max seq len - * kv_max_seq_len: key/value max seq len + q_max_seq_len: query max seq len + kv_max_seq_len: key/value max seq len Returns: + 2D tensor, q_max_seq_len x kv_max_seq_len - * 2D tensor, q_max_seq_len x kv_max_seq_len """ - # Create a matrix where entry (i, j) is True if i >= j mask = torch.triu(torch.ones(q_max_seq_len, kv_max_seq_len), diagonal=1) # Replace True with float('-inf') and False with 0 @@ -281,8 +268,7 @@ def ref_masked_attention( q_seq_lens: list | None = None, kv_seq_lens: list | None = None, ) -> torch.Tensor: - """ - "Golden" masked attention reference. Supports two types of masking: + """Reference ("golden") masked attention. Supports two types of masking: * Basic attention mask, utilizing {q,kv}_seq_lens args to mask out padding elements @@ -290,21 +276,18 @@ def ref_masked_attention( causal Arguments: - - * query: batch_size x q_padded_seq_len x num_heads x head_size - * key: batch_size x kv_padded_seq_len x num_heads x head_size - * value: batch_size x kv_padded_seq_len x num_heads x head_size - * scale: Attention scale factor - * custom_mask: custom attention mask; good place to inject a causal - attention mask - * q_seq_lens: list of unpadded query seq_lens for each batch index - * kv_seq_lens: list of unpadded key/value seq_lens for each batch index + query: batch_size x q_padded_seq_len x num_heads x head_size + key: batch_size x kv_padded_seq_len x num_heads x head_size + value: batch_size x kv_padded_seq_len x num_heads x head_size + scale: Attention scale factor + custom_mask: custom attention mask; good place to inject a causal attention mask + q_seq_lens: list of unpadded query seq_lens for each batch index + kv_seq_lens: list of unpadded key/value seq_lens for each batch index Returns: + Attention result, batch_size x q_padded_seq_len x num_heads x head_size - * Attention result, batch_size x q_padded_seq_len x num_heads x head_size """ - assert q_seq_lens is not None assert kv_seq_lens is not None @@ -346,8 +329,7 @@ def make_qkv( attn_type: AttentionType = AttentionType.ENCODER_DECODER, force_max_len: bool = False, ) -> tuple[QKVInputs, QKVInputs, QKVInputs]: - """ - Construct QKV test tensors for self- and cross-attention. + """Construct QKV test tensors for self- and cross-attention. Generates three query/key/value triplets: @@ -361,30 +343,28 @@ def make_qkv( seqlens Arguments: - - * batch_size - * max_q_seq_len: max query seq len - * max_kv_seq_len: max key/value seq len - * num_heads - * head_size - * is_encoder_decoder_attn: if True, query seqlen may differ from - key/value seqlen (as is often the case for cross-attention); - o/w, query/key/value seqlens match at each batch index - (max_kv_seq_len is unused) - * force_kv_seq_lens: if not None, overrides kv sequence lengths - * attn_type: encoder, decoder self, or enc/dec cross attention - * force_max_len: if True, all query seqlens are max_q_seq_len; o/w query - seqlens are random in [2,max_q_seq_lens]. Same for key/value seqlens - and max_kv_seq_len, unless forced by is_encoder_decoder_attn=False - * device: CPU or CUDA device + batch_size: number of sequences in the batch + max_q_seq_len: max query seq len + max_kv_seq_len: max key/value seq len + num_heads: number of attention heads + head_size: head dimension + is_encoder_decoder_attn: if True, query seqlen may differ from + key/value seqlen (as is often the case for cross-attention); + o/w, query/key/value seqlens match at each batch index + (max_kv_seq_len is unused) + force_kv_seq_lens: if not None, overrides kv sequence lengths + attn_type: encoder, decoder self, or enc/dec cross attention + force_max_len: if True, all query seqlens are max_q_seq_len; o/w query + seqlens are random in [2,max_q_seq_lens]. Same for key/value seqlens + and max_kv_seq_len, unless forced by is_encoder_decoder_attn=False + device: CPU or CUDA device Returns: + Overall QKVInputs structure (containing full unpacked Q/K/V tensors) + Prefill QKVInputs structure (containing all but the last sequence offset) + Decode QKVInputs structure (containing all only the last sequence offset) - * Overall QKVInputs structure (containing full unpacked Q/K/V tensors) - * Prefill QKVInputs structure (containing all but the last sequence offset) - * Decode QKVInputs structure (containing all only the last sequence offset) """ - if force_max_len: q_seq_lens = [max_q_seq_len for _ in range(batch_size)] else: @@ -474,24 +454,21 @@ def make_qkv( def pack_tensor( unpacked_tensor: torch.Tensor, seq_lens: list[int], device: torch.device | str ) -> tuple[torch.Tensor, list[int]]: - """ - Pack a batch_size x padded_seq_len x num_heads x head_size tensor into an + """Pack a batch_size x padded_seq_len x num_heads x head_size tensor into an unpadded number_of_tokens x num_heads x head_size tensor, where number_of_tokens = sum(seq_lens) Arguments: + unpacked_tensor: batch_size x padded_seq_len x num_heads x head_size + seq_lens: list of token counts for each seq + device: CPU or CUDA device - * unpacked_tensor: batch_size x padded_seq_len x num_heads x head_size - * seq_lens: list of token counts for each seq - * device: CPU or CUDA device + Returns: + packed_tensor: number_of_tokens x num_heads x head_size + start_loc_list: start idx of each batch elt in packed_tensor; [0] + + list(itertools.accumulate(seq_lens)) - Returns - - * packed_tensor: number_of_tokens x num_heads x head_size - * start_loc_list: start idx of each batch elt in packed_tensor; [0] + - list(itertools.accumulate(seq_lens)) """ - num_tok = sum(seq_lens) num_heads = unpacked_tensor.shape[-2] head_size = unpacked_tensor.shape[-1] @@ -507,8 +484,7 @@ def pack_tensor( def pack_qkv(qkv: QKVInputs, device: torch.device | str) -> PackedQKVInputs: - """ - Individually pack each of Q, K and V, each with dimensions batch_size x + """Individually pack each of Q, K and V, each with dimensions batch_size x padded_seq_len x num_heads x head_size, into respective number_of_tokens x num_heads x head_size tensors. @@ -517,17 +493,15 @@ def pack_qkv(qkv: QKVInputs, device: torch.device | str) -> PackedQKVInputs: For K and V, number_of_tokens = sum(kv_seq_lens) Arguments: + qkv: Unpacked (batch_size x padded_seq_len x num_heads x head_size) + attention inputs + device: CPU or CUDA device - * qkv: Unpacked (batch_size x padded_seq_len x num_heads x head_size) - attention inputs - * device: CPU or CUDA device + Returns: + Packed (number_of_tokens x num_heads x head_size) QKV inputs + derived from unpacked inputs - Returns - - * Packed (number_of_tokens x num_heads x head_size) QKV inputs - derived from unpacked inputs """ - if qkv.query is None: packed_query = None q_start_loc_list = None @@ -563,26 +537,24 @@ def _make_metadata_tensors( torch.Tensor, int | None, ]: - """ - Build scalar & tensor values required to build attention metadata structure. + """Build scalar & tensor values required to build attention metadata structure. Arguments: - - * seq_lens: list of token-counts for each decoder input seq - * context_lens: list of context length values for each seq - * encoder_seq_lens: list of token-counts for each encoder input seq - * device: CPU or CUDA device + seq_lens: list of token-counts for each decoder input seq + context_lens: list of context length values for each seq + encoder_seq_lens: list of token-counts for each encoder input seq + device: CPU or CUDA device Returns: + seq_lens_tensor: decoder seq_lens list, as tensor + context_lens_tensor: context_lens list, as tensor + max_context_len: max(context_lens) + max_seq_len: max(seq_lens) + seq_start_loc: start idx of each sequence + encoder_seq_lens_tensor: encoder seq_lens list, as tensor + encoder_seq_start_loc: start idx of each encoder sequence + max_encoder_seq_len: encoder seq_lens list, as tensor - * seq_lens_tensor: decoder seq_lens list, as tensor - * context_lens_tensor: context_lens list, as tensor - * max_context_len: max(context_lens) - * max_seq_len: max(seq_lens) - * seq_start_loc: start idx of each sequence - * encoder_seq_lens_tensor: encoder seq_lens list, as tensor - * encoder_seq_start_loc: start idx of each encoder sequence - * max_encoder_seq_len: encoder seq_lens list, as tensor """ seq_lens_tensor = maybe_make_int_tensor(seq_lens, device) context_lens_tensor = maybe_make_int_tensor(context_lens, device) @@ -637,22 +609,21 @@ def make_kv_cache( backend: str, default_val: float = 0.0, ) -> torch.Tensor: - """ - Create a fake KV cache. + """Create a fake KV cache. Arguments: - - * num_blocks: number of blocks in the KV cache - * num_heads: number of attention heads - * head_size: head dimension - * block_size: number of offsets within a block - * device: CPU or CUDA device - * default_val: initialization value for KV cache elements + num_blocks: number of blocks in the KV cache + num_heads: number of attention heads + head_size: head dimension + block_size: number of offsets within a block + device: CPU or CUDA device + backend: attention backend name; only 'FLASH_ATTN' is supported + default_val: initialization value for KV cache elements Returns: + kv_cache: 2 x num_blocks x block_size x num_heads x head_size + for backend 'FLASH_ATTN' - * kv_cache: 2 x num_blocks x block_size x num_heads x head_size - * for backend 'FLASH_ATTN' """ if backend != "FLASH_ATTN": raise ValueError(f"Unknown backend value: '{backend}'. Expected 'FLASH_ATTN'.") @@ -663,8 +634,7 @@ def make_kv_cache( def _num_tokens_to_min_blocks(num_tokens: int, block_size: int) -> int: - """ - Compute the minimum number of blocks required to hold num_tokens tokens, + """Compute the minimum number of blocks required to hold num_tokens tokens, given block_size """ return (num_tokens + block_size - 1) // block_size @@ -683,8 +653,7 @@ def split_slot_mapping( seq_lens: list[int], device: torch.device | str, ): - """ - Split a slot mapping into valid prefill- and decode-phase slot mappings. + """Split a slot mapping into valid prefill- and decode-phase slot mappings. Context: * Your goal is to test (1) prefill of N prompts, with prompt-lengths @@ -708,21 +677,19 @@ def split_slot_mapping( The N excised entries are appended to obtain the decode-phase slot mapping Arguments: - - * slot_mapping_list: Length-P 1D slot mapping (as list) reflecting all N - post-decode sequences - * seq_lens: list of N post-decode sequence lengths (K_i + 1 in the - description above) - * device: cuda, cpu, etc. + slot_mapping_list: Length-P 1D slot mapping (as list) reflecting all N + post-decode sequences + seq_lens: list of N post-decode sequence lengths (K_i + 1 in the + description above) + device: cuda, cpu, etc. Returns: + prefill_slot_mapping: Length-M 1D slot mapping (as Tensor) + reflecting all N prefill prompts + decode_slot_mapping: Length-N 1D slot mapping (as Tensor) reflecting + all N decoded tokens - * prefill_slot_mapping: Length-M 1D slot mapping (as Tensor) - reflecting all N prefill prompts - * decode_slot_mapping: Length-N 1D slot mapping (as Tensor) reflecting - all N decoded tokens """ - prefill_slot_mapping = [] decode_slot_mapping = [] @@ -746,8 +713,7 @@ def make_block_tables_slot_mapping( device: torch.device | str, block_base_addr: int = 0, ) -> tuple[torch.Tensor, list[int], int]: - """ - Construct fake block tables & slot mappings. + """Construct fake block tables & slot mappings. For a sequence with num_tokens tokens the minimum number of required KV cache blocks is @@ -772,19 +738,17 @@ def make_block_tables_slot_mapping( i.e. the total of prefill prompt tokens + decoded tokens. Arguments: - - * block_size: number of offsets per block - * seq_lens: list of token-counts for each sequence - * block_base_addr: the block table base address - * device: CPU or CUDA device + block_size: number of offsets per block + seq_lens: list of token-counts for each sequence + block_base_addr: the block table base address + device: CPU or CUDA device Return: - * block_tables_tensor: block table for sequence * slot_mapping_list: slot mapping for sequence * max_block_idx: the highest block address within this block table - """ + """ # Provision minimum number of KV cache blocks num_blocks_list = [ _num_tokens_to_min_blocks(num_tokens, block_size) for num_tokens in seq_lens @@ -824,14 +788,14 @@ def make_block_tables_slot_mapping( def assert_actual_matches_ideal( test_params: PhaseTestParameters, output_under_test: torch.Tensor, backend: str ) -> None: - """ - Assert that observed output matches the ideal output + """Assert that observed output matches the ideal output contained in the test parameters data structure. Arguments: + test_params: Test parameters including packed ideal output + output_under_test: actually observed output value + backend: attention backend name; only 'FLASH_ATTN' is supported - * test_params: Test parameters including packed ideal output - * output_under_test: actually observed output value """ ideal_output = test_params.packed_qkvo.ideal_output if backend != "FLASH_ATTN": @@ -852,9 +816,7 @@ def fp8_allclose( atol: float = 1e-08, equal_nan: bool = False, ) -> bool: - """ - Reference implementation of torch.allclose - """ + """Reference implementation of torch.allclose.""" torch._refs._check_close_args(name="torch.allclose", a=a, b=b, rtol=rtol, atol=atol) return bool( diff --git a/tests/lora/conftest.py b/tests/lora/conftest.py index 2c64fb2245..f5ca44f174 100644 --- a/tests/lora/conftest.py +++ b/tests/lora/conftest.py @@ -33,7 +33,6 @@ def should_do_global_cleanup_after_test(request) -> bool: This can provide a ~10x speedup for non-GPU unit tests since they don't need to initialize torch. """ - return not request.node.get_closest_marker("skip_global_cleanup") @@ -270,6 +269,30 @@ def qwen3_woofing_lora_files(): return hf_api().snapshot_download(repo_id="Jackmin108/Qwen3-0.6B-Woof-LoRA") +@pytest.fixture(scope="session") +def qwen3_guard_star_trek_lora_files(): + return hf_api().snapshot_download( + repo_id="geoffmunn/Qwen3Guard-StarTrek-Classification-0.6B", + allow_patterns=["adapter_config.json", "adapter_model.safetensors"], + ) + + +@pytest.fixture(scope="session") +def qwen3_guard_new_zealand_lora_files(): + return hf_api().snapshot_download( + repo_id="geoffmunn/Qwen3Guard-NewZealand-Classification-0.6B", + allow_patterns=["adapter_config.json", "adapter_model.safetensors"], + ) + + +@pytest.fixture(scope="session") +def skywork_qwen3_reward_lora_files(): + return hf_api().snapshot_download( + repo_id="AmirMohseni/skywork-qwen3-0.6b-reward-lora", + allow_patterns=["adapter_config.json", "adapter_model.safetensors"], + ) + + @pytest.fixture(scope="session") def deepseekv2_lora_files(): return hf_api().snapshot_download(repo_id="wuchen01/DeepSeek-V2-Lite-Chat-All-LoRA") @@ -343,8 +366,7 @@ def qwen36_moe_3d_lora_files(): @pytest.fixture def reset_default_device(): - """ - Some tests, such as `test_punica_ops.py`, explicitly set the + """Some tests, such as `test_punica_ops.py`, explicitly set the default device, which can affect subsequent tests. Adding this fixture helps avoid this problem. """ diff --git a/tests/lora/test_add_lora.py b/tests/lora/test_add_lora.py index 628f5bc28e..97c37c1895 100644 --- a/tests/lora/test_add_lora.py +++ b/tests/lora/test_add_lora.py @@ -55,8 +55,7 @@ async def requests_processing_time(llm, lora_requests: list[LoRARequest]) -> flo @pytest.mark.asyncio async def test_add_lora(chatglm3_lora_files): - """ - The add_lora function is used to preload some LoRA adapters into the + """The add_lora function is used to preload some LoRA adapters into the engine in anticipation of future requests using these adapters. To test this functionality, we use the async engine to process some requests - We do it twice, once with add_lora() preloading and once without. diff --git a/tests/lora/test_default_mm_loras.py b/tests/lora/test_default_mm_loras.py index 5e3c762f79..e858331c93 100644 --- a/tests/lora/test_default_mm_loras.py +++ b/tests/lora/test_default_mm_loras.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for applying default registered multimodal loras. -""" +"""Tests for applying default registered multimodal loras.""" import os import unittest.mock as mock diff --git a/tests/lora/test_fused_moe_lora_kernel.py b/tests/lora/test_fused_moe_lora_kernel.py index 444b609492..8b5b3d577a 100644 --- a/tests/lora/test_fused_moe_lora_kernel.py +++ b/tests/lora/test_fused_moe_lora_kernel.py @@ -41,8 +41,7 @@ def CEILDIV(x, y): def assign_loras_to_tokens(num_tokens: int, num_sequences: int, max_loras: int): - """ - Split `num_tokens` into `num_sequences` sequences. + """Split `num_tokens` into `num_sequences` sequences. Each sequence randomly selects 1 LoRA index from [0, max_loras), and all tokens in that sequence are assigned this LoRA index. @@ -54,6 +53,7 @@ def assign_loras_to_tokens(num_tokens: int, num_sequences: int, max_loras: int): Returns: torch.Tensor: 1D tensor of shape [num_tokens], where each value is the LoRA index assigned to that token. + """ assert num_sequences > 0 and max_loras > 0 assert num_tokens >= num_sequences, "num_tokens must be >= num_sequences" @@ -81,9 +81,8 @@ def assign_loras_to_tokens(num_tokens: int, num_sequences: int, max_loras: int): def assign_experts_to_tokens(num_tokens: int, num_experts: int, top_k_num: int): - """ - For each token, randomly select `top_k_num` distinct experts out of `num_experts`, - and assign normalized random weights that sum to 1. + """For each token, randomly select `top_k_num` distinct experts out of + `num_experts`, and assign normalized random weights that sum to 1. Args: num_tokens (int): Total number of tokens. @@ -95,6 +94,7 @@ def assign_experts_to_tokens(num_tokens: int, num_experts: int, top_k_num: int): expert index for each token. expert_weights (torch.Tensor): shape [num_tokens, top_k_num], normalized weights (sum = 1 per row). + """ assert top_k_num <= num_experts, "top_k_num must be <= num_experts" @@ -379,8 +379,7 @@ def use_fused_moe_lora_kernel_naive( offset=0, add_inputs=True, ): - """ - Test helper for naive_block_assignment path. + """Test helper for naive_block_assignment path. Skips moe_lora_align_block_size and uses flattened topk_ids as expert_ids. """ config = { @@ -472,8 +471,7 @@ def test_fused_moe_lora_kernel_naive_block_assignment( device, seed, ): - """ - Test the naive_block_assignment path of the fused_moe_lora kernel. + """Test the naive_block_assignment path of the fused_moe_lora kernel. This path is triggered when batch_size * top_k is much smaller than num_experts * max_loras, and skips the moe_lora_align_block_size kernel. """ diff --git a/tests/lora/test_layers.py b/tests/lora/test_layers.py index 2dab86bc0d..e6f84cf0bc 100644 --- a/tests/lora/test_layers.py +++ b/tests/lora/test_layers.py @@ -126,8 +126,7 @@ def clean_cache_reset_device(reset_default_device): @pytest.fixture(autouse=True) def skip_cuda_with_stage_false(request): - """ - On cuda-like platforms, we use the same kernels for prefill and decode + """On cuda-like platforms, we use the same kernels for prefill and decode stage, and 'stage' is generally ignored, so we only need to test once. """ if current_platform.is_cuda_alike() or current_platform.is_xpu(): @@ -153,8 +152,8 @@ def get_random_id_to_index( num_slots: The number of slots in the mapping. Must be larger than num_loras. log: Whether to log the output. - """ + """ if num_loras > num_slots: raise ValueError( f"num_loras is higher than num_slots: {num_loras} > {num_slots}. " @@ -190,8 +189,8 @@ def populate_loras( repeats: must only be set for column parallel packed layers. Indicates the number of loras to compose together to create a single lora layer. - """ + """ # Dictionary that maps the lora ID to the # corresponding lora weights. lora_dict: dict[int, LoRALayerWeights] = dict() @@ -246,8 +245,8 @@ def create_random_inputs( input_range: the range of values to include in the input. input_range[0] <= possible input values < input_range[1] input_type: the type of values in the input. - """ + """ low, high = input_range inputs: list[torch.Tensor] = [] diff --git a/tests/lora/test_lora_functions.py b/tests/lora/test_lora_functions.py index ecf2ad9ee6..1b841b70ce 100644 --- a/tests/lora/test_lora_functions.py +++ b/tests/lora/test_lora_functions.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Script to test add_lora, remove_lora, pin_lora, list_loras functions. -""" +"""Script to test add_lora, remove_lora, pin_lora, list_loras functions.""" import pytest diff --git a/tests/lora/test_mixtral.py b/tests/lora/test_mixtral.py index 3868bff796..a8f1fdda64 100644 --- a/tests/lora/test_mixtral.py +++ b/tests/lora/test_mixtral.py @@ -32,7 +32,7 @@ def do_sample( @pytest.mark.parametrize("tp_size", [4]) def test_mixtral_lora(mixtral_lora_files, tp_size): - """Original test, the LoRA model has the common target modules, not all""" + """Original test, the LoRA model has the common target modules, not all.""" if ( torch.accelerator.device_count() < tp_size and tp_size > 1 diff --git a/tests/lora/test_moe_lora_ep_load.py b/tests/lora/test_moe_lora_ep_load.py index ead40b2552..60d0c2c9ba 100644 --- a/tests/lora/test_moe_lora_ep_load.py +++ b/tests/lora/test_moe_lora_ep_load.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -End-to-end correctness test for 2D MoE LoRA expert-parallel +"""End-to-end correctness test for 2D MoE LoRA expert-parallel load-time slicing """ diff --git a/tests/lora/test_peft_helper.py b/tests/lora/test_peft_helper.py index 23a8aad1a5..9fb52ea727 100644 --- a/tests/lora/test_peft_helper.py +++ b/tests/lora/test_peft_helper.py @@ -20,7 +20,7 @@ ERROR_CASES = [ ( "test_modules_to_save", {"modules_to_save": ["lm_head"]}, - "only supports modules_to_save being None", + "Unsupported modules_to_save", ), ("test_rank_zero", {"r": 0}, "must be a positive integer"), ("test_rank_negative", {"r": -8}, "must be a positive integer"), diff --git a/tests/lora/test_punica_ops.py b/tests/lora/test_punica_ops.py index 9e1f99b3f6..34cbe061dc 100644 --- a/tests/lora/test_punica_ops.py +++ b/tests/lora/test_punica_ops.py @@ -160,8 +160,7 @@ def check_lora_shrink_kernel( seq_length: int, scaling: float, ): - """ - Compare outputs of torch_ops.sgmv_shrink and triton_ops.lora_shrink + """Compare outputs of torch_ops.sgmv_shrink and triton_ops.lora_shrink kernels. """ data: PunicaTensors = generate_data_for_nslices( @@ -234,8 +233,7 @@ def check_lora_expand_kernel( seq_length: int, add_inputs: bool, ): - """ - Compare outputs of torch_ops.sgmv_expand and triton_ops.lora_expand + """Compare outputs of torch_ops.sgmv_expand and triton_ops.lora_expand kernels. """ data: PunicaTensors = generate_data_for_nslices( @@ -438,9 +436,7 @@ def test_kernels( seed: int, op_type: str, ): - """ - Tests LoRA kernels. - """ + """Tests LoRA kernels.""" torch.set_default_device(device) torch.accelerator.set_device_index(device) set_random_seed(seed) @@ -491,9 +487,7 @@ def test_kernels_hidden_size( seed: int, op_type: str, ): - """ - Tests SGMV and LoRA kernels. - """ + """Tests SGMV and LoRA kernels.""" torch.set_default_device(device) torch.accelerator.set_device_index(device) set_random_seed(seed) @@ -526,8 +520,7 @@ def test_kernels_hidden_size( @pytest.mark.parametrize("device", DEVICES) def test_add_lora_fused_moe_early_exit(device): - """ - Ensures add_lora_fused_moe does not invoke the LoRA kernel or + """Ensures add_lora_fused_moe does not invoke the LoRA kernel or modify the output tensor when no_lora_flag_cpu is True """ from types import SimpleNamespace diff --git a/tests/lora/test_punica_xpu_ops.py b/tests/lora/test_punica_xpu_ops.py index 585c97cfa5..36966809eb 100644 --- a/tests/lora/test_punica_xpu_ops.py +++ b/tests/lora/test_punica_xpu_ops.py @@ -86,9 +86,7 @@ def check_bgmv_shrink( device: str, scaling: float, ): - """ - Compare vllm.bgmv_shrink against a reference implementation. - """ + """Compare vllm.bgmv_shrink against a reference implementation.""" seq_length = 1 data: PunicaTensors = generate_data( batches, @@ -130,9 +128,7 @@ def check_bgmv_expand( device: str, add_inputs: bool, ): - """ - Compare vllm.bgmv_expand against a reference implementation. - """ + """Compare vllm.bgmv_expand against a reference implementation.""" seq_length = 1 data: PunicaTensors = generate_data( batches, @@ -172,9 +168,7 @@ def check_bgmv_expand_slice( device: str, add_inputs: bool, ): - """ - Compare vllm.bgmv_expand_slice against a reference implementation. - """ + """Compare vllm.bgmv_expand_slice against a reference implementation.""" seq_length = 1 data: PunicaTensors = generate_data_for_expand_nslices( batches, diff --git a/tests/lora/test_qwen3_unembed.py b/tests/lora/test_qwen3_unembed.py index 4064b443ac..a70df902f7 100644 --- a/tests/lora/test_qwen3_unembed.py +++ b/tests/lora/test_qwen3_unembed.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for Qwen3 unembed LoRA support. +"""Tests for Qwen3 unembed LoRA support. This test creates synthetic LoRA weights that include lm_head (output embedding) to verify that Qwen3 properly supports LoRA on the unembed/lm_head layer. diff --git a/tests/lora/test_qwen3_with_multi_loras.py b/tests/lora/test_qwen3_with_multi_loras.py index 9870da6959..185ab91c7a 100644 --- a/tests/lora/test_qwen3_with_multi_loras.py +++ b/tests/lora/test_qwen3_with_multi_loras.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This script contains: +"""This script contains: 1. test multi loras service with tp >= 2 2. test multi loras request """ @@ -125,8 +124,7 @@ def test_multi_loras_with_tp_sync(): return output_text def reload_lora(name: str): - """ - reload a lora to simulate the case: + """Reload a lora to simulate the case: setting `VLLM_ALLOW_RUNTIME_LORA_UPDATING=true` for dynamic lora loading and unloading """ @@ -205,8 +203,7 @@ def test_multiple_lora_requests(): def test_load_inplace_offline_reload( qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str ) -> None: - """ - Test that load_inplace=True allows reloading LoRA adapters with the same ID + """Test that load_inplace=True allows reloading LoRA adapters with the same ID in offline mode (using LLM class directly). """ llm = LLM( @@ -255,8 +252,7 @@ def test_load_inplace_offline_reload( def test_load_inplace_false_no_reload( qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str ) -> None: - """ - Test that load_inplace=False prevents reloading when an adapter + """Test that load_inplace=False prevents reloading when an adapter with the same ID already exists. """ llm = LLM( diff --git a/tests/lora/test_qwenvl.py b/tests/lora/test_qwenvl.py index 3362aa46ed..063de2b587 100644 --- a/tests/lora/test_qwenvl.py +++ b/tests/lora/test_qwenvl.py @@ -46,7 +46,7 @@ class TestConfig: class Qwen2VLTester: - """Test helper for Qwen2 VL models with LoRA""" + """Test helper for Qwen2 VL models with LoRA.""" PROMPT_TEMPLATE = ( "<|im_start|>system\nYou are a helpful assistant.<|im_end|>" @@ -199,7 +199,7 @@ def _enable_deterministic_lora_shrink(monkeypatch: pytest.MonkeyPatch) -> None: def test_qwen2vl_lora(qwen2vl_lora_files): - """Test Qwen 2.0 VL model with LoRA""" + """Test Qwen 2.0 VL model with LoRA.""" config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files) with Qwen2VLTester(config) as tester: # Test with different LoRA IDs @@ -230,7 +230,7 @@ def test_qwen2vl_lora_beam_search(qwen2vl_lora_files): current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests" ) def test_qwen25vl_lora(qwen25vl_lora_files): - """Test Qwen 2.5 VL model with LoRA""" + """Test Qwen 2.5 VL model with LoRA.""" config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files) with Qwen2VLTester(config) as tester: # Test with different LoRA IDs @@ -292,8 +292,7 @@ def test_qwen2vl_multiple_lora_types( qwen2vl_vision_tower_lora_files, monkeypatch: pytest.MonkeyPatch, ): - """ - Test multiple LoRA adapter types (language, vision tower + connector, + """Test multiple LoRA adapter types (language, vision tower + connector, vision tower only) using the same LLM instance to verify mm_encoder_cache behavior with different LoRA requests. diff --git a/tests/lora/test_sequence_classification.py b/tests/lora/test_sequence_classification.py new file mode 100644 index 0000000000..86721bfb6e --- /dev/null +++ b/tests/lora/test_sequence_classification.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from tests.utils import multi_gpu_test +from vllm import LLM, PoolingParams +from vllm.distributed import cleanup_dist_env_and_memory +from vllm.lora.request import LoRARequest + +MODEL_NAME = "Qwen/Qwen3-0.6B" +NATIVE_MODEL_NAME = "Skywork/Skywork-Reward-V2-Qwen3-0.6B" +PROMPTS = { + "star_trek": "Does warp drive appear in Star Trek?", + "new_zealand": "Wellington is the capital of New Zealand.", +} +EXPECTED_OUT = { + "star_trek": [13.6640625, -13.6640625], + "new_zealand": [-4.80078125, 8.6171875], + "native": [-1.52734375], +} + + +def _classify_logits( + llm: LLM, + prompts: list[str], + requests: LoRARequest | list[LoRARequest], +) -> torch.Tensor: + params = [PoolingParams(task="classify", use_activation=False) for _ in prompts] + outputs = llm.classify( + prompts, + pooling_params=params, + lora_request=requests, + use_tqdm=False, + ) + return torch.tensor([output.outputs.probs for output in outputs]) + + +def test_converted_model_with_modules_to_save( + qwen3_guard_star_trek_lora_files: str, +) -> None: + prompt = PROMPTS["star_trek"] + llm = LLM( + model=MODEL_NAME, + runner="pooling", + convert="classify", + hf_overrides={"num_labels": 2}, + dtype="float16", + enable_lora=True, + max_lora_rank=16, + enforce_eager=True, + max_model_len=512, + gpu_memory_utilization=0.5, + ) + actual = _classify_logits( + llm, + [prompt], + LoRARequest("star-trek", 1, qwen3_guard_star_trek_lora_files), + )[0] + + torch.testing.assert_close( + actual, + torch.tensor(EXPECTED_OUT["star_trek"]), + atol=2e-2, + rtol=2e-2, + ) + + del llm + cleanup_dist_env_and_memory() + + +def test_native_classification_model_with_modules_to_save( + skywork_qwen3_reward_lora_files: str, +) -> None: + prompt = "Which response is more helpful and correct?" + llm = LLM( + model=NATIVE_MODEL_NAME, + runner="pooling", + dtype="float16", + enable_lora=True, + max_lora_rank=8, + enforce_eager=True, + max_model_len=512, + gpu_memory_utilization=0.5, + ) + actual = _classify_logits( + llm, + [prompt], + LoRARequest("native", 1, skywork_qwen3_reward_lora_files), + )[0] + + torch.testing.assert_close( + actual, + torch.tensor(EXPECTED_OUT["native"]), + atol=2e-2, + rtol=2e-2, + ) + + del llm + cleanup_dist_env_and_memory() + + +def _test_batched_loras( + qwen3_guard_star_trek_lora_files: str, + qwen3_guard_new_zealand_lora_files: str, + tp_size: int, +) -> None: + names = ["star_trek", "new_zealand"] + prompts = [PROMPTS[name] for name in names] + adapter_paths = [ + qwen3_guard_star_trek_lora_files, + qwen3_guard_new_zealand_lora_files, + ] + requests = [ + LoRARequest(name, index, adapter_path) + for index, (name, adapter_path) in enumerate(zip(names, adapter_paths), start=1) + ] + + llm = LLM( + model=MODEL_NAME, + runner="pooling", + convert="classify", + hf_overrides={"num_labels": 2}, + dtype="float16", + enable_lora=True, + max_loras=2, + max_cpu_loras=2, + max_lora_rank=16, + enforce_eager=True, + max_model_len=512, + gpu_memory_utilization=0.5, + tensor_parallel_size=tp_size, + ) + actual = _classify_logits(llm, prompts, requests) + + torch.testing.assert_close( + actual, + torch.tensor([EXPECTED_OUT[name] for name in names]), + atol=2e-2, + rtol=2e-2, + ) + + del llm + cleanup_dist_env_and_memory() + + +def test_batched_loras( + qwen3_guard_star_trek_lora_files: str, + qwen3_guard_new_zealand_lora_files: str, +) -> None: + _test_batched_loras( + qwen3_guard_star_trek_lora_files, + qwen3_guard_new_zealand_lora_files, + tp_size=1, + ) + + +@multi_gpu_test(num_gpus=2) +def test_batched_loras_tp( + qwen3_guard_star_trek_lora_files: str, + qwen3_guard_new_zealand_lora_files: str, +) -> None: + _test_batched_loras( + qwen3_guard_star_trek_lora_files, + qwen3_guard_new_zealand_lora_files, + tp_size=2, + ) diff --git a/tests/lora/test_whisper.py b/tests/lora/test_whisper.py index 6f1a894cf9..73cc550b29 100644 --- a/tests/lora/test_whisper.py +++ b/tests/lora/test_whisper.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Integration tests for Whisper models with LoRA adapters. +"""Integration tests for Whisper models with LoRA adapters. These tests verify that Whisper models can correctly load and use LoRA adapters for speech-to-text transcription tasks. diff --git a/tests/lora/utils.py b/tests/lora/utils.py index e5ce7a8846..8ee2f8af82 100644 --- a/tests/lora/utils.py +++ b/tests/lora/utils.py @@ -116,8 +116,7 @@ class PunicaTensors: token_lora_mapping: torch.Tensor def meta(self) -> tuple[int, int]: - """ - Infer max_seq_length and token_nums from the tensors + """Infer max_seq_length and token_nums from the tensors and return them. """ max_seq_length = self.seq_len_tensor.max() diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 507fd93e9d..08fb6818e3 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -808,7 +808,6 @@ def test_models_fse_init( monkeypatch: pytest.MonkeyPatch, ) -> None: """Model construction resolves FSE consistently with Quark quantization.""" - quantization_config: dict[str, Any] = ( get_deepseek_v4_quark_config(["layers.0.ffn.shared_experts"] if exclude else []) if model_type == "deepseek_v4" diff --git a/tests/model_executor/test_deep_gemm_warmup.py b/tests/model_executor/test_deep_gemm_warmup.py index 6f2a31cf27..e71061e733 100644 --- a/tests/model_executor/test_deep_gemm_warmup.py +++ b/tests/model_executor/test_deep_gemm_warmup.py @@ -71,7 +71,7 @@ def test_unstamped_and_mismatched_block_layers_are_skipped(monkeypatch) -> None: @pytest.mark.parametrize("is_bmm", [False, True]) def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None: - """bmm layers bypass the kernel at runtime, so they must not be warmed.""" + """Bmm layers bypass the kernel at runtime, so they must not be warmed.""" kernel = object.__new__(DeepGemmFp8BlockScaledMMKernel) kernel.is_deep_gemm_supported = False layer = torch.nn.Module() diff --git a/tests/model_executor/test_model_load_with_params.py b/tests/model_executor/test_model_load_with_params.py index 785335e991..7d1bddab0d 100644 --- a/tests/model_executor/test_model_load_with_params.py +++ b/tests/model_executor/test_model_load_with_params.py @@ -23,9 +23,7 @@ REVISION_ROBERTA = os.environ.get("REVISION", "main") current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." ) def test_model_loading_with_params(vllm_runner, monkeypatch): - """ - Test parameter weight loading with tp>1. - """ + """Test parameter weight loading with tp>1.""" # to use apply_model monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") with vllm_runner( @@ -68,9 +66,7 @@ def test_model_loading_with_params(vllm_runner, monkeypatch): current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." ) def test_roberta_model_loading_with_params(vllm_runner, monkeypatch): - """ - Test parameter weight loading with tp>1. - """ + """Test parameter weight loading with tp>1.""" # to use apply_model monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") with vllm_runner( @@ -113,9 +109,7 @@ def test_roberta_model_loading_with_params(vllm_runner, monkeypatch): current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm." ) def test_facebook_roberta_model_loading_with_params(vllm_runner, monkeypatch): - """ - Test loading roberta-base model with no lm_head. - """ + """Test loading roberta-base model with no lm_head.""" # to use apply_model monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") model_name = "FacebookAI/roberta-base" diff --git a/tests/model_executor/test_oink_integration.py b/tests/model_executor/test_oink_integration.py index 2f37472b73..b16d716413 100644 --- a/tests/model_executor/test_oink_integration.py +++ b/tests/model_executor/test_oink_integration.py @@ -62,7 +62,6 @@ def test_oink_availability_checks( expected_fused: bool, ): """Test OINK support detection with clean import state for each parameter set.""" - # Use spawn to run function in fresh process with clean imports # TODO migrate to spawn utility: # https://github.com/vllm-project/vllm/issues/41415 diff --git a/tests/model_executor/test_qwen3_omni.py b/tests/model_executor/test_qwen3_omni.py index 30d9909a5a..cf3df85caa 100644 --- a/tests/model_executor/test_qwen3_omni.py +++ b/tests/model_executor/test_qwen3_omni.py @@ -16,8 +16,7 @@ from vllm.multimodal.processing import InputProcessingContext # Helper function to print input IDs with coalesced audio/video tokens. def print_input_ids(input_ids): - """ - Print input IDs, compressing consecutive special tokens. + """Print input IDs, compressing consecutive special tokens. - 151675: <|audio_pad|> - 151656: <|video_pad|> """ @@ -137,7 +136,6 @@ def test_qwen3_omni_get_updates_use_audio_in_video( mock_image_processor, ): """Test the get_updates_use_audio_in_video method directly.""" - from vllm.model_executor.models.qwen3_omni_moe_thinker import ( Qwen3OmniMoeThinkerMultiModalProcessor, Qwen3OmniMoeThinkerProcessingInfo, diff --git a/tests/model_executor/test_qwen3_vl_mrope.py b/tests/model_executor/test_qwen3_vl_mrope.py index 90d9fd6e4f..a50bf319c9 100644 --- a/tests/model_executor/test_qwen3_vl_mrope.py +++ b/tests/model_executor/test_qwen3_vl_mrope.py @@ -51,8 +51,7 @@ class DummyConfig: def make_video_embedding( t, h, w, interleave_text_tokens: tuple[int, int], video_pruning_rate: float = 0.0 ): - """ - Helper function to make a video embedding for a given video size and pruning rate. + """Make a video embedding for a given video size and pruning rate. Args: t: Number of frames. @@ -64,6 +63,7 @@ def make_video_embedding( Returns: Tuple of (unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask) + """ unpruned_tokens_sequence = [] population = list(range(1, 100)) diff --git a/tests/model_executor/test_routed_experts_capture.py b/tests/model_executor/test_routed_experts_capture.py index 49986a991e..b146a235c1 100644 --- a/tests/model_executor/test_routed_experts_capture.py +++ b/tests/model_executor/test_routed_experts_capture.py @@ -278,7 +278,7 @@ def test_routed_experts_capturer_single_dp_no_metadata(): def test_routed_experts_capturer_dp_naive_concatenated_all_ranks(): - """n == sum(num_tokens_dp): slice this rank's segment from concatenated topk.""" + """N == sum(num_tokens_dp): slice this rank's segment from concatenated topk.""" capturer = _capturer_with_buffer(dp_rank=1) num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32) ctx = SimpleNamespace( @@ -295,7 +295,7 @@ def test_routed_experts_capturer_dp_naive_concatenated_all_ranks(): def test_routed_experts_capturer_dp_modular_local_tokens(): - """n == token_num_per_dp: topk is already local to this DP rank.""" + """N == token_num_per_dp: topk is already local to this DP rank.""" capturer = _capturer_with_buffer(dp_rank=1) num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32) ctx = SimpleNamespace( diff --git a/tests/model_executor/test_weight_utils.py b/tests/model_executor/test_weight_utils.py index e577690c92..3dea587075 100644 --- a/tests/model_executor/test_weight_utils.py +++ b/tests/model_executor/test_weight_utils.py @@ -72,28 +72,28 @@ class TestMaybeRemapKvScaleName: assert result == "model.layers.0.self_attn.attn.v_scale" def test_modelopt_k_proj_k_scale(self): - """ModelOpt format: k_proj.k_scale -> attn.k_scale""" + """ModelOpt format: k_proj.k_scale -> attn.k_scale.""" result = maybe_remap_kv_scale_name( "model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT ) assert result == "model.layers.0.self_attn.attn.k_scale" def test_modelopt_v_proj_v_scale(self): - """ModelOpt format: v_proj.v_scale -> attn.v_scale""" + """ModelOpt format: v_proj.v_scale -> attn.v_scale.""" result = maybe_remap_kv_scale_name( "model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT ) assert result == "model.layers.0.self_attn.attn.v_scale" def test_deprecated_kv_scale(self): - """Old format: kv_scale -> attn.k_scale (deprecated)""" + """Old format: kv_scale -> attn.k_scale (deprecated).""" result = maybe_remap_kv_scale_name( "model.layers.0.self_attn.kv_scale", self.PARAMS_DICT ) assert result == "model.layers.0.self_attn.attn.k_scale" def test_default_bare_k_scale(self): - """Default format: .k_scale -> .attn.k_scale""" + """Default format: .k_scale -> .attn.k_scale.""" result = maybe_remap_kv_scale_name( "model.layers.0.self_attn.k_scale", self.PARAMS_DICT ) diff --git a/tests/models/kimi_k3/test_kda_metadata.py b/tests/models/kimi_k3/test_kda_metadata.py index c3f423804a..fa810217b1 100644 --- a/tests/models/kimi_k3/test_kda_metadata.py +++ b/tests/models/kimi_k3/test_kda_metadata.py @@ -934,3 +934,91 @@ def test_aligned_block_table_matches_shared_gdn(): ) torch.testing.assert_close(actual, expected) + + +def _build_non_spec(batch, is_prefilling, full_cuda_graph=False): + common_attn_metadata = create_common_attn_metadata( + batch, BLOCK_SIZE, DEVICE + ).replace( + is_prefilling=None + if is_prefilling is None + else torch.tensor(is_prefilling, dtype=torch.bool) + ) + builder = _make_builder( + KimiK3KDAMetadataBuilder, + num_speculative_tokens=0, + full_cuda_graph=full_cuda_graph, + ) + return builder, common_attn_metadata, builder.build(0, common_attn_metadata) + + +def test_one_token_first_chunk_excludes_padding(): + """Neither padding requests nor padding tokens count as prefill work.""" + common = create_common_attn_metadata( + BatchSpec(seq_lens=[100, 1, 0, 0], query_lens=[1, 1, 0, 0]), + BLOCK_SIZE, + DEVICE, + ).replace( + is_prefilling=torch.tensor([False, True, False, False], dtype=torch.bool), + num_actual_tokens=4, + ) + builder = _make_builder( + KimiK3KDAMetadataBuilder, num_speculative_tokens=0, full_cuda_graph=False + ) + actual = builder.build(0, common) + + assert actual.num_decodes == 1 + assert actual.num_prefills == 1 + assert actual.num_decode_tokens == 1 + assert actual.num_prefill_tokens == 1 + + +@pytest.mark.parametrize( + ("seq_len", "query_len", "is_prefilling", "num_prefills"), + [ + pytest.param(1, 1, True, 1, id="first-chunk"), + pytest.param(65, 1, True, 0, id="resumed-chunk"), + pytest.param(0, 0, True, 0, id="padding"), + pytest.param(1, 1, None, 0, id="missing-prefill-flag"), + ], +) +def test_one_token_chunk_classification( + seq_len, query_len, is_prefilling, num_prefills +): + """Only a real first chunk with a prefill flag needs state initialization.""" + _, _, actual = _build_non_spec( + BatchSpec(seq_lens=[100, seq_len], query_lens=[1, query_len]), + is_prefilling=None if is_prefilling is None else [False, is_prefilling], + ) + + assert actual.num_prefills == num_prefills + assert actual.num_decodes == 2 - num_prefills + assert actual.num_prefill_tokens == num_prefills + assert actual.num_decode_tokens == 1 + query_len - num_prefills + if num_prefills: + assert actual.has_initial_state is not None + assert actual.has_initial_state.tolist() == [True, False] + else: + assert actual.has_initial_state is None + + +def test_cudagraph_capture_batch_stays_decode_only(): + """Capture rows have no history, but must still select decode kernels.""" + batch = BatchSpec(seq_lens=[1] * 4, query_lens=[1] * 4) + common_attn_metadata = create_common_attn_metadata( + batch, BLOCK_SIZE, DEVICE + ).replace(is_prefilling=torch.zeros(4, dtype=torch.bool)) + builder = _make_builder( + KimiK3KDAMetadataBuilder, + num_speculative_tokens=0, + full_cuda_graph=True, + ) + actual = builder.build_for_cudagraph_capture(common_attn_metadata) + + assert actual.num_prefills == 0 + assert actual.num_decodes == 4 + assert actual.has_initial_state is None + staged = actual.non_spec_state_indices_tensor + assert staged is not None + assert staged.data_ptr() == builder.non_spec_state_indices_tensor.data_ptr() + torch.testing.assert_close(staged, common_attn_metadata.block_table_tensor[:, 0]) diff --git a/tests/models/language/generation/test_hybrid.py b/tests/models/language/generation/test_hybrid.py index c5406fae48..b77670956a 100644 --- a/tests/models/language/generation/test_hybrid.py +++ b/tests/models/language/generation/test_hybrid.py @@ -164,8 +164,7 @@ def test_chunked_prefill_with_parallel_sampling( max_tokens: int, conv_state_layout: str, ) -> None: - """ - Tests chunked prefill in conjunction with n > 1. + """Tests chunked prefill in conjunction with n > 1. In this case, prefill is populated with decoding tokens and we test that it doesn't fail. @@ -199,8 +198,7 @@ def test_mamba_cache_cg_padding( max_tokens: int, conv_state_layout: str, ) -> None: - """ - This test is for verifying that mamba cache is padded to CG captured + """This test is for verifying that mamba cache is padded to CG captured batch size. If it's not, a torch RuntimeError will be raised because tensor dimensions aren't compatible. """ @@ -236,8 +234,7 @@ def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks( example_prompts, model: str, ) -> None: - """ - This test is for verifying that the hybrid inner state management doesn't + """This test is for verifying that the hybrid inner state management doesn't collapse in case where the number of incoming requests and finished_requests_ids is larger than the maximum mamba block capacity. @@ -263,8 +260,7 @@ def test_state_cleanup( example_prompts, model: str, ) -> None: - """ - This test is for verifying that the Hybrid state is cleaned up between + """This test is for verifying that the Hybrid state is cleaned up between steps. If it's not cleaned, an error would be expected. diff --git a/tests/models/language/pooling/test_max_tokens_per_doc.py b/tests/models/language/pooling/test_max_tokens_per_doc.py index d99aa49909..b109b935df 100644 --- a/tests/models/language/pooling/test_max_tokens_per_doc.py +++ b/tests/models/language/pooling/test_max_tokens_per_doc.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tests for max_tokens_per_doc and max_tokens_per_query. -""" +"""Tests for max_tokens_per_doc and max_tokens_per_query.""" import json import os diff --git a/tests/models/multimodal/generation/test_interleaved.py b/tests/models/multimodal/generation/test_interleaved.py index a773db1982..07ce235853 100644 --- a/tests/models/multimodal/generation/test_interleaved.py +++ b/tests/models/multimodal/generation/test_interleaved.py @@ -23,11 +23,9 @@ NONINTERLEAVED_PROMPT = base_prompt("\n\n\ndef\n\n or abc\n\ndef @@ -60,8 +59,7 @@ class Ernie45ReasoningParser(BaseThinkingReasoningParser): current_token_ids: Sequence[int], delta_token_ids: Sequence[int], ) -> DeltaMessage | None: - """ - Extract reasoning content from a delta message. + """Extract reasoning content from a delta message. Handles streaming output where previous + delta = current. Uses token IDs for faster processing. The Ernie45 thinking model output format is @@ -138,8 +136,7 @@ class Ernie45ReasoningParser(BaseThinkingReasoningParser): def extract_reasoning( self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" ) -> tuple[str | None, str | None]: - """ - Extract reasoning content from the model output. + """Extract reasoning content from the model output. The Ernie45 thinking model output format is abc\n\n\n\n\ndef\n\n or abc\n\ndef diff --git a/vllm/reasoning/gemma4_utils.py b/vllm/reasoning/gemma4_utils.py index 9cdac72039..9566b8af56 100644 --- a/vllm/reasoning/gemma4_utils.py +++ b/vllm/reasoning/gemma4_utils.py @@ -75,6 +75,7 @@ def parse_thinking_output(text: str) -> dict[str, str | None]: >>> result = parse_thinking_output(output_text) >>> print(result["thinking"]) # chain-of-thought reasoning or None >>> print(result["answer"]) # final answer + """ if _THINKING_END_TAG in text: parts = text.split(_THINKING_END_TAG, 1) diff --git a/vllm/reasoning/gptoss_reasoning_parser.py b/vllm/reasoning/gptoss_reasoning_parser.py index a6f1080158..5505f1de44 100644 --- a/vllm/reasoning/gptoss_reasoning_parser.py +++ b/vllm/reasoning/gptoss_reasoning_parser.py @@ -14,8 +14,7 @@ if TYPE_CHECKING: class GptOssReasoningParser(ReasoningParser): - """ - Reasoning parser for GptOss model. + """Reasoning parser for GptOss model. The GptOss model uses harmony to extract reasoning content and this parser is only used for detecting the end of the reasoning content. diff --git a/vllm/reasoning/granite_reasoning_parser.py b/vllm/reasoning/granite_reasoning_parser.py index 463659f8d0..dc6118224f 100644 --- a/vllm/reasoning/granite_reasoning_parser.py +++ b/vllm/reasoning/granite_reasoning_parser.py @@ -16,8 +16,7 @@ if TYPE_CHECKING: class GraniteReasoningParser(ReasoningParser): - """ - Reasoning parser for IBM Granite. + """Reasoning parser for IBM Granite. IBM granite models currently use "Here is my thought process:" and "Here is my response:" to separate its thinking / response outputs. @@ -65,6 +64,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: tuple[Optional[str], Optional[str]]: Tuple pair containing the reasoning content and non-reasoning content. + """ re_match = self.reasoning_regex.findall(model_output) if not re_match: @@ -108,6 +108,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: Union[DeltaMessage, None] DeltaMessage with either reasoning content or content, or None. + """ reasoning, resp_seq_len, content = self._get_content_sections(current_text) # Either we haven't finished the start of the reasoning sequence, @@ -142,6 +143,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: bool: True if any of the possible reasoning start seqs match. + """ return any( think_start.startswith(text) for think_start in self.valid_think_starts @@ -155,6 +157,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: bool: True if any of the possible response start seqs match. + """ return any( response_start.startswith(text) @@ -175,6 +178,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: DeltaMessage: Message containing the parsed content. + """ prev_longest_length = len(current_text) - len(delta_text) is_substr = self._is_reasoning_start_substr(current_text) @@ -213,6 +217,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: DeltaMessage: Message containing the parsed content. + """ # If we have no reasoning content or explicitly end with the start of # response sequence, we are in transition to the response; need to be @@ -293,6 +298,7 @@ class GraniteReasoningParser(ReasoningParser): Returns: DeltaMessage: Message containing the parsed content. + """ # Always have content; take length to the end delta_content = delta_text[-len(response_content) :] @@ -329,6 +335,7 @@ class GraniteReasoningParser(ReasoningParser): tuple[Optional[str], Optional[int], Optional[str]]: Tuple of len 3 containing the reasoning content, the length of the response seq (if there is one) and the non-reasoning content. + """ current_chunk_start = 0 start_reasoning = None diff --git a/vllm/reasoning/hunyuan_a13b_reasoning_parser.py b/vllm/reasoning/hunyuan_a13b_reasoning_parser.py index b15b5608fb..0a7e021a46 100644 --- a/vllm/reasoning/hunyuan_a13b_reasoning_parser.py +++ b/vllm/reasoning/hunyuan_a13b_reasoning_parser.py @@ -16,8 +16,7 @@ if TYPE_CHECKING: class HunyuanA13BReasoningParser(ReasoningParser): - """ - Reasoning parser for Hunyuan A13B Model + """Reasoning parser for Hunyuan A13B Model. HunyuanReasoningParser @@ -102,8 +101,8 @@ class HunyuanA13BReasoningParser(ReasoningParser): Returns: tuple[Optional[str], Optional[str]]: Tuple pair containing the reasoning content and non-reasoning content. - """ + """ re_match = self.full_match_reasoning_regex.findall(model_output) if re_match: reasoning, response_content = re_match[0] @@ -139,7 +138,7 @@ class HunyuanA13BReasoningParser(ReasoningParser): current_token_ids: Sequence[int], delta_token_ids: Sequence[int], ) -> DeltaMessage | None: - """Extract content using token ID sequence state machine""" + """Extract content using token ID sequence state machine.""" # Define sequences think_start_sequence = self.think_start_ids response_start_sequence = self.response_start_ids diff --git a/vllm/reasoning/hy_v3_reasoning_parser.py b/vllm/reasoning/hy_v3_reasoning_parser.py index 0765be74af..77a1a271d4 100644 --- a/vllm/reasoning/hy_v3_reasoning_parser.py +++ b/vllm/reasoning/hy_v3_reasoning_parser.py @@ -18,8 +18,7 @@ logger = init_logger(__name__) class HYV3ReasoningParser(BaseThinkingReasoningParser): - """ - HYV3 parser that delegates to either HYV3ReasoningParser or + """HYV3 parser that delegates to either HYV3ReasoningParser or IdentityReasoningParser based on `reasoning_effort`. The HYV3 model uses ... tokens to denote reasoning text. diff --git a/vllm/reasoning/hy_v4_reasoning_parser.py b/vllm/reasoning/hy_v4_reasoning_parser.py index b65db92b92..65035ed53a 100644 --- a/vllm/reasoning/hy_v4_reasoning_parser.py +++ b/vllm/reasoning/hy_v4_reasoning_parser.py @@ -46,8 +46,8 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str: RuntimeError: The tokenizer declares the structural tokens through ``model_specific_special_tokens``, which transformers 5 no longer round-trips. - """ + """ import transformers if int(transformers.__version__.split(".")[0]) >= 5: diff --git a/vllm/reasoning/identity_reasoning_parser.py b/vllm/reasoning/identity_reasoning_parser.py index 52fab99141..f2c9a1e21e 100644 --- a/vllm/reasoning/identity_reasoning_parser.py +++ b/vllm/reasoning/identity_reasoning_parser.py @@ -15,8 +15,7 @@ if TYPE_CHECKING: class IdentityReasoningParser(ReasoningParser): - """ - Identity reasoning parser. + """Identity reasoning parser. This parser does not attempt to parse or strip out reasoning tokens. It treats the entire model output as content and ignores reasoning. diff --git a/vllm/reasoning/minimax_m2_reasoning_parser.py b/vllm/reasoning/minimax_m2_reasoning_parser.py index b26ea8889a..8fa5b45d58 100644 --- a/vllm/reasoning/minimax_m2_reasoning_parser.py +++ b/vllm/reasoning/minimax_m2_reasoning_parser.py @@ -15,8 +15,7 @@ if TYPE_CHECKING: class MiniMaxM2ReasoningParser(MinimaxM2ParserReasoningAdapter): # type: ignore[valid-type, misc] - """ - Reasoning parser for MiniMax M2 model. + """Reasoning parser for MiniMax M2 model. MiniMax M2 models don't generate start token, only end token. All content before is reasoning, content after is the @@ -25,9 +24,7 @@ class MiniMaxM2ReasoningParser(MinimaxM2ParserReasoningAdapter): # type: ignore class MiniMaxM2AppendThinkReasoningParser(ReasoningParser): - """ - Reasoning parser for MiniMax M2 model. - """ + """Reasoning parser for MiniMax M2 model.""" def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): super().__init__(tokenizer, *args, **kwargs) diff --git a/vllm/reasoning/olmo3_reasoning_parser.py b/vllm/reasoning/olmo3_reasoning_parser.py index b658b92bcb..8ac75f91ff 100644 --- a/vllm/reasoning/olmo3_reasoning_parser.py +++ b/vllm/reasoning/olmo3_reasoning_parser.py @@ -32,8 +32,7 @@ class Indices: def string_overlap(a: str, b: str) -> tuple[Indices | None, Indices | None]: - """ - Find the longest overlap where the end of string a matches the start + """Find the longest overlap where the end of string a matches the start of string b. Args: @@ -43,8 +42,8 @@ def string_overlap(a: str, b: str) -> tuple[Indices | None, Indices | None]: Returns: Tuple of IndicesTuples representing the overlapping portions in each string, or a tuple of None if no overlap exists - """ + """ # swap so a is always the shorter string a, b, swap = (a, b, False) if len(a) < len(b) else (b, a, True) @@ -188,8 +187,7 @@ class Olmo3ReasoningBuffer: class Olmo3ReasoningParser(ReasoningParser): - """ - Reasoning parser for Olmo 3 model + """Reasoning parser for Olmo 3 model. Olmo3ReasoningParser @@ -285,8 +283,8 @@ class Olmo3ReasoningParser(ReasoningParser): Returns: tuple[Optional[str], Optional[str]]: Tuple pair containing the reasoning content and non-reasoning content. - """ + """ re_match = self.reasoning_regex.match(model_output) if re_match: reasoning = re_match.group("reasoning") or None @@ -305,8 +303,7 @@ class Olmo3ReasoningParser(ReasoningParser): current_token_ids: Sequence[int], delta_token_ids: Sequence[int], ) -> DeltaMessage | None: - """Extract content using token ID sequence state machine""" - + """Extract content using token ID sequence state machine.""" delta_message = self.buffer.add_text(delta_text) if delta_message is None and self.buffer.think_end in self.buffer.buffer: # this is a bit hacky, but, because of how the buffer is diff --git a/vllm/reasoning/poolside_v1_reasoning_parser.py b/vllm/reasoning/poolside_v1_reasoning_parser.py index 30031d8513..e854afe4bc 100644 --- a/vllm/reasoning/poolside_v1_reasoning_parser.py +++ b/vllm/reasoning/poolside_v1_reasoning_parser.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Laguna reasoning parser. +"""Laguna reasoning parser. ``DeepSeekV3ReasoningParser.is_reasoning_end`` walks the entire token sequence backwards and returns ``True`` on the first ```` it diff --git a/vllm/reasoning/step3_reasoning_parser.py b/vllm/reasoning/step3_reasoning_parser.py index d2a575c74e..16ab1cf9f4 100644 --- a/vllm/reasoning/step3_reasoning_parser.py +++ b/vllm/reasoning/step3_reasoning_parser.py @@ -17,8 +17,7 @@ if TYPE_CHECKING: class Step3ReasoningParser(ReasoningParser): - """ - Reasoning parser for Step3 model. + """Reasoning parser for Step3 model. The Step3 model uses token to denote the end of reasoning text. This parser extracts all content before as reasoning content. @@ -62,8 +61,7 @@ class Step3ReasoningParser(ReasoningParser): current_token_ids: Sequence[int], delta_token_ids: Sequence[int], ) -> DeltaMessage | None: - """ - Extract reasoning content from a delta message. + """Extract reasoning content from a delta message. Handles streaming output where previous + delta = current. Uses token IDs for faster processing. For text "abcxyz": diff --git a/vllm/reasoning/step3p5_reasoning_parser.py b/vllm/reasoning/step3p5_reasoning_parser.py index cc7c227e6d..5969ca5228 100644 --- a/vllm/reasoning/step3p5_reasoning_parser.py +++ b/vllm/reasoning/step3p5_reasoning_parser.py @@ -14,8 +14,7 @@ if TYPE_CHECKING: class Step3p5ReasoningParser(BaseThinkingReasoningParser): - """ - Reasoning parser for Step3p5 model. + """Reasoning parser for Step3p5 model. Step3p5 uses the ... format, but it tends to emit an extra newline immediately before and/or after the token. This parser diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 4aca17fa43..2cb4dd646c 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -340,8 +340,7 @@ class BaseRenderer(ABC, Generic[_T]): self._mm_warmup_future = None def warmup(self, chat_params: ChatParams) -> None: - """ - Warm up this renderer to avoid first-request latency. + """Warm up this renderer to avoid first-request latency. For chat requests: - Jinja2 template compilation @@ -408,8 +407,7 @@ class BaseRenderer(ABC, Generic[_T]): return self.tokenizer.eos_token_id def get_dec_start_token_id(self) -> int: - """ - Obtain the decoder start token id employed by an encoder/decoder model, + """Obtain the decoder start token id employed by an encoder/decoder model, raising an error if it is not available. """ dec_start_token_id = getattr( diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 268a9aa463..a32824eebd 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -900,6 +900,7 @@ def rebuild_mm_uuids_from_mm_data( Returns: Updated UUIDs dictionary with chunk UUIDs + """ vision_chunks = mm_data.get("vision_chunk") if vision_chunks is None: @@ -932,6 +933,7 @@ def build_video_prompts_from_mm_data( Returns: List of video prompts, one per video. + """ vision_chunks = mm_data.get("vision_chunk") if vision_chunks is None: diff --git a/vllm/renderers/inputs/preprocess.py b/vllm/renderers/inputs/preprocess.py index dcf80ac738..82a49f5638 100644 --- a/vllm/renderers/inputs/preprocess.py +++ b/vllm/renderers/inputs/preprocess.py @@ -1,6 +1,4 @@ -""" -Schemas and utilities for preprocessing inputs. -""" +"""Schemas and utilities for preprocessing inputs.""" # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project @@ -90,8 +88,7 @@ that has been standardized into a dictionary. class EncoderDecoderDictPrompt(TypedDict): - """ - A [`EncoderDecoderPrompt`][vllm.inputs.llm.EncoderDecoderPrompt] + """A [`EncoderDecoderPrompt`][vllm.inputs.llm.EncoderDecoderPrompt] that has been standardized into a dictionary. """ @@ -130,9 +127,7 @@ def _validate_prompt_dict(prompt: Mapping[str, object]) -> None: def parse_dec_only_prompt(prompt: PromptType | object) -> DecoderOnlyDictPrompt: - """ - Parse a prompt for a decoder-only model and normalize it to a dictionary. - """ + """Parse a prompt for a decoder-only model and normalize it to a dictionary.""" if isinstance(prompt, str): return TextPrompt(prompt=prompt) @@ -216,9 +211,7 @@ def _parse_dec_prompt(prompt: PromptType | object) -> DecoderDictPrompt: def parse_enc_dec_prompt(prompt: PromptType | object) -> EncoderDecoderDictPrompt: - """ - Parse a prompt for an encoder-decoder model and normalize it to a dictionary. - """ + """Parse a prompt for an encoder-decoder model and normalize it to a dictionary.""" if isinstance(prompt, dict) and "encoder_prompt" in prompt: enc_prompt = prompt["encoder_prompt"] # type: ignore[typeddict-item] dec_prompt = prompt["decoder_prompt"] # type: ignore[typeddict-item] diff --git a/vllm/renderers/inputs/tokenize.py b/vllm/renderers/inputs/tokenize.py index 4168e20120..bf85051c37 100644 --- a/vllm/renderers/inputs/tokenize.py +++ b/vllm/renderers/inputs/tokenize.py @@ -1,6 +1,4 @@ -""" -Schemas and utilities for tokenization inputs. -""" +"""Schemas and utilities for tokenization inputs.""" # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project @@ -30,8 +28,7 @@ that has been tokenized. class EncoderDecoderTokPrompt(TypedDict): - """ - A + """A [`EncoderDecoderDictPrompt`][vllm.renderers.inputs.preprocess.EncoderDecoderDictPrompt] that has been tokenized. """ diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index ca70bb6ace..5f30b552ba 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -233,6 +233,7 @@ class OnlineDerenderer: Returns: (new_text, updated_state) — the delta text for this chunk and the state to pass to the next call. + """ prev_tokens = list(state.prev_tokens) prefix_offset = state.prefix_offset @@ -296,6 +297,7 @@ class OnlineDerenderer: Returns: (chunk, updated_state) — the derendered SSE chunk and the state the client must pass to the next call. + """ if state is None: state = DerenderStreamState() @@ -461,6 +463,7 @@ class OnlineDerenderer: Returns: (chunk, updated_state) — the derendered chunk and updated state. + """ if state is None: state = DerenderStreamState() diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index b651a06095..86470f8db1 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -425,8 +425,7 @@ class TokenizeParams: tokenizer: TokenizerLike | None, prompt: TextPrompt, ) -> TextPrompt: - """ - Ensure that the prompt meets the requirements set out by this config. + """Ensure that the prompt meets the requirements set out by this config. If that is not possible, raise a `VLLMValidationError`. This method is run before tokenization occurs. @@ -538,8 +537,7 @@ class TokenizeParams: tokenizer: TokenizerLike | None, prompt: TokensPrompt | EmbedsPrompt, ) -> TokensPrompt | EmbedsPrompt: - """ - Ensure that the prompt meets the requirements set out by this config. + """Ensure that the prompt meets the requirements set out by this config. If that is not possible, raise a `VLLMValidationError`. This method is run after tokenization occurs. diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index c0ab45ced7..6cd7b4e81c 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -127,9 +127,7 @@ class StructuredOutputsParams: ) def all_constraints_none(self) -> bool: - """ - Returns True if all structured-output constraint fields are None. - """ + """Returns True if all structured-output constraint fields are None.""" return all( getattr(self, field) is None for field in ( @@ -143,9 +141,7 @@ class StructuredOutputsParams: ) def all_non_structural_tag_constraints_none(self) -> bool: - """ - Returns True if all structured-output constraint fields are None. - """ + """Returns True if all structured-output constraint fields are None.""" return all( getattr(self, field) is None for field in ( @@ -692,7 +688,7 @@ class SamplingParams( generation_config: dict[str, Any], eos_token_id: int | None = None, ) -> None: - """Update if there are non-default values from generation_config""" + """Update if there are non-default values from generation_config.""" if not self.ignore_eos: self._eos_token_id = eos_token_id diff --git a/vllm/scalar_type.py b/vllm/scalar_type.py index db52e93465..d73827f499 100644 --- a/vllm/scalar_type.py +++ b/vllm/scalar_type.py @@ -21,8 +21,7 @@ class NanRepr(Enum): # in sync until the inductor fully supports custom C++ classes. @dataclass(frozen=True) class ScalarType: - """ - ScalarType can represent a wide range of floating point and integer + """ScalarType can represent a wide range of floating point and integer types, in particular it can be used to represent sub-byte data types (something that torch.dtype currently does not support). It is also capable of representing types with a bias, i.e.: @@ -135,8 +134,7 @@ class ScalarType: @functools.cached_property def id(self) -> int: - """ - Convert the ScalarType to an int which can be passed to pytorch custom + """Convert the ScalarType to an int which can be passed to pytorch custom ops. This layout of the int must be kept in sync with the C++ ScalarType's from_id method. """ @@ -168,56 +166,51 @@ class ScalarType: return self.exponent + self.mantissa + int(self.signed) def min(self) -> int | float: - """ - Min representable value for this scalar type. + """Min representable value for this scalar type. (accounting for bias if there is one) """ return self._raw_min() - self.bias def max(self) -> int | float: - """ - Max representable value for this scalar type. + """Max representable value for this scalar type. (accounting for bias if there is one) """ return self._raw_max() - self.bias def is_signed(self) -> bool: - """ - If the type is signed (i.e. has a sign bit), same as `signed` + """If the type is signed (i.e. has a sign bit), same as `signed` added for consistency with: https://pytorch.org/docs/stable/generated/torch.Tensor.is_signed.html """ return self.signed def is_floating_point(self) -> bool: - "If the type is a floating point type" + """If the type is a floating point type.""" return self.exponent != 0 def is_integer(self) -> bool: - "If the type is an integer type" + """If the type is an integer type.""" return self.exponent == 0 def has_bias(self) -> bool: - "If the type has a non-zero bias" + """If the type has a non-zero bias.""" return self.bias != 0 def has_infs(self) -> bool: - "If the type is floating point and supports infinity" + """If the type is floating point and supports infinity.""" return not self._finite_values_only def has_nans(self) -> bool: return self.nan_repr != NanRepr.NONE.value def is_ieee_754(self) -> bool: - """ - If the type is a floating point type that follows IEEE 754 + """If the type is a floating point type that follows IEEE 754 conventions """ return self.nan_repr == NanRepr.IEEE_754.value and not self._finite_values_only def __str__(self) -> str: - """ - naming generally follows: https://github.com/jax-ml/ml_dtypes + """Naming generally follows: https://github.com/jax-ml/ml_dtypes for floating point types (leading f) the scheme is: `float_em[flags]` flags: @@ -265,7 +258,7 @@ class ScalarType: @classmethod def int_(cls, size_bits: int, bias: int | None) -> "ScalarType": - "Create a signed integer scalar type (size_bits includes sign-bit)." + """Create a signed integer scalar type (size_bits includes sign-bit).""" ret = cls(0, size_bits - 1, True, bias if bias else 0) ret.id # noqa B018: make sure the id is cached return ret @@ -279,8 +272,7 @@ class ScalarType: @classmethod def float_IEEE754(cls, exponent: int, mantissa: int) -> "ScalarType": - """ - Create a standard floating point type + """Create a standard floating point type (i.e. follows IEEE 754 conventions). """ assert mantissa > 0 and exponent > 0 @@ -292,8 +284,7 @@ class ScalarType: def float_( cls, exponent: int, mantissa: int, finite_values_only: bool, nan_repr: NanRepr ) -> "ScalarType": - """ - Create a non-standard floating point type + """Create a non-standard floating point type (i.e. does not follow IEEE 754 conventions). """ assert mantissa > 0 and exponent > 0 diff --git a/vllm/third_party/flash_linear_attention/ops/fused_norm_gate.py b/vllm/third_party/flash_linear_attention/ops/fused_norm_gate.py index 60fc533507..8f58ddcf8c 100644 --- a/vllm/third_party/flash_linear_attention/ops/fused_norm_gate.py +++ b/vllm/third_party/flash_linear_attention/ops/fused_norm_gate.py @@ -15,6 +15,7 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv, next_power_of_2 + @triton.heuristics( { "STORE_RESIDUAL_OUT": lambda args: args["residual_out"] is not None, diff --git a/vllm/third_party/flash_linear_attention/ops/kda.py b/vllm/third_party/flash_linear_attention/ops/kda.py index 548f600812..b5c3a92b65 100644 --- a/vllm/third_party/flash_linear_attention/ops/kda.py +++ b/vllm/third_party/flash_linear_attention/ops/kda.py @@ -1349,6 +1349,7 @@ def fused_kda_gate_chunk_cumsum( ) return y + def _chunk_kda_fwd_with_cumulative_g( q: torch.Tensor, k: torch.Tensor, diff --git a/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py b/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py index 79f5abf031..997df5a173 100644 --- a/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py +++ b/vllm/third_party/flash_linear_attention/ops/layernorm_guard.py @@ -335,9 +335,7 @@ class LayerNormFwdKernel(VllmJitKernel["LayerNormFwdKernel.CompileKey"]): assert warmup is not None variant = compile_key.input_variant - def pointer( - name: str, dtype: torch.dtype | None - ) -> TritonWarmupTensor | None: + def pointer(name: str, dtype: torch.dtype | None) -> TritonWarmupTensor | None: return None if dtype is None else variant.pointer(name, dtype) warmup( diff --git a/vllm/third_party/pynvml.py b/vllm/third_party/pynvml.py index 6aabbc217d..3bb53e8d93 100644 --- a/vllm/third_party/pynvml.py +++ b/vllm/third_party/pynvml.py @@ -45,52 +45,54 @@ import string ## C Type mappings ## ## Enums _nvmlEnableState_t = c_uint -NVML_FEATURE_DISABLED = 0 -NVML_FEATURE_ENABLED = 1 +NVML_FEATURE_DISABLED = 0 +NVML_FEATURE_ENABLED = 1 _nvmlBrandType_t = c_uint -NVML_BRAND_UNKNOWN = 0 -NVML_BRAND_QUADRO = 1 -NVML_BRAND_TESLA = 2 -NVML_BRAND_NVS = 3 -NVML_BRAND_GRID = 4 # Deprecated from API reporting. Keeping definition for backward compatibility. -NVML_BRAND_GEFORCE = 5 -NVML_BRAND_TITAN = 6 -NVML_BRAND_NVIDIA_VAPPS = 7 # NVIDIA Virtual Applications -NVML_BRAND_NVIDIA_VPC = 8 # NVIDIA Virtual PC -NVML_BRAND_NVIDIA_VCS = 9 # NVIDIA Virtual Compute Server -NVML_BRAND_NVIDIA_VWS = 10 # NVIDIA RTX Virtual Workstation +NVML_BRAND_UNKNOWN = 0 +NVML_BRAND_QUADRO = 1 +NVML_BRAND_TESLA = 2 +NVML_BRAND_NVS = 3 +NVML_BRAND_GRID = ( + 4 # Deprecated from API reporting. Keeping definition for backward compatibility. +) +NVML_BRAND_GEFORCE = 5 +NVML_BRAND_TITAN = 6 +NVML_BRAND_NVIDIA_VAPPS = 7 # NVIDIA Virtual Applications +NVML_BRAND_NVIDIA_VPC = 8 # NVIDIA Virtual PC +NVML_BRAND_NVIDIA_VCS = 9 # NVIDIA Virtual Compute Server +NVML_BRAND_NVIDIA_VWS = 10 # NVIDIA RTX Virtual Workstation NVML_BRAND_NVIDIA_CLOUD_GAMING = 11 # NVIDIA Cloud Gaming -NVML_BRAND_NVIDIA_VGAMING = NVML_BRAND_NVIDIA_CLOUD_GAMING # Deprecated from API reporting. Keeping definition for backward compatibility. -NVML_BRAND_QUADRO_RTX = 12 -NVML_BRAND_NVIDIA_RTX = 13 -NVML_BRAND_NVIDIA = 14 -NVML_BRAND_GEFORCE_RTX = 15 # Unused -NVML_BRAND_TITAN_RTX = 16 # Unused -NVML_BRAND_COUNT = 17 +NVML_BRAND_NVIDIA_VGAMING = NVML_BRAND_NVIDIA_CLOUD_GAMING # Deprecated from API reporting. Keeping definition for backward compatibility. +NVML_BRAND_QUADRO_RTX = 12 +NVML_BRAND_NVIDIA_RTX = 13 +NVML_BRAND_NVIDIA = 14 +NVML_BRAND_GEFORCE_RTX = 15 # Unused +NVML_BRAND_TITAN_RTX = 16 # Unused +NVML_BRAND_COUNT = 17 _nvmlTemperatureThresholds_t = c_uint -NVML_TEMPERATURE_THRESHOLD_SHUTDOWN = 0 -NVML_TEMPERATURE_THRESHOLD_SLOWDOWN = 1 -NVML_TEMPERATURE_THRESHOLD_MEM_MAX = 2 -NVML_TEMPERATURE_THRESHOLD_GPU_MAX = 3 -NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MIN = 4 +NVML_TEMPERATURE_THRESHOLD_SHUTDOWN = 0 +NVML_TEMPERATURE_THRESHOLD_SLOWDOWN = 1 +NVML_TEMPERATURE_THRESHOLD_MEM_MAX = 2 +NVML_TEMPERATURE_THRESHOLD_GPU_MAX = 3 +NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MIN = 4 NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_CURR = 5 -NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MAX = 6 -NVML_TEMPERATURE_THRESHOLD_GPS_CURR = 7 -NVML_TEMPERATURE_THRESHOLD_COUNT = 8 +NVML_TEMPERATURE_THRESHOLD_ACOUSTIC_MAX = 6 +NVML_TEMPERATURE_THRESHOLD_GPS_CURR = 7 +NVML_TEMPERATURE_THRESHOLD_COUNT = 8 _nvmlTemperatureSensors_t = c_uint -NVML_TEMPERATURE_GPU = 0 -NVML_TEMPERATURE_COUNT = 1 +NVML_TEMPERATURE_GPU = 0 +NVML_TEMPERATURE_COUNT = 1 _nvmlComputeMode_t = c_uint -NVML_COMPUTEMODE_DEFAULT = 0 -NVML_COMPUTEMODE_EXCLUSIVE_THREAD = 1 ## Support Removed -NVML_COMPUTEMODE_PROHIBITED = 2 +NVML_COMPUTEMODE_DEFAULT = 0 +NVML_COMPUTEMODE_EXCLUSIVE_THREAD = 1 ## Support Removed +NVML_COMPUTEMODE_PROHIBITED = 2 NVML_COMPUTEMODE_EXCLUSIVE_PROCESS = 3 -NVML_COMPUTEMODE_COUNT = 4 +NVML_COMPUTEMODE_COUNT = 4 _nvmlMemoryLocation_t = c_uint NVML_MEMORY_LOCATION_L1_CACHE = 0 @@ -127,158 +129,158 @@ NVML_NVLINK_ERROR_DL_ECC_COUNT = 5 _nvmlNvLinkCapability_t = c_uint NVML_NVLINK_CAP_P2P_SUPPORTED = 0 NVML_NVLINK_CAP_SYSMEM_ACCESS = 1 -NVML_NVLINK_CAP_P2P_ATOMICS = 2 -NVML_NVLINK_CAP_SYSMEM_ATOMICS= 3 -NVML_NVLINK_CAP_SLI_BRIDGE = 4 -NVML_NVLINK_CAP_VALID = 5 -NVML_NVLINK_CAP_COUNT = 6 +NVML_NVLINK_CAP_P2P_ATOMICS = 2 +NVML_NVLINK_CAP_SYSMEM_ATOMICS = 3 +NVML_NVLINK_CAP_SLI_BRIDGE = 4 +NVML_NVLINK_CAP_VALID = 5 +NVML_NVLINK_CAP_COUNT = 6 _nvmlNvLinkUtilizationCountPktTypes_t = c_uint -NVML_NVLINK_COUNTER_PKTFILTER_NOP = 0x1 -NVML_NVLINK_COUNTER_PKTFILTER_READ = 0x2 -NVML_NVLINK_COUNTER_PKTFILTER_WRITE = 0x4 -NVML_NVLINK_COUNTER_PKTFILTER_RATOM = 0x8 -NVML_NVLINK_COUNTER_PKTFILTER_NRATOM = 0x10 -NVML_NVLINK_COUNTER_PKTFILTER_FLUSH = 0x20 -NVML_NVLINK_COUNTER_PKTFILTER_RESPDATA = 0x40 +NVML_NVLINK_COUNTER_PKTFILTER_NOP = 0x1 +NVML_NVLINK_COUNTER_PKTFILTER_READ = 0x2 +NVML_NVLINK_COUNTER_PKTFILTER_WRITE = 0x4 +NVML_NVLINK_COUNTER_PKTFILTER_RATOM = 0x8 +NVML_NVLINK_COUNTER_PKTFILTER_NRATOM = 0x10 +NVML_NVLINK_COUNTER_PKTFILTER_FLUSH = 0x20 +NVML_NVLINK_COUNTER_PKTFILTER_RESPDATA = 0x40 NVML_NVLINK_COUNTER_PKTFILTER_RESPNODATA = 0x80 -NVML_NVLINK_COUNTER_PKTFILTER_ALL = 0xFF +NVML_NVLINK_COUNTER_PKTFILTER_ALL = 0xFF _nvmlNvLinkUtilizationCountUnits_t = c_uint -NVML_NVLINK_COUNTER_UNIT_CYCLES = 0 -NVML_NVLINK_COUNTER_UNIT_PACKETS = 1 -NVML_NVLINK_COUNTER_UNIT_BYTES = 2 +NVML_NVLINK_COUNTER_UNIT_CYCLES = 0 +NVML_NVLINK_COUNTER_UNIT_PACKETS = 1 +NVML_NVLINK_COUNTER_UNIT_BYTES = 2 NVML_NVLINK_COUNTER_UNIT_RESERVED = 3 -NVML_NVLINK_COUNTER_UNIT_COUNT = 4 +NVML_NVLINK_COUNTER_UNIT_COUNT = 4 _nvmlNvLinkDeviceType_t = c_uint -NVML_NVLINK_DEVICE_TYPE_GPU = 0x00 -NVML_NVLINK_DEVICE_TYPE_IBMNPU = 0x01 -NVML_NVLINK_DEVICE_TYPE_SWITCH = 0x02 +NVML_NVLINK_DEVICE_TYPE_GPU = 0x00 +NVML_NVLINK_DEVICE_TYPE_IBMNPU = 0x01 +NVML_NVLINK_DEVICE_TYPE_SWITCH = 0x02 NVML_NVLINK_DEVICE_TYPE_UNKNOWN = 0xFF # These are deprecated, instead use _nvmlMemoryErrorType_t _nvmlEccBitType_t = c_uint -NVML_SINGLE_BIT_ECC = 0 -NVML_DOUBLE_BIT_ECC = 1 +NVML_SINGLE_BIT_ECC = 0 +NVML_DOUBLE_BIT_ECC = 1 NVML_ECC_ERROR_TYPE_COUNT = 2 _nvmlEccCounterType_t = c_uint -NVML_VOLATILE_ECC = 0 -NVML_AGGREGATE_ECC = 1 +NVML_VOLATILE_ECC = 0 +NVML_AGGREGATE_ECC = 1 NVML_ECC_COUNTER_TYPE_COUNT = 2 _nvmlMemoryErrorType_t = c_uint -NVML_MEMORY_ERROR_TYPE_CORRECTED = 0 +NVML_MEMORY_ERROR_TYPE_CORRECTED = 0 NVML_MEMORY_ERROR_TYPE_UNCORRECTED = 1 -NVML_MEMORY_ERROR_TYPE_COUNT = 2 +NVML_MEMORY_ERROR_TYPE_COUNT = 2 _nvmlClockType_t = c_uint -NVML_CLOCK_GRAPHICS = 0 -NVML_CLOCK_SM = 1 -NVML_CLOCK_MEM = 2 -NVML_CLOCK_VIDEO = 3 -NVML_CLOCK_COUNT = 4 +NVML_CLOCK_GRAPHICS = 0 +NVML_CLOCK_SM = 1 +NVML_CLOCK_MEM = 2 +NVML_CLOCK_VIDEO = 3 +NVML_CLOCK_COUNT = 4 _nvmlClockId_t = c_uint -NVML_CLOCK_ID_CURRENT = 0 -NVML_CLOCK_ID_APP_CLOCK_TARGET = 1 -NVML_CLOCK_ID_APP_CLOCK_DEFAULT = 2 +NVML_CLOCK_ID_CURRENT = 0 +NVML_CLOCK_ID_APP_CLOCK_TARGET = 1 +NVML_CLOCK_ID_APP_CLOCK_DEFAULT = 2 NVML_CLOCK_ID_CUSTOMER_BOOST_MAX = 3 -NVML_CLOCK_ID_COUNT = 4 +NVML_CLOCK_ID_COUNT = 4 _nvmlDriverModel_t = c_uint -NVML_DRIVER_WDDM = 0 -NVML_DRIVER_WDM = 1 -NVML_DRIVER_MCDM = 2 +NVML_DRIVER_WDDM = 0 +NVML_DRIVER_WDM = 1 +NVML_DRIVER_MCDM = 2 NVML_MAX_GPU_PERF_PSTATES = 16 _nvmlPstates_t = c_uint -NVML_PSTATE_0 = 0 -NVML_PSTATE_1 = 1 -NVML_PSTATE_2 = 2 -NVML_PSTATE_3 = 3 -NVML_PSTATE_4 = 4 -NVML_PSTATE_5 = 5 -NVML_PSTATE_6 = 6 -NVML_PSTATE_7 = 7 -NVML_PSTATE_8 = 8 -NVML_PSTATE_9 = 9 -NVML_PSTATE_10 = 10 -NVML_PSTATE_11 = 11 -NVML_PSTATE_12 = 12 -NVML_PSTATE_13 = 13 -NVML_PSTATE_14 = 14 -NVML_PSTATE_15 = 15 -NVML_PSTATE_UNKNOWN = 32 +NVML_PSTATE_0 = 0 +NVML_PSTATE_1 = 1 +NVML_PSTATE_2 = 2 +NVML_PSTATE_3 = 3 +NVML_PSTATE_4 = 4 +NVML_PSTATE_5 = 5 +NVML_PSTATE_6 = 6 +NVML_PSTATE_7 = 7 +NVML_PSTATE_8 = 8 +NVML_PSTATE_9 = 9 +NVML_PSTATE_10 = 10 +NVML_PSTATE_11 = 11 +NVML_PSTATE_12 = 12 +NVML_PSTATE_13 = 13 +NVML_PSTATE_14 = 14 +NVML_PSTATE_15 = 15 +NVML_PSTATE_UNKNOWN = 32 _nvmlInforomObject_t = c_uint -NVML_INFOROM_OEM = 0 -NVML_INFOROM_ECC = 1 -NVML_INFOROM_POWER = 2 -NVML_INFOROM_DEN = 3 -NVML_INFOROM_COUNT = 4 +NVML_INFOROM_OEM = 0 +NVML_INFOROM_ECC = 1 +NVML_INFOROM_POWER = 2 +NVML_INFOROM_DEN = 3 +NVML_INFOROM_COUNT = 4 _nvmlReturn_t = c_uint -NVML_SUCCESS = 0 -NVML_ERROR_UNINITIALIZED = 1 -NVML_ERROR_INVALID_ARGUMENT = 2 -NVML_ERROR_NOT_SUPPORTED = 3 -NVML_ERROR_NO_PERMISSION = 4 -NVML_ERROR_ALREADY_INITIALIZED = 5 -NVML_ERROR_NOT_FOUND = 6 -NVML_ERROR_INSUFFICIENT_SIZE = 7 -NVML_ERROR_INSUFFICIENT_POWER = 8 -NVML_ERROR_DRIVER_NOT_LOADED = 9 -NVML_ERROR_TIMEOUT = 10 -NVML_ERROR_IRQ_ISSUE = 11 -NVML_ERROR_LIBRARY_NOT_FOUND = 12 -NVML_ERROR_FUNCTION_NOT_FOUND = 13 -NVML_ERROR_CORRUPTED_INFOROM = 14 -NVML_ERROR_GPU_IS_LOST = 15 -NVML_ERROR_RESET_REQUIRED = 16 -NVML_ERROR_OPERATING_SYSTEM = 17 -NVML_ERROR_LIB_RM_VERSION_MISMATCH = 18 -NVML_ERROR_IN_USE = 19 -NVML_ERROR_MEMORY = 20 -NVML_ERROR_NO_DATA = 21 -NVML_ERROR_VGPU_ECC_NOT_SUPPORTED = 22 -NVML_ERROR_INSUFFICIENT_RESOURCES = 23 -NVML_ERROR_FREQ_NOT_SUPPORTED = 24 +NVML_SUCCESS = 0 +NVML_ERROR_UNINITIALIZED = 1 +NVML_ERROR_INVALID_ARGUMENT = 2 +NVML_ERROR_NOT_SUPPORTED = 3 +NVML_ERROR_NO_PERMISSION = 4 +NVML_ERROR_ALREADY_INITIALIZED = 5 +NVML_ERROR_NOT_FOUND = 6 +NVML_ERROR_INSUFFICIENT_SIZE = 7 +NVML_ERROR_INSUFFICIENT_POWER = 8 +NVML_ERROR_DRIVER_NOT_LOADED = 9 +NVML_ERROR_TIMEOUT = 10 +NVML_ERROR_IRQ_ISSUE = 11 +NVML_ERROR_LIBRARY_NOT_FOUND = 12 +NVML_ERROR_FUNCTION_NOT_FOUND = 13 +NVML_ERROR_CORRUPTED_INFOROM = 14 +NVML_ERROR_GPU_IS_LOST = 15 +NVML_ERROR_RESET_REQUIRED = 16 +NVML_ERROR_OPERATING_SYSTEM = 17 +NVML_ERROR_LIB_RM_VERSION_MISMATCH = 18 +NVML_ERROR_IN_USE = 19 +NVML_ERROR_MEMORY = 20 +NVML_ERROR_NO_DATA = 21 +NVML_ERROR_VGPU_ECC_NOT_SUPPORTED = 22 +NVML_ERROR_INSUFFICIENT_RESOURCES = 23 +NVML_ERROR_FREQ_NOT_SUPPORTED = 24 NVML_ERROR_ARGUMENT_VERSION_MISMATCH = 25 -NVML_ERROR_DEPRECATED = 26 -NVML_ERROR_NOT_READY = 27 -NVML_ERROR_GPU_NOT_FOUND = 28 -NVML_ERROR_INVALID_STATE = 29 -NVML_ERROR_UNKNOWN = 999 +NVML_ERROR_DEPRECATED = 26 +NVML_ERROR_NOT_READY = 27 +NVML_ERROR_GPU_NOT_FOUND = 28 +NVML_ERROR_INVALID_STATE = 29 +NVML_ERROR_UNKNOWN = 999 _nvmlFanState_t = c_uint -NVML_FAN_NORMAL = 0 -NVML_FAN_FAILED = 1 +NVML_FAN_NORMAL = 0 +NVML_FAN_FAILED = 1 _nvmlFanControlPolicy_t = c_uint NVML_FAN_POLICY_TEMPERATURE_CONTINOUS_SW = 0 -NVML_FAN_POLICY_MANUAL = 1 +NVML_FAN_POLICY_MANUAL = 1 _nvmlLedColor_t = c_uint -NVML_LED_COLOR_GREEN = 0 -NVML_LED_COLOR_AMBER = 1 +NVML_LED_COLOR_GREEN = 0 +NVML_LED_COLOR_AMBER = 1 _nvmlGpuOperationMode_t = c_uint -NVML_GOM_ALL_ON = 0 -NVML_GOM_COMPUTE = 1 -NVML_GOM_LOW_DP = 2 +NVML_GOM_ALL_ON = 0 +NVML_GOM_COMPUTE = 1 +NVML_GOM_LOW_DP = 2 _nvmlPageRetirementCause_t = c_uint NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS = 0 -NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR = 1 -NVML_PAGE_RETIREMENT_CAUSE_COUNT = 2 +NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR = 1 +NVML_PAGE_RETIREMENT_CAUSE_COUNT = 2 _nvmlRestrictedAPI_t = c_uint -NVML_RESTRICTED_API_SET_APPLICATION_CLOCKS = 0 -NVML_RESTRICTED_API_SET_AUTO_BOOSTED_CLOCKS = 1 -NVML_RESTRICTED_API_COUNT = 2 +NVML_RESTRICTED_API_SET_APPLICATION_CLOCKS = 0 +NVML_RESTRICTED_API_SET_AUTO_BOOSTED_CLOCKS = 1 +NVML_RESTRICTED_API_COUNT = 2 _nvmlBridgeChipType_t = c_uint NVML_BRIDGE_CHIP_PLX = 0 @@ -365,9 +367,9 @@ NVML_TOPOLOGY_CPU = NVML_TOPOLOGY_NODE NVML_TOPOLOGY_SYSTEM = 50 _nvmlGpuP2PCapsIndex_t = c_uint -NVML_P2P_CAPS_INDEX_READ = 0, +NVML_P2P_CAPS_INDEX_READ = (0,) NVML_P2P_CAPS_INDEX_WRITE = 1 -NVML_P2P_CAPS_INDEX_NVLINK =2 +NVML_P2P_CAPS_INDEX_NVLINK = 2 NVML_P2P_CAPS_INDEX_ATOMICS = 3 # # NVML_P2P_CAPS_INDEX_PROP is deprecated. @@ -378,39 +380,39 @@ NVML_P2P_CAPS_INDEX_PCI = 4 NVML_P2P_CAPS_INDEX_UNKNOWN = 5 _nvmlGpuP2PStatus_t = c_uint -NVML_P2P_STATUS_OK = 0 +NVML_P2P_STATUS_OK = 0 NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED = 1 NVML_P2P_STATUS_CHIPSET_NOT_SUPPORTED = NVML_P2P_STATUS_CHIPSET_NOT_SUPPORED NVML_P2P_STATUS_GPU_NOT_SUPPORTED = 2 -NVML_P2P_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED =3 -NVML_P2P_STATUS_DISABLED_BY_REGKEY =4 -NVML_P2P_STATUS_NOT_SUPPORTED =5 -NVML_P2P_STATUS_UNKNOWN =6 +NVML_P2P_STATUS_IOH_TOPOLOGY_NOT_SUPPORTED = 3 +NVML_P2P_STATUS_DISABLED_BY_REGKEY = 4 +NVML_P2P_STATUS_NOT_SUPPORTED = 5 +NVML_P2P_STATUS_UNKNOWN = 6 _nvmlDeviceArchitecture_t = c_uint -NVML_DEVICE_ARCH_KEPLER = 2 -NVML_DEVICE_ARCH_MAXWELL = 3 -NVML_DEVICE_ARCH_PASCAL = 4 -NVML_DEVICE_ARCH_VOLTA = 5 -NVML_DEVICE_ARCH_TURING = 6 -NVML_DEVICE_ARCH_AMPERE = 7 -NVML_DEVICE_ARCH_ADA = 8 -NVML_DEVICE_ARCH_HOPPER = 9 -NVML_DEVICE_ARCH_BLACKWELL = 10 -NVML_DEVICE_ARCH_T23X = 11 -NVML_DEVICE_ARCH_UNKNOWN = 0xffffffff +NVML_DEVICE_ARCH_KEPLER = 2 +NVML_DEVICE_ARCH_MAXWELL = 3 +NVML_DEVICE_ARCH_PASCAL = 4 +NVML_DEVICE_ARCH_VOLTA = 5 +NVML_DEVICE_ARCH_TURING = 6 +NVML_DEVICE_ARCH_AMPERE = 7 +NVML_DEVICE_ARCH_ADA = 8 +NVML_DEVICE_ARCH_HOPPER = 9 +NVML_DEVICE_ARCH_BLACKWELL = 10 +NVML_DEVICE_ARCH_T23X = 11 +NVML_DEVICE_ARCH_UNKNOWN = 0xFFFFFFFF # PCI bus Types _nvmlBusType_t = c_uint NVML_BUS_TYPE_UNKNOWN = 0 -NVML_BUS_TYPE_PCI = 1 -NVML_BUS_TYPE_PCIE = 2 -NVML_BUS_TYPE_FPCI = 3 -NVML_BUS_TYPE_AGP = 4 +NVML_BUS_TYPE_PCI = 1 +NVML_BUS_TYPE_PCIE = 2 +NVML_BUS_TYPE_FPCI = 3 +NVML_BUS_TYPE_AGP = 4 _nvmlPowerSource_t = c_uint -NVML_POWER_SOURCE_AC = 0x00000000 -NVML_POWER_SOURCE_BATTERY = 0x00000001 +NVML_POWER_SOURCE_AC = 0x00000000 +NVML_POWER_SOURCE_BATTERY = 0x00000001 NVML_POWER_SOURCE_UNDERSIZED = 0x00000002 _nvmlAdaptiveClockInfoStatus_t = c_uint @@ -418,196 +420,290 @@ NVML_ADAPTIVE_CLOCKING_INFO_STATUS_DISABLED = 0x00000000 NVML_ADAPTIVE_CLOCKING_INFO_STATUS_ENABLED = 0x00000001 _nvmlClockLimitId_t = c_uint -NVML_CLOCK_LIMIT_ID_RANGE_START = 0xffffff00 -NVML_CLOCK_LIMIT_ID_TDP = 0xffffff01 -NVML_CLOCK_LIMIT_ID_UNLIMITED = 0xffffff02 +NVML_CLOCK_LIMIT_ID_RANGE_START = 0xFFFFFF00 +NVML_CLOCK_LIMIT_ID_TDP = 0xFFFFFF01 +NVML_CLOCK_LIMIT_ID_UNLIMITED = 0xFFFFFF02 _nvmlPcieLinkMaxSpeed_t = c_uint -NVML_PCIE_LINK_MAX_SPEED_INVALID = 0x00000000 -NVML_PCIE_LINK_MAX_SPEED_2500MBPS = 0x00000001 -NVML_PCIE_LINK_MAX_SPEED_5000MBPS = 0x00000002 -NVML_PCIE_LINK_MAX_SPEED_8000MBPS = 0x00000003 +NVML_PCIE_LINK_MAX_SPEED_INVALID = 0x00000000 +NVML_PCIE_LINK_MAX_SPEED_2500MBPS = 0x00000001 +NVML_PCIE_LINK_MAX_SPEED_5000MBPS = 0x00000002 +NVML_PCIE_LINK_MAX_SPEED_8000MBPS = 0x00000003 NVML_PCIE_LINK_MAX_SPEED_16000MBPS = 0x00000004 NVML_PCIE_LINK_MAX_SPEED_32000MBPS = 0x00000005 NVML_PCIE_LINK_MAX_SPEED_64000MBPS = 0x00000006 _nvmlPcieAtomicsCapability_t = c_uint -NVML_PCIE_ATOMICS_CAP_FETCHADD32 = 0x01 -NVML_PCIE_ATOMICS_CAP_FETCHADD64 = 0x02 -NVML_PCIE_ATOMICS_CAP_SWAP32 = 0x04 -NVML_PCIE_ATOMICS_CAP_SWAP64 = 0x08 -NVML_PCIE_ATOMICS_CAP_CAS32 = 0x10 -NVML_PCIE_ATOMICS_CAP_CAS64 = 0x20 -NVML_PCIE_ATOMICS_CAP_CAS128 = 0x40 -NVML_PCIE_ATOMICS_OPS_MAX = 7 +NVML_PCIE_ATOMICS_CAP_FETCHADD32 = 0x01 +NVML_PCIE_ATOMICS_CAP_FETCHADD64 = 0x02 +NVML_PCIE_ATOMICS_CAP_SWAP32 = 0x04 +NVML_PCIE_ATOMICS_CAP_SWAP64 = 0x08 +NVML_PCIE_ATOMICS_CAP_CAS32 = 0x10 +NVML_PCIE_ATOMICS_CAP_CAS64 = 0x20 +NVML_PCIE_ATOMICS_CAP_CAS128 = 0x40 +NVML_PCIE_ATOMICS_OPS_MAX = 7 _nvmlAffinityScope_t = c_uint -NVML_AFFINITY_SCOPE_NODE = 0 +NVML_AFFINITY_SCOPE_NODE = 0 NVML_AFFINITY_SCOPE_SOCKET = 1 _nvmlDeviceGpuRecoveryAction_t = c_uint -NVML_GPU_RECOVERY_ACTION_NONE = 0 -NVML_GPU_RECOVERY_ACTION_GPU_RESET = 1 +NVML_GPU_RECOVERY_ACTION_NONE = 0 +NVML_GPU_RECOVERY_ACTION_GPU_RESET = 1 NVML_GPU_RECOVERY_ACTION_NODE_REBOOT = 2 -NVML_GPU_RECOVERY_ACTION_DRAIN_P2P = 3 +NVML_GPU_RECOVERY_ACTION_DRAIN_P2P = 3 NVML_GPU_RECOVERY_ACTION_DRAIN_AND_RESET = 4 # C preprocessor defined values -nvmlFlagDefault = 0 -nvmlFlagForce = 1 -NVML_INIT_FLAG_NO_GPUS = 1 -NVML_INIT_FLAG_NO_ATTACH = 2 +nvmlFlagDefault = 0 +nvmlFlagForce = 1 +NVML_INIT_FLAG_NO_GPUS = 1 +NVML_INIT_FLAG_NO_ATTACH = 2 -NVML_MAX_GPC_COUNT = 32 +NVML_MAX_GPC_COUNT = 32 # buffer size -NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE = 16 -NVML_DEVICE_UUID_BUFFER_SIZE = 80 -NVML_DEVICE_UUID_V2_BUFFER_SIZE = 96 -NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE = 80 -NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE = 80 -NVML_DEVICE_NAME_BUFFER_SIZE = 64 -NVML_DEVICE_NAME_V2_BUFFER_SIZE = 96 -NVML_DEVICE_SERIAL_BUFFER_SIZE = 30 -NVML_DEVICE_PART_NUMBER_BUFFER_SIZE = 80 -NVML_DEVICE_GPU_PART_NUMBER_BUFFER_SIZE = 80 -NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE = 32 -NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE = 32 -NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE = 16 -NVML_GRID_LICENSE_BUFFER_SIZE = 128 -NVML_VGPU_NAME_BUFFER_SIZE = 64 -NVML_GRID_LICENSE_FEATURE_MAX_COUNT = 3 -NVML_VGPU_METADATA_OPAQUE_DATA_SIZE = sizeof(c_uint) + 256 -NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE = 256 -NVML_DEVICE_GPU_FRU_PART_NUMBER_BUFFER_SIZE = 0x14 # NV2080_GPU_MAX_PRODUCT_PART_NUMBER_LENGTH -NVML_PERF_MODES_BUFFER_SIZE = 2048 +NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE = 16 +NVML_DEVICE_UUID_BUFFER_SIZE = 80 +NVML_DEVICE_UUID_V2_BUFFER_SIZE = 96 +NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE = 80 +NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE = 80 +NVML_DEVICE_NAME_BUFFER_SIZE = 64 +NVML_DEVICE_NAME_V2_BUFFER_SIZE = 96 +NVML_DEVICE_SERIAL_BUFFER_SIZE = 30 +NVML_DEVICE_PART_NUMBER_BUFFER_SIZE = 80 +NVML_DEVICE_GPU_PART_NUMBER_BUFFER_SIZE = 80 +NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE = 32 +NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE = 32 +NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE = 16 +NVML_GRID_LICENSE_BUFFER_SIZE = 128 +NVML_VGPU_NAME_BUFFER_SIZE = 64 +NVML_GRID_LICENSE_FEATURE_MAX_COUNT = 3 +NVML_VGPU_METADATA_OPAQUE_DATA_SIZE = sizeof(c_uint) + 256 +NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE = 256 +NVML_DEVICE_GPU_FRU_PART_NUMBER_BUFFER_SIZE = ( + 0x14 # NV2080_GPU_MAX_PRODUCT_PART_NUMBER_LENGTH +) +NVML_PERF_MODES_BUFFER_SIZE = 2048 # Format strings -NVML_DEVICE_PCI_BUS_ID_LEGACY_FMT = "%04X:%02X:%02X.0" -NVML_DEVICE_PCI_BUS_ID_FMT = "%08X:%02X:%02X.0" +NVML_DEVICE_PCI_BUS_ID_LEGACY_FMT = "%04X:%02X:%02X.0" +NVML_DEVICE_PCI_BUS_ID_FMT = "%08X:%02X:%02X.0" NVML_VALUE_NOT_AVAILABLE_ulonglong = c_ulonglong(-1) NVML_VALUE_NOT_AVAILABLE_uint = c_uint(-1) -''' +""" Field Identifiers. All Identifiers pertain to a device. Each ID is only used once and is guaranteed never to change. -''' -NVML_FI_DEV_ECC_CURRENT = 1 # Current ECC mode. 1=Active. 0=Inactive -NVML_FI_DEV_ECC_PENDING = 2 # Pending ECC mode. 1=Active. 0=Inactive +""" +NVML_FI_DEV_ECC_CURRENT = 1 # Current ECC mode. 1=Active. 0=Inactive +NVML_FI_DEV_ECC_PENDING = 2 # Pending ECC mode. 1=Active. 0=Inactive -#ECC Count Totals -NVML_FI_DEV_ECC_SBE_VOL_TOTAL = 3 # Total single bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_TOTAL = 4 # Total double bit volatile ECC errors -NVML_FI_DEV_ECC_SBE_AGG_TOTAL = 5 # Total single bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_TOTAL = 6 # Total double bit aggregate (persistent) ECC errors -#Individual ECC locations -NVML_FI_DEV_ECC_SBE_VOL_L1 = 7 # L1 cache single bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_L1 = 8 # L1 cache double bit volatile ECC errors -NVML_FI_DEV_ECC_SBE_VOL_L2 = 9 # L2 cache single bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_L2 = 10 # L2 cache double bit volatile ECC errors -NVML_FI_DEV_ECC_SBE_VOL_DEV = 11 # Device memory single bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_DEV = 12 # Device memory double bit volatile ECC errors -NVML_FI_DEV_ECC_SBE_VOL_REG = 13 # Register file single bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_REG = 14 # Register file double bit volatile ECC errors -NVML_FI_DEV_ECC_SBE_VOL_TEX = 15 # Texture memory single bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_TEX = 16 # Texture memory double bit volatile ECC errors -NVML_FI_DEV_ECC_DBE_VOL_CBU = 17 # CBU double bit volatile ECC errors -NVML_FI_DEV_ECC_SBE_AGG_L1 = 18 # L1 cache single bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_L1 = 19 # L1 cache double bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_SBE_AGG_L2 = 20 # L2 cache single bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_L2 = 21 # L2 cache double bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_SBE_AGG_DEV = 22 # Device memory single bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_DEV = 23 # Device memory double bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_SBE_AGG_REG = 24 # Register File single bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_REG = 25 # Register File double bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_SBE_AGG_TEX = 26 # Texture memory single bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_TEX = 27 # Texture memory double bit aggregate (persistent) ECC errors -NVML_FI_DEV_ECC_DBE_AGG_CBU = 28 # CBU double bit aggregate ECC errors +# ECC Count Totals +NVML_FI_DEV_ECC_SBE_VOL_TOTAL = 3 # Total single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_TOTAL = 4 # Total double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_AGG_TOTAL = 5 # Total single bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_DBE_AGG_TOTAL = 6 # Total double bit aggregate (persistent) ECC errors +# Individual ECC locations +NVML_FI_DEV_ECC_SBE_VOL_L1 = 7 # L1 cache single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_L1 = 8 # L1 cache double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_L2 = 9 # L2 cache single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_L2 = 10 # L2 cache double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_DEV = 11 # Device memory single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_DEV = 12 # Device memory double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_REG = 13 # Register file single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_REG = 14 # Register file double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_VOL_TEX = 15 # Texture memory single bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_TEX = 16 # Texture memory double bit volatile ECC errors +NVML_FI_DEV_ECC_DBE_VOL_CBU = 17 # CBU double bit volatile ECC errors +NVML_FI_DEV_ECC_SBE_AGG_L1 = 18 # L1 cache single bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_DBE_AGG_L1 = 19 # L1 cache double bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_SBE_AGG_L2 = 20 # L2 cache single bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_DBE_AGG_L2 = 21 # L2 cache double bit aggregate (persistent) ECC errors +NVML_FI_DEV_ECC_SBE_AGG_DEV = ( + 22 # Device memory single bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_DEV = ( + 23 # Device memory double bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_SBE_AGG_REG = ( + 24 # Register File single bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_REG = ( + 25 # Register File double bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_SBE_AGG_TEX = ( + 26 # Texture memory single bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_TEX = ( + 27 # Texture memory double bit aggregate (persistent) ECC errors +) +NVML_FI_DEV_ECC_DBE_AGG_CBU = 28 # CBU double bit aggregate ECC errors # Page Retirement -NVML_FI_DEV_RETIRED_SBE = 29 # Number of retired pages because of single bit errors -NVML_FI_DEV_RETIRED_DBE = 30 # Number of retired pages because of double bit errors -NVML_FI_DEV_RETIRED_PENDING = 31 # If any pages are pending retirement. 1=yes. 0=no. +NVML_FI_DEV_RETIRED_SBE = 29 # Number of retired pages because of single bit errors +NVML_FI_DEV_RETIRED_DBE = 30 # Number of retired pages because of double bit errors +NVML_FI_DEV_RETIRED_PENDING = 31 # If any pages are pending retirement. 1=yes. 0=no. # NvLink Flit Error Counters -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0 = 32 # NVLink flow control CRC Error Counter for Lane 0 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1 = 33 # NVLink flow control CRC Error Counter for Lane 1 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2 = 34 # NVLink flow control CRC Error Counter for Lane 2 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3 = 35 # NVLink flow control CRC Error Counter for Lane 3 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4 = 36 # NVLink flow control CRC Error Counter for Lane 4 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5 = 37 # NVLink flow control CRC Error Counter for Lane 5 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL = 38 # NVLink flow control CRC Error Counter total for all Lanes +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L0 = ( + 32 # NVLink flow control CRC Error Counter for Lane 0 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L1 = ( + 33 # NVLink flow control CRC Error Counter for Lane 1 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L2 = ( + 34 # NVLink flow control CRC Error Counter for Lane 2 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L3 = ( + 35 # NVLink flow control CRC Error Counter for Lane 3 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L4 = ( + 36 # NVLink flow control CRC Error Counter for Lane 4 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L5 = ( + 37 # NVLink flow control CRC Error Counter for Lane 5 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL = ( + 38 # NVLink flow control CRC Error Counter total for all Lanes +) # NvLink CRC Data Error Counters -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0 = 39 # NVLink data CRC Error Counter for Lane 0 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1 = 40 # NVLink data CRC Error Counter for Lane 1 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2 = 41 # NVLink data CRC Error Counter for Lane 2 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3 = 42 # NVLink data CRC Error Counter for Lane 3 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4 = 43 # NVLink data CRC Error Counter for Lane 4 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5 = 44 # NVLink data CRC Error Counter for Lane 5 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL = 45 # NvLink data CRC Error Counter total for all Lanes +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L0 = ( + 39 # NVLink data CRC Error Counter for Lane 0 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L1 = ( + 40 # NVLink data CRC Error Counter for Lane 1 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L2 = ( + 41 # NVLink data CRC Error Counter for Lane 2 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L3 = ( + 42 # NVLink data CRC Error Counter for Lane 3 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L4 = ( + 43 # NVLink data CRC Error Counter for Lane 4 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L5 = ( + 44 # NVLink data CRC Error Counter for Lane 5 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL = ( + 45 # NvLink data CRC Error Counter total for all Lanes +) # NvLink Replay Error Counters -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0 = 46 # NVLink Replay Error Counter for Lane 0 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1 = 47 # NVLink Replay Error Counter for Lane 1 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2 = 48 # NVLink Replay Error Counter for Lane 2 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3 = 49 # NVLink Replay Error Counter for Lane 3 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4 = 50 # NVLink Replay Error Counter for Lane 4 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5 = 51 # NVLink Replay Error Counter for Lane 5 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL = 52 # NVLink Replay Error Counter total for all Lanes +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L0 = 46 # NVLink Replay Error Counter for Lane 0 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L1 = 47 # NVLink Replay Error Counter for Lane 1 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L2 = 48 # NVLink Replay Error Counter for Lane 2 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L3 = 49 # NVLink Replay Error Counter for Lane 3 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L4 = 50 # NVLink Replay Error Counter for Lane 4 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L5 = 51 # NVLink Replay Error Counter for Lane 5 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL = ( + 52 # NVLink Replay Error Counter total for all Lanes +) # NvLink Recovery Error Counters -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0 = 53 # NVLink Recovery Error Counter for Lane 0 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1 = 54 # NVLink Recovery Error Counter for Lane 1 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2 = 55 # NVLink Recovery Error Counter for Lane 2 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3 = 56 # NVLink Recovery Error Counter for Lane 3 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4 = 57 # NVLink Recovery Error Counter for Lane 4 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5 = 58 # NVLink Recovery Error Counter for Lane 5 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL = 59 # NVLink Recovery Error Counter total for all Lanes +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L0 = ( + 53 # NVLink Recovery Error Counter for Lane 0 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L1 = ( + 54 # NVLink Recovery Error Counter for Lane 1 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L2 = ( + 55 # NVLink Recovery Error Counter for Lane 2 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L3 = ( + 56 # NVLink Recovery Error Counter for Lane 3 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L4 = ( + 57 # NVLink Recovery Error Counter for Lane 4 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L5 = ( + 58 # NVLink Recovery Error Counter for Lane 5 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL = ( + 59 # NVLink Recovery Error Counter total for all Lanes +) # NvLink Bandwidth Counters -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L0 = 60 # NVLink Bandwidth Counter for Counter Set 0, Lane 0 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L1 = 61 # NVLink Bandwidth Counter for Counter Set 0, Lane 1 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L2 = 62 # NVLink Bandwidth Counter for Counter Set 0, Lane 2 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L3 = 63 # NVLink Bandwidth Counter for Counter Set 0, Lane 3 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L4 = 64 # NVLink Bandwidth Counter for Counter Set 0, Lane 4 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L5 = 65 # NVLink Bandwidth Counter for Counter Set 0, Lane 5 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_TOTAL = 66 # NVLink Bandwidth Counter Total for Counter Set 0, All Lanes +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L0 = ( + 60 # NVLink Bandwidth Counter for Counter Set 0, Lane 0 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L1 = ( + 61 # NVLink Bandwidth Counter for Counter Set 0, Lane 1 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L2 = ( + 62 # NVLink Bandwidth Counter for Counter Set 0, Lane 2 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L3 = ( + 63 # NVLink Bandwidth Counter for Counter Set 0, Lane 3 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L4 = ( + 64 # NVLink Bandwidth Counter for Counter Set 0, Lane 4 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L5 = ( + 65 # NVLink Bandwidth Counter for Counter Set 0, Lane 5 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_TOTAL = ( + 66 # NVLink Bandwidth Counter Total for Counter Set 0, All Lanes +) # NvLink Bandwidth Counters -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L0 = 67 # NVLink Bandwidth Counter for Counter Set 1, Lane 0 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L1 = 68 # NVLink Bandwidth Counter for Counter Set 1, Lane 1 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L2 = 69 # NVLink Bandwidth Counter for Counter Set 1, Lane 2 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L3 = 70 # NVLink Bandwidth Counter for Counter Set 1, Lane 3 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L4 = 71 # NVLink Bandwidth Counter for Counter Set 1, Lane 4 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L5 = 72 # NVLink Bandwidth Counter for Counter Set 1, Lane 5 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_TOTAL = 73 # NVLink Bandwidth Counter Total for Counter Set 1, All Lanes +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L0 = ( + 67 # NVLink Bandwidth Counter for Counter Set 1, Lane 0 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L1 = ( + 68 # NVLink Bandwidth Counter for Counter Set 1, Lane 1 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L2 = ( + 69 # NVLink Bandwidth Counter for Counter Set 1, Lane 2 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L3 = ( + 70 # NVLink Bandwidth Counter for Counter Set 1, Lane 3 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L4 = ( + 71 # NVLink Bandwidth Counter for Counter Set 1, Lane 4 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L5 = ( + 72 # NVLink Bandwidth Counter for Counter Set 1, Lane 5 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_TOTAL = ( + 73 # NVLink Bandwidth Counter Total for Counter Set 1, All Lanes +) # Perf Policy Counters -NVML_FI_DEV_PERF_POLICY_POWER = 74 # Perf Policy Counter for Power Policy -NVML_FI_DEV_PERF_POLICY_THERMAL = 75 # Perf Policy Counter for Thermal Policy -NVML_FI_DEV_PERF_POLICY_SYNC_BOOST = 76 # Perf Policy Counter for Sync boost Policy -NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT = 77 # Perf Policy Counter for Board Limit -NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION = 78 # Perf Policy Counter for Low GPU Utilization Policy -NVML_FI_DEV_PERF_POLICY_RELIABILITY = 79 # Perf Policy Counter for Reliability Policy -NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS = 80 # Perf Policy Counter for Total App Clock Policy -NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS = 81 # Perf Policy Counter for Total Base Clocks Policy +NVML_FI_DEV_PERF_POLICY_POWER = 74 # Perf Policy Counter for Power Policy +NVML_FI_DEV_PERF_POLICY_THERMAL = 75 # Perf Policy Counter for Thermal Policy +NVML_FI_DEV_PERF_POLICY_SYNC_BOOST = 76 # Perf Policy Counter for Sync boost Policy +NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT = 77 # Perf Policy Counter for Board Limit +NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION = ( + 78 # Perf Policy Counter for Low GPU Utilization Policy +) +NVML_FI_DEV_PERF_POLICY_RELIABILITY = 79 # Perf Policy Counter for Reliability Policy +NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS = ( + 80 # Perf Policy Counter for Total App Clock Policy +) +NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS = ( + 81 # Perf Policy Counter for Total Base Clocks Policy +) # Memory temperatures -NVML_FI_DEV_MEMORY_TEMP = 82 # Memory temperature for the device +NVML_FI_DEV_MEMORY_TEMP = 82 # Memory temperature for the device # Energy Counter -NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION = 83 # Total energy consumption for the GPU in mJ since the driver was last reloaded +NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION = ( + 83 # Total energy consumption for the GPU in mJ since the driver was last reloaded +) # NVLink Speed -NVML_FI_DEV_NVLINK_SPEED_MBPS_L0 = 84 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L1 = 85 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L2 = 86 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L3 = 87 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L4 = 88 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L5 = 89 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L0 = 84 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L1 = 85 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L2 = 86 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L3 = 87 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L4 = 88 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L5 = 89 NVML_FI_DEV_NVLINK_SPEED_MBPS_COMMON = 90 # NVLink Link Count @@ -622,307 +718,451 @@ NVML_FI_DEV_PCIE_REPLAY_COUNTER = 94 NVML_FI_DEV_PCIE_REPLAY_ROLLOVER_COUNTER = 95 # NvLink Flit Error Counters -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6 = 96 # NVLink flow control CRC Error Counter for Lane 6 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7 = 97 # NVLink flow control CRC Error Counter for Lane 7 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8 = 98 # NVLink flow control CRC Error Counter for Lane 8 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9 = 99 # NVLink flow control CRC Error Counter for Lane 9 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10 = 100 # NVLink flow control CRC Error Counter for Lane 10 -NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11 = 101 # NVLink flow control CRC Error Counter for Lane 11 +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L6 = ( + 96 # NVLink flow control CRC Error Counter for Lane 6 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L7 = ( + 97 # NVLink flow control CRC Error Counter for Lane 7 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L8 = ( + 98 # NVLink flow control CRC Error Counter for Lane 8 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L9 = ( + 99 # NVLink flow control CRC Error Counter for Lane 9 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L10 = ( + 100 # NVLink flow control CRC Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_L11 = ( + 101 # NVLink flow control CRC Error Counter for Lane 11 +) # NvLink CRC Data Error Counters -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6 = 102 # NVLink data CRC Error Counter for Lane 6 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7 = 103 # NVLink data CRC Error Counter for Lane 7 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8 = 104 # NVLink data CRC Error Counter for Lane 8 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9 = 105 # NVLink data CRC Error Counter for Lane 9 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10 = 106 # NVLink data CRC Error Counter for Lane 10 -NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11 = 107 # NVLink data CRC Error Counter for Lane 11 +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L6 = ( + 102 # NVLink data CRC Error Counter for Lane 6 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L7 = ( + 103 # NVLink data CRC Error Counter for Lane 7 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L8 = ( + 104 # NVLink data CRC Error Counter for Lane 8 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L9 = ( + 105 # NVLink data CRC Error Counter for Lane 9 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L10 = ( + 106 # NVLink data CRC Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_L11 = ( + 107 # NVLink data CRC Error Counter for Lane 11 +) # NvLink Replay Error Counters -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6 = 108 # NVLink Replay Error Counter for Lane 6 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7 = 109 # NVLink Replay Error Counter for Lane 7 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8 = 110 # NVLink Replay Error Counter for Lane 8 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9 = 111 # NVLink Replay Error Counter for Lane 9 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10 = 112 # NVLink Replay Error Counter for Lane 10 -NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11 = 113 # NVLink Replay Error Counter for Lane 11 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L6 = 108 # NVLink Replay Error Counter for Lane 6 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L7 = 109 # NVLink Replay Error Counter for Lane 7 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L8 = 110 # NVLink Replay Error Counter for Lane 8 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L9 = 111 # NVLink Replay Error Counter for Lane 9 +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L10 = ( + 112 # NVLink Replay Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_L11 = ( + 113 # NVLink Replay Error Counter for Lane 11 +) # NvLink Recovery Error Counters -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6 = 114 # NVLink Recovery Error Counter for Lane 6 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7 = 115 # NVLink Recovery Error Counter for Lane 7 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8 = 116 # NVLink Recovery Error Counter for Lane 8 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9 = 117 # NVLink Recovery Error Counter for Lane 9 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10 = 118 # NVLink Recovery Error Counter for Lane 10 -NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11 = 119 # NVLink Recovery Error Counter for Lane 11 +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L6 = ( + 114 # NVLink Recovery Error Counter for Lane 6 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L7 = ( + 115 # NVLink Recovery Error Counter for Lane 7 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L8 = ( + 116 # NVLink Recovery Error Counter for Lane 8 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L9 = ( + 117 # NVLink Recovery Error Counter for Lane 9 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L10 = ( + 118 # NVLink Recovery Error Counter for Lane 10 +) +NVML_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_L11 = ( + 119 # NVLink Recovery Error Counter for Lane 11 +) # NvLink Bandwidth Counters -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L6 = 120 # NVLink Bandwidth Counter for Counter Set 0, Lane 6 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L7 = 121 # NVLink Bandwidth Counter for Counter Set 0, Lane 7 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L8 = 122 # NVLink Bandwidth Counter for Counter Set 0, Lane 8 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L9 = 123 # NVLink Bandwidth Counter for Counter Set 0, Lane 9 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L10 = 124 # NVLink Bandwidth Counter for Counter Set 0, Lane 10 -NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L11 = 125 # NVLink Bandwidth Counter for Counter Set 0, Lane 11 +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L6 = ( + 120 # NVLink Bandwidth Counter for Counter Set 0, Lane 6 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L7 = ( + 121 # NVLink Bandwidth Counter for Counter Set 0, Lane 7 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L8 = ( + 122 # NVLink Bandwidth Counter for Counter Set 0, Lane 8 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L9 = ( + 123 # NVLink Bandwidth Counter for Counter Set 0, Lane 9 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L10 = ( + 124 # NVLink Bandwidth Counter for Counter Set 0, Lane 10 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C0_L11 = ( + 125 # NVLink Bandwidth Counter for Counter Set 0, Lane 11 +) # NvLink Bandwidth Counters -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L6 = 126 # NVLink Bandwidth Counter for Counter Set 1, Lane 6 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L7 = 127 # NVLink Bandwidth Counter for Counter Set 1, Lane 7 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L8 = 128 # NVLink Bandwidth Counter for Counter Set 1, Lane 8 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L9 = 129 # NVLink Bandwidth Counter for Counter Set 1, Lane 9 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L10 = 130 # NVLink Bandwidth Counter for Counter Set 1, Lane 10 -NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L11 = 131 # NVLink Bandwidth Counter for Counter Set 1, Lane 11 +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L6 = ( + 126 # NVLink Bandwidth Counter for Counter Set 1, Lane 6 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L7 = ( + 127 # NVLink Bandwidth Counter for Counter Set 1, Lane 7 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L8 = ( + 128 # NVLink Bandwidth Counter for Counter Set 1, Lane 8 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L9 = ( + 129 # NVLink Bandwidth Counter for Counter Set 1, Lane 9 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L10 = ( + 130 # NVLink Bandwidth Counter for Counter Set 1, Lane 10 +) +NVML_FI_DEV_NVLINK_BANDWIDTH_C1_L11 = ( + 131 # NVLink Bandwidth Counter for Counter Set 1, Lane 11 +) # NVLink Speed -NVML_FI_DEV_NVLINK_SPEED_MBPS_L6 = 132 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L7 = 133 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L8 = 134 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L9 = 135 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L10 = 136 -NVML_FI_DEV_NVLINK_SPEED_MBPS_L11 = 137 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L6 = 132 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L7 = 133 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L8 = 134 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L9 = 135 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L10 = 136 +NVML_FI_DEV_NVLINK_SPEED_MBPS_L11 = 137 # NVLink Throughput Counters -NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX = 138 # NVLink TX Data throughput in KiB -NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX = 139 # NVLink RX Data throughput in KiB -NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_TX = 140 # NVLink TX Data + protocol overhead in KiB -NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_RX = 141 # NVLink RX Data + protocol overhead in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX = 138 # NVLink TX Data throughput in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX = 139 # NVLink RX Data throughput in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_TX = 140 # NVLink TX Data + protocol overhead in KiB +NVML_FI_DEV_NVLINK_THROUGHPUT_RAW_RX = 141 # NVLink RX Data + protocol overhead in KiB # Row Remapper -NVML_FI_DEV_REMAPPED_COR = 142 -NVML_FI_DEV_REMAPPED_UNC = 143 -NVML_FI_DEV_REMAPPED_PENDING = 144 -NVML_FI_DEV_REMAPPED_FAILURE = 145 +NVML_FI_DEV_REMAPPED_COR = 142 +NVML_FI_DEV_REMAPPED_UNC = 143 +NVML_FI_DEV_REMAPPED_PENDING = 144 +NVML_FI_DEV_REMAPPED_FAILURE = 145 -#Remote device NVLink ID +# Remote device NVLink ID NVML_FI_DEV_NVLINK_REMOTE_NVLINK_ID = 146 # Number of NVLinks connected to NVSwitch NVML_FI_DEV_NVSWITCH_CONNECTED_LINK_COUNT = 147 # NvLink ECC Data Error Counters -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L0 = 148 #< NVLink data ECC Error Counter for Link 0 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L1 = 149 #< NVLink data ECC Error Counter for Link 1 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L2 = 150 #< NVLink data ECC Error Counter for Link 2 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L3 = 151 #< NVLink data ECC Error Counter for Link 3 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L4 = 152 #< NVLink data ECC Error Counter for Link 4 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L5 = 153 #< NVLink data ECC Error Counter for Link 5 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L6 = 154 #< NVLink data ECC Error Counter for Link 6 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L7 = 155 #< NVLink data ECC Error Counter for Link 7 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L8 = 156 #< NVLink data ECC Error Counter for Link 8 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L9 = 157 #< NVLink data ECC Error Counter for Link 9 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L10 = 158 #< NVLink data ECC Error Counter for Link 10 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L11 = 159 #< NVLink data ECC Error Counter for Link 11 -NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL = 160 #< NvLink data ECC Error Counter total for all Links +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L0 = ( + 148 # < NVLink data ECC Error Counter for Link 0 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L1 = ( + 149 # < NVLink data ECC Error Counter for Link 1 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L2 = ( + 150 # < NVLink data ECC Error Counter for Link 2 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L3 = ( + 151 # < NVLink data ECC Error Counter for Link 3 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L4 = ( + 152 # < NVLink data ECC Error Counter for Link 4 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L5 = ( + 153 # < NVLink data ECC Error Counter for Link 5 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L6 = ( + 154 # < NVLink data ECC Error Counter for Link 6 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L7 = ( + 155 # < NVLink data ECC Error Counter for Link 7 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L8 = ( + 156 # < NVLink data ECC Error Counter for Link 8 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L9 = ( + 157 # < NVLink data ECC Error Counter for Link 9 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L10 = ( + 158 # < NVLink data ECC Error Counter for Link 10 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_L11 = ( + 159 # < NVLink data ECC Error Counter for Link 11 +) +NVML_FI_DEV_NVLINK_ECC_DATA_ERROR_COUNT_TOTAL = ( + 160 # < NvLink data ECC Error Counter total for all Links +) -NVML_FI_DEV_NVLINK_ERROR_DL_REPLAY = 161 -NVML_FI_DEV_NVLINK_ERROR_DL_RECOVERY = 162 -NVML_FI_DEV_NVLINK_ERROR_DL_CRC = 163 -NVML_FI_DEV_NVLINK_GET_SPEED = 164 -NVML_FI_DEV_NVLINK_GET_STATE = 165 -NVML_FI_DEV_NVLINK_GET_VERSION = 166 +NVML_FI_DEV_NVLINK_ERROR_DL_REPLAY = 161 +NVML_FI_DEV_NVLINK_ERROR_DL_RECOVERY = 162 +NVML_FI_DEV_NVLINK_ERROR_DL_CRC = 163 +NVML_FI_DEV_NVLINK_GET_SPEED = 164 +NVML_FI_DEV_NVLINK_GET_STATE = 165 +NVML_FI_DEV_NVLINK_GET_VERSION = 166 -NVML_FI_DEV_NVLINK_GET_POWER_STATE = 167 -NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD = 168 +NVML_FI_DEV_NVLINK_GET_POWER_STATE = 167 +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD = 168 -NVML_FI_DEV_PCIE_L0_TO_RECOVERY_COUNTER = 169 +NVML_FI_DEV_PCIE_L0_TO_RECOVERY_COUNTER = 169 -NVML_FI_DEV_C2C_LINK_COUNT = 170 -NVML_FI_DEV_C2C_LINK_GET_STATUS = 171 -NVML_FI_DEV_C2C_LINK_GET_MAX_BW = 172 +NVML_FI_DEV_C2C_LINK_COUNT = 170 +NVML_FI_DEV_C2C_LINK_GET_STATUS = 171 +NVML_FI_DEV_C2C_LINK_GET_MAX_BW = 172 -NVML_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS = 173 -NVML_FI_DEV_PCIE_COUNT_NAKS_RECEIVED = 174 -NVML_FI_DEV_PCIE_COUNT_RECEIVER_ERROR = 175 -NVML_FI_DEV_PCIE_COUNT_BAD_TLP = 176 -NVML_FI_DEV_PCIE_COUNT_NAKS_SENT = 177 -NVML_FI_DEV_PCIE_COUNT_BAD_DLLP = 178 -NVML_FI_DEV_PCIE_COUNT_NON_FATAL_ERROR = 179 -NVML_FI_DEV_PCIE_COUNT_FATAL_ERROR = 180 -NVML_FI_DEV_PCIE_COUNT_UNSUPPORTED_REQ = 181 -NVML_FI_DEV_PCIE_COUNT_LCRC_ERROR = 182 -NVML_FI_DEV_PCIE_COUNT_LANE_ERROR = 183 +NVML_FI_DEV_PCIE_COUNT_CORRECTABLE_ERRORS = 173 +NVML_FI_DEV_PCIE_COUNT_NAKS_RECEIVED = 174 +NVML_FI_DEV_PCIE_COUNT_RECEIVER_ERROR = 175 +NVML_FI_DEV_PCIE_COUNT_BAD_TLP = 176 +NVML_FI_DEV_PCIE_COUNT_NAKS_SENT = 177 +NVML_FI_DEV_PCIE_COUNT_BAD_DLLP = 178 +NVML_FI_DEV_PCIE_COUNT_NON_FATAL_ERROR = 179 +NVML_FI_DEV_PCIE_COUNT_FATAL_ERROR = 180 +NVML_FI_DEV_PCIE_COUNT_UNSUPPORTED_REQ = 181 +NVML_FI_DEV_PCIE_COUNT_LCRC_ERROR = 182 +NVML_FI_DEV_PCIE_COUNT_LANE_ERROR = 183 -NVML_FI_DEV_IS_RESETLESS_MIG_SUPPORTED = 184 +NVML_FI_DEV_IS_RESETLESS_MIG_SUPPORTED = 184 -NVML_FI_DEV_POWER_AVERAGE = 185 -NVML_FI_DEV_POWER_INSTANT = 186 -NVML_FI_DEV_POWER_MIN_LIMIT = 187 -NVML_FI_DEV_POWER_MAX_LIMIT = 188 -NVML_FI_DEV_POWER_DEFAULT_LIMIT = 189 -NVML_FI_DEV_POWER_CURRENT_LIMIT = 190 -NVML_FI_DEV_ENERGY = 191 -NVML_FI_DEV_POWER_REQUESTED_LIMIT = 192 +NVML_FI_DEV_POWER_AVERAGE = 185 +NVML_FI_DEV_POWER_INSTANT = 186 +NVML_FI_DEV_POWER_MIN_LIMIT = 187 +NVML_FI_DEV_POWER_MAX_LIMIT = 188 +NVML_FI_DEV_POWER_DEFAULT_LIMIT = 189 +NVML_FI_DEV_POWER_CURRENT_LIMIT = 190 +NVML_FI_DEV_ENERGY = 191 +NVML_FI_DEV_POWER_REQUESTED_LIMIT = 192 -NVML_FI_DEV_TEMPERATURE_SHUTDOWN_TLIMIT = 193 -NVML_FI_DEV_TEMPERATURE_SLOWDOWN_TLIMIT = 194 -NVML_FI_DEV_TEMPERATURE_MEM_MAX_TLIMIT = 195 -NVML_FI_DEV_TEMPERATURE_GPU_MAX_TLIMIT = 196 +NVML_FI_DEV_TEMPERATURE_SHUTDOWN_TLIMIT = 193 +NVML_FI_DEV_TEMPERATURE_SLOWDOWN_TLIMIT = 194 +NVML_FI_DEV_TEMPERATURE_MEM_MAX_TLIMIT = 195 +NVML_FI_DEV_TEMPERATURE_GPU_MAX_TLIMIT = 196 -NVML_FI_DEV_PCIE_COUNT_TX_BYTES = 197 -NVML_FI_DEV_PCIE_COUNT_RX_BYTES = 198 +NVML_FI_DEV_PCIE_COUNT_TX_BYTES = 197 +NVML_FI_DEV_PCIE_COUNT_RX_BYTES = 198 -NVML_FI_DEV_IS_MIG_MODE_INDEPENDENT_MIG_QUERY_CAPABLE = 199 +NVML_FI_DEV_IS_MIG_MODE_INDEPENDENT_MIG_QUERY_CAPABLE = 199 -NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MAX = 200 +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MAX = 200 -NVML_FI_DEV_NVLINK_COUNT_XMIT_PACKETS = 201 -NVML_FI_DEV_NVLINK_COUNT_XMIT_BYTES = 202 -NVML_FI_DEV_NVLINK_COUNT_RCV_PACKETS = 203 -NVML_FI_DEV_NVLINK_COUNT_RCV_BYTES = 204 -NVML_FI_DEV_NVLINK_COUNT_VL15_DROPPED = 205 # Deprecated, do not use -NVML_FI_DEV_NVLINK_COUNT_MALFORMED_PACKET_ERRORS = 206 -NVML_FI_DEV_NVLINK_COUNT_BUFFER_OVERRUN_ERRORS = 207 -NVML_FI_DEV_NVLINK_COUNT_RCV_ERRORS = 208 -NVML_FI_DEV_NVLINK_COUNT_RCV_REMOTE_ERRORS = 209 -NVML_FI_DEV_NVLINK_COUNT_RCV_GENERAL_ERRORS = 210 -NVML_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS = 211 -NVML_FI_DEV_NVLINK_COUNT_XMIT_DISCARDS = 212 +NVML_FI_DEV_NVLINK_COUNT_XMIT_PACKETS = 201 +NVML_FI_DEV_NVLINK_COUNT_XMIT_BYTES = 202 +NVML_FI_DEV_NVLINK_COUNT_RCV_PACKETS = 203 +NVML_FI_DEV_NVLINK_COUNT_RCV_BYTES = 204 +NVML_FI_DEV_NVLINK_COUNT_VL15_DROPPED = 205 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_MALFORMED_PACKET_ERRORS = 206 +NVML_FI_DEV_NVLINK_COUNT_BUFFER_OVERRUN_ERRORS = 207 +NVML_FI_DEV_NVLINK_COUNT_RCV_ERRORS = 208 +NVML_FI_DEV_NVLINK_COUNT_RCV_REMOTE_ERRORS = 209 +NVML_FI_DEV_NVLINK_COUNT_RCV_GENERAL_ERRORS = 210 +NVML_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS = 211 +NVML_FI_DEV_NVLINK_COUNT_XMIT_DISCARDS = 212 NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_SUCCESSFUL_EVENTS = 213 -NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS = 214 -NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS = 215 +NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS = 214 +NVML_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_EVENTS = 215 -NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE0 = 216 # Deprecated, do not use -NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE1 = 217 # Deprecated, do not use -NVML_FI_DEV_NVLINK_COUNT_RAW_BER = 218 # Deprecated, do not use -NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS = 219 -NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER = 220 -NVML_FI_DEV_NVLINK_COUNT_SYMBOL_ERRORS = 221 -NVML_FI_DEV_NVLINK_COUNT_SYMBOL_BER = 222 +NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE0 = 216 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_RAW_BER_LANE1 = 217 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_RAW_BER = 218 # Deprecated, do not use +NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_ERRORS = 219 +NVML_FI_DEV_NVLINK_COUNT_EFFECTIVE_BER = 220 +NVML_FI_DEV_NVLINK_COUNT_SYMBOL_ERRORS = 221 +NVML_FI_DEV_NVLINK_COUNT_SYMBOL_BER = 222 -NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MIN = 223 -NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS = 224 # Values are in the form NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_* -NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_SUPPORTED = 225 +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_MIN = 223 +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS = ( + 224 # Values are in the form NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_* +) +NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_SUPPORTED = 225 -NVML_FI_DEV_RESET_STATUS = 226 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead -NVML_FI_DEV_DRAIN_AND_RESET_STATUS = 227 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead -NVML_FI_DEV_PCIE_OUTBOUND_ATOMICS_MASK = 228 -NVML_FI_DEV_PCIE_INBOUND_ATOMICS_MASK = 229 -NVML_FI_DEV_GET_GPU_RECOVERY_ACTION = 230 +NVML_FI_DEV_RESET_STATUS = ( + 226 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead +) +NVML_FI_DEV_DRAIN_AND_RESET_STATUS = ( + 227 # Deprecated use NVML_FI_DEV_GET_GPU_RECOVERY_ACTION instead +) +NVML_FI_DEV_PCIE_OUTBOUND_ATOMICS_MASK = 228 +NVML_FI_DEV_PCIE_INBOUND_ATOMICS_MASK = 229 +NVML_FI_DEV_GET_GPU_RECOVERY_ACTION = 230 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0 = 235 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1 = 236 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2 = 237 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3 = 238 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4 = 239 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5 = 240 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6 = 241 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7 = 242 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8 = 243 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9 = 244 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10 = 245 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11 = 246 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12 = 247 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13 = 248 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14 = 249 -NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15 = 250 -NVML_FI_PWR_SMOOTHING_ENABLED = 251 # Enablement (0/DISABLED or 1/ENABLED) -NVML_FI_PWR_SMOOTHING_PRIV_LVL = 252 # Current privilege level -NVML_FI_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED = 253 # Immediate ramp down enablement (0/DISABLED or 1/ENABLED) -NVML_FI_PWR_SMOOTHING_APPLIED_TMP_CEIL = 254 # Applied TMP ceiling value -NVML_FI_PWR_SMOOTHING_APPLIED_TMP_FLOOR = 255 # Applied TMP floor value -NVML_FI_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING = 256 # Max % TMP Floor value -NVML_FI_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING = 257 # Min % TMP Floor value -NVML_FI_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING = 258 # HW Circuitry % lifetime remaining -NVML_FI_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES = 259 # Max number of preset profiles -NVML_FI_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR = 260 # % TMP floor for a given profile -NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE = 261 # Ramp up rate in mW/s for a given profile -NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE = 262 # Ramp down rate in mW/s for a given profile -NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL = 263 # Ramp down hysteresis value in ms for a given profile -NVML_FI_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE = 264 # Active preset profile number -NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR = 265 # % TMP floor for a given profile -NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE = 266 # Ramp up rate in mW/s for a given profile -NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE = 267 # Ramp down rate in mW/s for a given profile -NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL = 268 # Ramp down hysteresis value in ms for a given profile +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_0 = 235 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_1 = 236 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_2 = 237 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_3 = 238 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_4 = 239 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_5 = 240 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_6 = 241 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_7 = 242 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_8 = 243 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_9 = 244 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_10 = 245 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_11 = 246 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_12 = 247 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_13 = 248 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_14 = 249 +NVML_FI_DEV_NVLINK_COUNT_FEC_HISTORY_15 = 250 +NVML_FI_PWR_SMOOTHING_ENABLED = 251 # Enablement (0/DISABLED or 1/ENABLED) +NVML_FI_PWR_SMOOTHING_PRIV_LVL = 252 # Current privilege level +NVML_FI_PWR_SMOOTHING_IMM_RAMP_DOWN_ENABLED = ( + 253 # Immediate ramp down enablement (0/DISABLED or 1/ENABLED) +) +NVML_FI_PWR_SMOOTHING_APPLIED_TMP_CEIL = 254 # Applied TMP ceiling value +NVML_FI_PWR_SMOOTHING_APPLIED_TMP_FLOOR = 255 # Applied TMP floor value +NVML_FI_PWR_SMOOTHING_MAX_PERCENT_TMP_FLOOR_SETTING = 256 # Max % TMP Floor value +NVML_FI_PWR_SMOOTHING_MIN_PERCENT_TMP_FLOOR_SETTING = 257 # Min % TMP Floor value +NVML_FI_PWR_SMOOTHING_HW_CIRCUITRY_PERCENT_LIFETIME_REMAINING = ( + 258 # HW Circuitry % lifetime remaining +) +NVML_FI_PWR_SMOOTHING_MAX_NUM_PRESET_PROFILES = 259 # Max number of preset profiles +NVML_FI_PWR_SMOOTHING_PROFILE_PERCENT_TMP_FLOOR = 260 # % TMP floor for a given profile +NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_UP_RATE = ( + 261 # Ramp up rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_RATE = ( + 262 # Ramp down rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_PROFILE_RAMP_DOWN_HYST_VAL = ( + 263 # Ramp down hysteresis value in ms for a given profile +) +NVML_FI_PWR_SMOOTHING_ACTIVE_PRESET_PROFILE = 264 # Active preset profile number +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_PERCENT_TMP_FLOOR = ( + 265 # % TMP floor for a given profile +) +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_UP_RATE = ( + 266 # Ramp up rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_RATE = ( + 267 # Ramp down rate in mW/s for a given profile +) +NVML_FI_PWR_SMOOTHING_ADMIN_OVERRIDE_RAMP_DOWN_HYST_VAL = ( + 268 # Ramp down hysteresis value in ms for a given profile +) -NVML_FI_MAX = 269 # One greater than the largest field ID defined above +NVML_FI_MAX = 269 # One greater than the largest field ID defined above # NVML_FI_DEV_NVLINK_GET_STATE state enums NVML_NVLINK_STATE_INACTIVE = 0x0 -NVML_NVLINK_STATE_ACTIVE = 0x1 -NVML_NVLINK_STATE_SLEEP = 0x2 +NVML_NVLINK_STATE_ACTIVE = 0x1 +NVML_NVLINK_STATE_SLEEP = 0x2 -NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_100US = 0 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS -NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_50US = 1 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS +NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_100US = ( + 0 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS +) +NVML_NVLINK_LOW_POWER_THRESHOLD_UNIT_50US = ( + 1 # NVML_FI_DEV_NVLINK_GET_POWER_THRESHOLD_UNITS +) ## Enums needed for the method nvmlDeviceGetVirtualizationMode and nvmlDeviceSetVirtualizationMode -NVML_GPU_VIRTUALIZATION_MODE_NONE = 0 # Represents Bare Metal GPU -NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH = 1 # Device is associated with GPU-Passthorugh -NVML_GPU_VIRTUALIZATION_MODE_VGPU = 2 # Device is associated with vGPU inside virtual machine. -NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU = 3 # Device is associated with VGX hypervisor in vGPU mode -NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA = 4 # Device is associated with VGX hypervisor in vSGA mode +NVML_GPU_VIRTUALIZATION_MODE_NONE = 0 # Represents Bare Metal GPU +NVML_GPU_VIRTUALIZATION_MODE_PASSTHROUGH = ( + 1 # Device is associated with GPU-Passthorugh +) +NVML_GPU_VIRTUALIZATION_MODE_VGPU = ( + 2 # Device is associated with vGPU inside virtual machine. +) +NVML_GPU_VIRTUALIZATION_MODE_HOST_VGPU = ( + 3 # Device is associated with VGX hypervisor in vGPU mode +) +NVML_GPU_VIRTUALIZATION_MODE_HOST_VSGA = ( + 4 # Device is associated with VGX hypervisor in vSGA mode +) ## Lib loading ## nvmlLib = None libLoadLock = threading.Lock() -_nvmlLib_refcount = 0 # Incremented on each nvmlInit and decremented on nvmlShutdown +_nvmlLib_refcount = 0 # Incremented on each nvmlInit and decremented on nvmlShutdown ## vGPU Management -_nvmlVgpuTypeId_t = c_uint +_nvmlVgpuTypeId_t = c_uint _nvmlVgpuInstance_t = c_uint _nvmlVgpuVmIdType_t = c_uint -NVML_VGPU_VM_ID_DOMAIN_ID = 0 -NVML_VGPU_VM_ID_UUID = 1 +NVML_VGPU_VM_ID_DOMAIN_ID = 0 +NVML_VGPU_VM_ID_UUID = 1 _nvmlGridLicenseFeatureCode_t = c_uint -NVML_GRID_LICENSE_FEATURE_CODE_UNKNOWN = 0 -NVML_GRID_LICENSE_FEATURE_CODE_VGPU = 1 -NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX = 2 -NVML_GRID_LICENSE_FEATURE_CODE_VWORKSTATION = 2 # deprecated, use NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX. -NVML_GRID_LICENSE_FEATURE_CODE_GAMING = 3 -NVML_GRID_LICENSE_FEATURE_CODE_COMPUTE = 4 +NVML_GRID_LICENSE_FEATURE_CODE_UNKNOWN = 0 +NVML_GRID_LICENSE_FEATURE_CODE_VGPU = 1 +NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX = 2 +NVML_GRID_LICENSE_FEATURE_CODE_VWORKSTATION = ( + 2 # deprecated, use NVML_GRID_LICENSE_FEATURE_CODE_NVIDIA_RTX. +) +NVML_GRID_LICENSE_FEATURE_CODE_GAMING = 3 +NVML_GRID_LICENSE_FEATURE_CODE_COMPUTE = 4 _nvmlGridLicenseExpiryStatus_t = c_uint8 -NVML_GRID_LICENSE_EXPIRY_NOT_AVAILABLE = 0, # Expiry information not available -NVML_GRID_LICENSE_EXPIRY_INVALID = 1, # Invalid expiry or error fetching expiry -NVML_GRID_LICENSE_EXPIRY_VALID = 2, # Valid expiry -NVML_GRID_LICENSE_EXPIRY_NOT_APPLICABLE = 3, # Expiry not applicable -NVML_GRID_LICENSE_EXPIRY_PERMANENT = 4, # Permanent expiry +NVML_GRID_LICENSE_EXPIRY_NOT_AVAILABLE = (0,) # Expiry information not available +NVML_GRID_LICENSE_EXPIRY_INVALID = (1,) # Invalid expiry or error fetching expiry +NVML_GRID_LICENSE_EXPIRY_VALID = (2,) # Valid expiry +NVML_GRID_LICENSE_EXPIRY_NOT_APPLICABLE = (3,) # Expiry not applicable +NVML_GRID_LICENSE_EXPIRY_PERMANENT = (4,) # Permanent expiry _nvmlVgpuCapability_t = c_uint -NVML_VGPU_CAP_NVLINK_P2P = 0 # vGPU P2P over NVLink is supported -NVML_VGPU_CAP_GPUDIRECT = 1 # GPUDirect capability is supported -NVML_VGPU_CAP_MULTI_VGPU_EXCLUSIVE = 2 # vGPU profile cannot be mixed with other vGPU profiles in same VM -NVML_VGPU_CAP_EXCLUSIVE_TYPE = 3 # vGPU profile cannot run on a GPU alongside other profiles of different type -NVML_VGPU_CAP_EXCLUSIVE_SIZE = 4 # vGPU profile cannot run on a GPU alongside other profiles of different size -NVML_VGPU_CAP_COUNT = 5 +NVML_VGPU_CAP_NVLINK_P2P = 0 # vGPU P2P over NVLink is supported +NVML_VGPU_CAP_GPUDIRECT = 1 # GPUDirect capability is supported +NVML_VGPU_CAP_MULTI_VGPU_EXCLUSIVE = ( + 2 # vGPU profile cannot be mixed with other vGPU profiles in same VM +) +NVML_VGPU_CAP_EXCLUSIVE_TYPE = ( + 3 # vGPU profile cannot run on a GPU alongside other profiles of different type +) +NVML_VGPU_CAP_EXCLUSIVE_SIZE = ( + 4 # vGPU profile cannot run on a GPU alongside other profiles of different size +) +NVML_VGPU_CAP_COUNT = 5 _nvmlVgpuDriverCapability_t = c_uint -NVML_VGPU_DRIVER_CAP_HETEROGENEOUS_MULTI_VGPU = 0 # Supports mixing of different vGPU profiles within one guest VM -NVML_VGPU_DRIVER_CAP_WARM_UPDATE = 1 # Supports FSR and warm update of vGPU host driver without terminating the running guest VM -NVML_VGPU_DRIVER_CAP_COUNT = 2 +NVML_VGPU_DRIVER_CAP_HETEROGENEOUS_MULTI_VGPU = ( + 0 # Supports mixing of different vGPU profiles within one guest VM +) +NVML_VGPU_DRIVER_CAP_WARM_UPDATE = 1 # Supports FSR and warm update of vGPU host driver without terminating the running guest VM +NVML_VGPU_DRIVER_CAP_COUNT = 2 _nvmlDeviceVgpuCapability_t = c_uint -NVML_DEVICE_VGPU_CAP_FRACTIONAL_MULTI_VGPU = 0 # Query whether the fractional vGPU profiles on this GPU can be used in multi-vGPU configurations -NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_PROFILES = 1 # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing types -NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_SIZES = 2 # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing framebuffer sizes -NVML_DEVICE_VGPU_CAP_READ_DEVICE_BUFFER_BW = 3 # Query the GPU's read_device_buffer expected bandwidth capacity in megabytes per second -NVML_DEVICE_VGPU_CAP_WRITE_DEVICE_BUFFER_BW = 4 # Query the GPU's write_device_buffer expected bandwidth capacity in megabytes per second -NVML_DEVICE_VGPU_CAP_DEVICE_STREAMING = 5 # Query whether the vGPU profiles on the GPU supports migration data streaming -NVML_DEVICE_VGPU_CAP_MINI_QUARTER_GPU = 6 # Set/Get support of mini-quarter vGPU profiles -NVML_DEVICE_VGPU_CAP_COMPUTE_MEDIA_ENGINE_GPU = 7 # Set/Get support for compute media engine vGPU profiles -NVML_DEVICE_VGPU_CAP_WARM_UPDATE = 8 # Query whether the GPU supports FSR and warm update -NVML_DEVICE_VGPU_CAP_HOMOGENEOUS_PLACEMENTS = 9 # Query whether the GPU supports reporting of placements of timesliced vGPU profiles with identical framebuffer sizes -NVML_DEVICE_VGPU_CAP_COUNT = 10 +NVML_DEVICE_VGPU_CAP_FRACTIONAL_MULTI_VGPU = 0 # Query whether the fractional vGPU profiles on this GPU can be used in multi-vGPU configurations +NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_PROFILES = 1 # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing types +NVML_DEVICE_VGPU_CAP_HETEROGENEOUS_TIMESLICE_SIZES = 2 # Query whether the GPU supports concurrent execution of timesliced vGPU profiles of differing framebuffer sizes +NVML_DEVICE_VGPU_CAP_READ_DEVICE_BUFFER_BW = 3 # Query the GPU's read_device_buffer expected bandwidth capacity in megabytes per second +NVML_DEVICE_VGPU_CAP_WRITE_DEVICE_BUFFER_BW = 4 # Query the GPU's write_device_buffer expected bandwidth capacity in megabytes per second +NVML_DEVICE_VGPU_CAP_DEVICE_STREAMING = ( + 5 # Query whether the vGPU profiles on the GPU supports migration data streaming +) +NVML_DEVICE_VGPU_CAP_MINI_QUARTER_GPU = ( + 6 # Set/Get support of mini-quarter vGPU profiles +) +NVML_DEVICE_VGPU_CAP_COMPUTE_MEDIA_ENGINE_GPU = ( + 7 # Set/Get support for compute media engine vGPU profiles +) +NVML_DEVICE_VGPU_CAP_WARM_UPDATE = ( + 8 # Query whether the GPU supports FSR and warm update +) +NVML_DEVICE_VGPU_CAP_HOMOGENEOUS_PLACEMENTS = 9 # Query whether the GPU supports reporting of placements of timesliced vGPU profiles with identical framebuffer sizes +NVML_DEVICE_VGPU_CAP_COUNT = 10 _nvmlVgpuGuestInfoState_t = c_uint NVML_VGPU_INSTANCE_GUEST_INFO_STATE_UNINITIALIZED = 0 -NVML_VGPU_INSTANCE_GUEST_INFO_STATE_INITIALIZED = 1 +NVML_VGPU_INSTANCE_GUEST_INFO_STATE_INITIALIZED = 1 _nvmlVgpuVmCompatibility_t = c_uint -NVML_VGPU_VM_COMPATIBILITY_NONE = 0x0 -NVML_VGPU_VM_COMPATIBILITY_COLD = 0x1 -NVML_VGPU_VM_COMPATIBILITY_HIBERNATE = 0x2 -NVML_VGPU_VM_COMPATIBILITY_SLEEP = 0x4 -NVML_VGPU_VM_COMPATIBILITY_LIVE = 0x8 +NVML_VGPU_VM_COMPATIBILITY_NONE = 0x0 +NVML_VGPU_VM_COMPATIBILITY_COLD = 0x1 +NVML_VGPU_VM_COMPATIBILITY_HIBERNATE = 0x2 +NVML_VGPU_VM_COMPATIBILITY_SLEEP = 0x4 +NVML_VGPU_VM_COMPATIBILITY_LIVE = 0x8 _nvmlVgpuPgpuCompatibilityLimitCode_t = c_uint -NVML_VGPU_COMPATIBILITY_LIMIT_NONE = 0x0 -NVML_VGPU_COMPATIBILITY_LIMIT_HOST_DRIVER = 0x1 -NVML_VGPU_COMPATIBILITY_LIMIT_GUEST_DRIVER = 0x2 -NVML_VGPU_COMPATIBILITY_LIMIT_GPU = 0x4 -NVML_VGPU_COMPATIBILITY_LIMIT_OTHER = 0x80000000 +NVML_VGPU_COMPATIBILITY_LIMIT_NONE = 0x0 +NVML_VGPU_COMPATIBILITY_LIMIT_HOST_DRIVER = 0x1 +NVML_VGPU_COMPATIBILITY_LIMIT_GUEST_DRIVER = 0x2 +NVML_VGPU_COMPATIBILITY_LIMIT_GPU = 0x4 +NVML_VGPU_COMPATIBILITY_LIMIT_OTHER = 0x80000000 _nvmlHostVgpuMode_t = c_uint -NVML_HOST_VGPU_MODE_NON_SRIOV = 0 -NVML_HOST_VGPU_MODE_SRIOV = 1 +NVML_HOST_VGPU_MODE_NON_SRIOV = 0 +NVML_HOST_VGPU_MODE_SRIOV = 1 _nvmlConfComputeGpusReadyState_t = c_uint NVML_CC_ACCEPTING_CLIENT_REQUESTS_FALSE = 0 @@ -945,11 +1185,11 @@ NVML_CC_SYSTEM_DEVTOOLS_MODE_ON = 1 NVML_CC_SYSTEM_MULTIGPU_NONE = 0 NVML_CC_SYSTEM_MULTIGPU_PROTECTED_PCIE = 1 - + NVML_CC_SYSTEM_ENVIRONMENT_UNAVAILABLE = 0 NVML_CC_SYSTEM_ENVIRONMENT_SIM = 1 NVML_CC_SYSTEM_ENVIRONMENT_PROD = 2 - + _nvmlConfComputeCcFeature_t = c_uint NVML_CC_SYSTEM_FEATURE_DISABLED = 0 NVML_CC_SYSTEM_FEATURE_ENABLED = 1 @@ -961,62 +1201,71 @@ NVML_CC_KEY_ROTATION_THRESH_ATTACKER_ADVANTAGE_MAX = 65 # GSP firmware NVML_GSP_FIRMWARE_VERSION_BUF_SIZE = 0x40 + class NVMLLibraryMismatchError(Exception): pass + ## Error Checking ## class NVMLError(Exception): _valClassMapping = dict() # List of currently known error codes _errcode_to_string = { - NVML_ERROR_UNINITIALIZED: "Uninitialized", - NVML_ERROR_INVALID_ARGUMENT: "Invalid Argument", - NVML_ERROR_NOT_SUPPORTED: "Not Supported", - NVML_ERROR_NO_PERMISSION: "Insufficient Permissions", + NVML_ERROR_UNINITIALIZED: "Uninitialized", + NVML_ERROR_INVALID_ARGUMENT: "Invalid Argument", + NVML_ERROR_NOT_SUPPORTED: "Not Supported", + NVML_ERROR_NO_PERMISSION: "Insufficient Permissions", NVML_ERROR_ALREADY_INITIALIZED: "Already Initialized", - NVML_ERROR_NOT_FOUND: "Not Found", - NVML_ERROR_INSUFFICIENT_SIZE: "Insufficient Size", - NVML_ERROR_INSUFFICIENT_POWER: "Insufficient External Power", - NVML_ERROR_DRIVER_NOT_LOADED: "Driver Not Loaded", - NVML_ERROR_TIMEOUT: "Timeout", - NVML_ERROR_IRQ_ISSUE: "Interrupt Request Issue", - NVML_ERROR_LIBRARY_NOT_FOUND: "NVML Shared Library Not Found", - NVML_ERROR_FUNCTION_NOT_FOUND: "Function Not Found", - NVML_ERROR_CORRUPTED_INFOROM: "Corrupted infoROM", - NVML_ERROR_GPU_IS_LOST: "GPU is lost", - NVML_ERROR_RESET_REQUIRED: "GPU requires restart", - NVML_ERROR_OPERATING_SYSTEM: "The operating system has blocked the request.", + NVML_ERROR_NOT_FOUND: "Not Found", + NVML_ERROR_INSUFFICIENT_SIZE: "Insufficient Size", + NVML_ERROR_INSUFFICIENT_POWER: "Insufficient External Power", + NVML_ERROR_DRIVER_NOT_LOADED: "Driver Not Loaded", + NVML_ERROR_TIMEOUT: "Timeout", + NVML_ERROR_IRQ_ISSUE: "Interrupt Request Issue", + NVML_ERROR_LIBRARY_NOT_FOUND: "NVML Shared Library Not Found", + NVML_ERROR_FUNCTION_NOT_FOUND: "Function Not Found", + NVML_ERROR_CORRUPTED_INFOROM: "Corrupted infoROM", + NVML_ERROR_GPU_IS_LOST: "GPU is lost", + NVML_ERROR_RESET_REQUIRED: "GPU requires restart", + NVML_ERROR_OPERATING_SYSTEM: "The operating system has blocked the request.", NVML_ERROR_LIB_RM_VERSION_MISMATCH: "RM has detected an NVML/RM version mismatch.", - NVML_ERROR_MEMORY: "Insufficient Memory", - NVML_ERROR_UNKNOWN: "Unknown Error", - } + NVML_ERROR_MEMORY: "Insufficient Memory", + NVML_ERROR_UNKNOWN: "Unknown Error", + } + def __new__(typ, value): - ''' + """ Maps value to a proper subclass of NVMLError. See _extractNVMLErrorsAsClasses function for more details - ''' + """ if typ == NVMLError: typ = NVMLError._valClassMapping.get(value, typ) obj = Exception.__new__(typ) obj.value = value return obj + def __str__(self): try: if self.value not in NVMLError._errcode_to_string: - NVMLError._errcode_to_string[self.value] = str(nvmlErrorString(self.value)) + NVMLError._errcode_to_string[self.value] = str( + nvmlErrorString(self.value) + ) return NVMLError._errcode_to_string[self.value] except NVMLError: return "NVML Error with code %d" % self.value + def __eq__(self, other): return self.value == other.value + def nvmlExceptionClass(nvmlErrorCode): if nvmlErrorCode not in NVMLError._valClassMapping: - raise ValueError('nvmlErrorCode %s is not valid' % nvmlErrorCode) + raise ValueError("nvmlErrorCode %s is not valid" % nvmlErrorCode) return NVMLError._valClassMapping[nvmlErrorCode] + def _extractNVMLErrorsAsClasses(): - ''' + """ Generates a hierarchy of classes on top of NVMLError class. Each NVML Error gets a new NVMLError subclass. This way try,except blocks can filter appropriate @@ -1024,31 +1273,44 @@ def _extractNVMLErrorsAsClasses(): NVMLError is a parent class. Each NVML_ERROR_* gets its own subclass. e.g. NVML_ERROR_ALREADY_INITIALIZED will be turned into NVMLError_AlreadyInitialized - ''' + """ this_module = sys.modules[__name__] nvmlErrorsNames = [x for x in dir(this_module) if x.startswith("NVML_ERROR_")] for err_name in nvmlErrorsNames: # e.g. Turn NVML_ERROR_ALREADY_INITIALIZED into NVMLError_AlreadyInitialized - class_name = "NVMLError_" + string.capwords(err_name.replace("NVML_ERROR_", ""), "_").replace("_", "") + class_name = "NVMLError_" + string.capwords( + err_name.replace("NVML_ERROR_", ""), "_" + ).replace("_", "") err_val = getattr(this_module, err_name) + def gen_new(val): def new(typ): obj = NVMLError.__new__(typ, val) return obj + return new - new_error_class = type(class_name, (NVMLError,), {'__new__': gen_new(err_val)}) + + new_error_class = type(class_name, (NVMLError,), {"__new__": gen_new(err_val)}) new_error_class.__module__ = __name__ setattr(this_module, class_name, new_error_class) NVMLError._valClassMapping[err_val] = new_error_class + + _extractNVMLErrorsAsClasses() + def _nvmlCheckReturn(ret): - if (ret != NVML_SUCCESS): + if ret != NVML_SUCCESS: raise NVMLError(ret) return ret + ## Function access ## -_nvmlGetFunctionPointer_cache = dict() # function pointers are cached to prevent unnecessary libLoadLock locking +_nvmlGetFunctionPointer_cache = ( + dict() +) # function pointers are cached to prevent unnecessary libLoadLock locking + + def _nvmlGetFunctionPointer(name): global nvmlLib @@ -1058,7 +1320,7 @@ def _nvmlGetFunctionPointer(name): libLoadLock.acquire() try: # ensure library was loaded - if (nvmlLib == None): + if nvmlLib == None: raise NVMLError(NVML_ERROR_UNINITIALIZED) try: _nvmlGetFunctionPointer_cache[name] = getattr(nvmlLib, name) @@ -1069,6 +1331,7 @@ def _nvmlGetFunctionPointer(name): # lock is always freed libLoadLock.release() + ## Alternative object # Allows the object to be printed # Allows mismatched types to be assigned @@ -1077,9 +1340,11 @@ class nvmlFriendlyObject(object): def __init__(self, dictionary): for x in dictionary: setattr(self, x, dictionary[x]) + def __str__(self): return self.__dict__.__str__() + def nvmlStructToFriendlyObject(struct): d = {} for x in struct._fields_: @@ -1090,6 +1355,7 @@ def nvmlStructToFriendlyObject(struct): obj = nvmlFriendlyObject(d) return obj + # pack the object so it can be passed to the NVML library def nvmlFriendlyObjectToStruct(obj, model): for x in model._fields_: @@ -1102,11 +1368,15 @@ def nvmlFriendlyObjectToStruct(obj, model): setattr(model, key, value) return model + ## Unit structures class struct_c_nvmlUnit_t(Structure): - pass # opaque handle + pass # opaque handle + + c_nvmlUnit_t = POINTER(struct_c_nvmlUnit_t) + class _PrintableStructure(Structure): """ Abstract class that produces nicer __str__ output than ctypes.Structure. @@ -1126,7 +1396,9 @@ class _PrintableStructure(Structure): Exact format of returned str from this class is subject to change in the future. """ + _fmt_ = {} + def __str__(self): result = [] for x in self._fields_: @@ -1138,7 +1410,7 @@ class _PrintableStructure(Structure): elif "" in self._fmt_: fmt = self._fmt_[""] result.append(("%s: " + fmt) % (key, value)) - return self.__class__.__name__ + "(" + ", ".join(result) + ")" + return self.__class__.__name__ + "(" + ", ".join(result) + ")" def __getattribute__(self, name): res = super(_PrintableStructure, self).__getattribute__(name) @@ -1159,185 +1431,200 @@ class _PrintableStructure(Structure): value = value.encode() super(_PrintableStructure, self).__setattr__(name, value) + class c_nvmlUnitInfo_t(_PrintableStructure): _fields_ = [ - ('name', c_char * 96), - ('id', c_char * 96), - ('serial', c_char * 96), - ('firmwareVersion', c_char * 96), + ("name", c_char * 96), + ("id", c_char * 96), + ("serial", c_char * 96), + ("firmwareVersion", c_char * 96), ] + class c_nvmlC2cModeInfo_v1_t(_PrintableStructure): - _fields_ = [ - ('isC2cEnabled', c_uint) - ] + _fields_ = [("isC2cEnabled", c_uint)] + + +nvmlC2cModeInfo_v1 = 0x1000008 -nvmlC2cModeInfo_v1 = 0x1000008; class c_nvmlLedState_t(_PrintableStructure): _fields_ = [ - ('cause', c_char * 256), - ('color', _nvmlLedColor_t), + ("cause", c_char * 256), + ("color", _nvmlLedColor_t), ] + class c_nvmlPSUInfo_t(_PrintableStructure): _fields_ = [ - ('state', c_char * 256), - ('current', c_uint), - ('voltage', c_uint), - ('power', c_uint), + ("state", c_char * 256), + ("current", c_uint), + ("voltage", c_uint), + ("power", c_uint), ] + class c_nvmlUnitFanInfo_t(_PrintableStructure): _fields_ = [ - ('speed', c_uint), - ('state', _nvmlFanState_t), + ("speed", c_uint), + ("state", _nvmlFanState_t), ] + class c_nvmlUnitFanSpeeds_t(_PrintableStructure): - _fields_ = [ - ('fans', c_nvmlUnitFanInfo_t * 24), - ('count', c_uint) - ] + _fields_ = [("fans", c_nvmlUnitFanInfo_t * 24), ("count", c_uint)] + ## Device structures class struct_c_nvmlDevice_t(Structure): - pass # opaque handle + pass # opaque handle + + c_nvmlDevice_t = POINTER(struct_c_nvmlDevice_t) + class nvmlPciInfoExt_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('domain', c_uint), - ('bus', c_uint), - ('device', c_uint), - ('pciDeviceId', c_uint), - ('pciSubSystemId', c_uint), - ('baseClass', c_uint), - ('subClass', c_uint), - ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE), + ("version", c_uint), + ("domain", c_uint), + ("bus", c_uint), + ("device", c_uint), + ("pciDeviceId", c_uint), + ("pciSubSystemId", c_uint), + ("baseClass", c_uint), + ("subClass", c_uint), + ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE), ] _fmt_ = { - 'version' : "0x%04X", - 'domain' : "0x%04X", - 'bus' : "0x%02X", - 'device' : "0x%02X", - 'pciDeviceId' : "0x%08X", - 'pciSubSystemId' : "0x%08X", - 'baseClass' : "0x%01X", - 'subClass' : "0x%01X", - } + "version": "0x%04X", + "domain": "0x%04X", + "bus": "0x%02X", + "device": "0x%02X", + "pciDeviceId": "0x%08X", + "pciSubSystemId": "0x%08X", + "baseClass": "0x%01X", + "subClass": "0x%01X", + } + nvmlPciInfoExt_v1 = 0x1000040 + # Legacy pciInfo used for _v1 and _v2 class nvmlPciInfo_v2_t(_PrintableStructure): _fields_ = [ - ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE), - ('domain', c_uint), - ('bus', c_uint), - ('device', c_uint), - ('pciDeviceId', c_uint), - + ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE), + ("domain", c_uint), + ("bus", c_uint), + ("device", c_uint), + ("pciDeviceId", c_uint), # Added in 2.285 - ('pciSubSystemId', c_uint), - ('reserved0', c_uint), - ('reserved1', c_uint), - ('reserved2', c_uint), - ('reserved3', c_uint), + ("pciSubSystemId", c_uint), + ("reserved0", c_uint), + ("reserved1", c_uint), + ("reserved2", c_uint), + ("reserved3", c_uint), ] _fmt_ = { - 'domain' : "0x%04X", - 'bus' : "0x%02X", - 'device' : "0x%02X", - 'pciDeviceId' : "0x%08X", - 'pciSubSystemId' : "0x%08X", - } + "domain": "0x%04X", + "bus": "0x%02X", + "device": "0x%02X", + "pciDeviceId": "0x%08X", + "pciSubSystemId": "0x%08X", + } + class nvmlPciInfo_t(_PrintableStructure): _fields_ = [ # Moved to the new busId location below - ('busIdLegacy', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE), - ('domain', c_uint), - ('bus', c_uint), - ('device', c_uint), - ('pciDeviceId', c_uint), - + ("busIdLegacy", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_V2_SIZE), + ("domain", c_uint), + ("bus", c_uint), + ("device", c_uint), + ("pciDeviceId", c_uint), # Added in 2.285 - ('pciSubSystemId', c_uint), + ("pciSubSystemId", c_uint), # New busId replaced the long deprecated and reserved fields with a # field of the same size in 9.0 - ('busId', c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE), + ("busId", c_char * NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE), ] _fmt_ = { - 'domain' : "0x%08X", - 'bus' : "0x%02X", - 'device' : "0x%02X", - 'pciDeviceId' : "0x%08X", - 'pciSubSystemId' : "0x%08X", - } + "domain": "0x%08X", + "bus": "0x%02X", + "device": "0x%02X", + "pciDeviceId": "0x%08X", + "pciSubSystemId": "0x%08X", + } + class c_nvmlSystemDriverBranchInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), + ("version", c_uint), ("branch", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), ] + SystemDriverBranchInfo_v1 = 0x1000054 + class c_nvmlExcludedDeviceInfo_t(_PrintableStructure): - _fields_ = [ - ('pci', nvmlPciInfo_t), - ('uuid', c_char * NVML_DEVICE_UUID_BUFFER_SIZE) - ] + _fields_ = [("pci", nvmlPciInfo_t), ("uuid", c_char * NVML_DEVICE_UUID_BUFFER_SIZE)] + class nvmlNvLinkUtilizationControl_t(_PrintableStructure): _fields_ = [ - ('units', _nvmlNvLinkUtilizationCountUnits_t), - ('pktfilter', _nvmlNvLinkUtilizationCountPktTypes_t), + ("units", _nvmlNvLinkUtilizationCountUnits_t), + ("pktfilter", _nvmlNvLinkUtilizationCountPktTypes_t), ] + class c_nvmlMemory_t(_PrintableStructure): _fields_ = [ - ('total', c_ulonglong), - ('free', c_ulonglong), - ('used', c_ulonglong), + ("total", c_ulonglong), + ("free", c_ulonglong), + ("used", c_ulonglong), ] - _fmt_ = {'': "%d B"} + _fmt_ = {"": "%d B"} + class c_nvmlMemory_v2_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('total', c_ulonglong), - ('reserved', c_ulonglong), - ('free', c_ulonglong), - ('used', c_ulonglong), + ("version", c_uint), + ("total", c_ulonglong), + ("reserved", c_ulonglong), + ("free", c_ulonglong), + ("used", c_ulonglong), ] - _fmt_ = {'': "%d B"} + _fmt_ = {"": "%d B"} + nvmlMemory_v2 = 0x02000028 + class c_nvmlBAR1Memory_t(_PrintableStructure): _fields_ = [ - ('bar1Total', c_ulonglong), - ('bar1Free', c_ulonglong), - ('bar1Used', c_ulonglong), + ("bar1Total", c_ulonglong), + ("bar1Free", c_ulonglong), + ("bar1Used", c_ulonglong), ] - _fmt_ = {'': "%d B"} + _fmt_ = {"": "%d B"} + class nvmlClkMonFaultInfo_t(Structure): - _fields_ = [("clkApiDomain", c_uint), - ("clkDomainFaultMask", c_uint) - ] + _fields_ = [("clkApiDomain", c_uint), ("clkDomainFaultMask", c_uint)] + MAX_CLK_DOMAINS = 32 + class nvmlClkMonStatus_t(Structure): - _fields_ = [("bGlobalStatus", c_uint), - ("clkMonListSize", c_uint), - ("clkMonList", nvmlClkMonFaultInfo_t * MAX_CLK_DOMAINS) + _fields_ = [ + ("bGlobalStatus", c_uint), + ("clkMonListSize", c_uint), + ("clkMonList", nvmlClkMonFaultInfo_t * MAX_CLK_DOMAINS), ] + # On Windows with the WDDM driver, usedGpuMemory is reported as None # Code that processes this structure should check for None, I.E. # @@ -1351,887 +1638,1031 @@ class nvmlClkMonStatus_t(Structure): # See NVML documentation for more information class c_nvmlProcessInfo_v2_t(_PrintableStructure): _fields_ = [ - ('pid', c_uint), - ('usedGpuMemory', c_ulonglong), - ('gpuInstanceId', c_uint), - ('computeInstanceId', c_uint), + ("pid", c_uint), + ("usedGpuMemory", c_ulonglong), + ("gpuInstanceId", c_uint), + ("computeInstanceId", c_uint), ] - _fmt_ = {'usedGpuMemory': "%d B"} + _fmt_ = {"usedGpuMemory": "%d B"} + c_nvmlProcessInfo_v3_t = c_nvmlProcessInfo_v2_t c_nvmlProcessInfo_t = c_nvmlProcessInfo_v3_t _nvmlProcessMode_t = c_uint -NVML_PROCESS_MODE_COMPUTE = 0 +NVML_PROCESS_MODE_COMPUTE = 0 NVML_PROCESS_MODE_GRAPHICS = 1 -NVML_PROCESS_MODE_MPS = 2 +NVML_PROCESS_MODE_MPS = 2 + class c_nvmlProcessDetail_v1_t(Structure): _fields_ = [ - ('pid', c_uint), - ('usedGpuMemory', c_ulonglong), - ('gpuInstanceId', c_uint), - ('computeInstanceId', c_uint), - ('usedGpuCcProtectedMemory', c_ulonglong), + ("pid", c_uint), + ("usedGpuMemory", c_ulonglong), + ("gpuInstanceId", c_uint), + ("computeInstanceId", c_uint), + ("usedGpuCcProtectedMemory", c_ulonglong), ] + class c_nvmlProcessDetailList_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('mode', _nvmlProcessMode_t), - ('numProcArrayEntries', c_uint), - ('procArray', POINTER(c_nvmlProcessDetail_v1_t)), + ("version", c_uint), + ("mode", _nvmlProcessMode_t), + ("numProcArrayEntries", c_uint), + ("procArray", POINTER(c_nvmlProcessDetail_v1_t)), ] - _fmt_ = {'numProcArrayEntries': "%d B"} + _fmt_ = {"numProcArrayEntries": "%d B"} + c_nvmlProcessDetailList_t = c_nvmlProcessDetailList_v1_t nvmlProcessDetailList_v1 = 0x1000018 + class c_nvmlBridgeChipInfo_t(_PrintableStructure): _fields_ = [ - ('type', _nvmlBridgeChipType_t), - ('fwVersion', c_uint), + ("type", _nvmlBridgeChipType_t), + ("fwVersion", c_uint), ] + class c_nvmlBridgeChipHierarchy_t(_PrintableStructure): _fields_ = [ - ('bridgeCount', c_uint), - ('bridgeChipInfo', c_nvmlBridgeChipInfo_t * 128), + ("bridgeCount", c_uint), + ("bridgeChipInfo", c_nvmlBridgeChipInfo_t * 128), ] + class c_nvmlEccErrorCounts_t(_PrintableStructure): _fields_ = [ - ('l1Cache', c_ulonglong), - ('l2Cache', c_ulonglong), - ('deviceMemory', c_ulonglong), - ('registerFile', c_ulonglong), + ("l1Cache", c_ulonglong), + ("l2Cache", c_ulonglong), + ("deviceMemory", c_ulonglong), + ("registerFile", c_ulonglong), ] + class c_nvmlUtilization_t(_PrintableStructure): _fields_ = [ - ('gpu', c_uint), - ('memory', c_uint), + ("gpu", c_uint), + ("memory", c_uint), ] - _fmt_ = {'': "%d %%"} + _fmt_ = {"": "%d %%"} + # Added in 2.285 class c_nvmlHwbcEntry_t(_PrintableStructure): _fields_ = [ - ('hwbcId', c_uint), - ('firmwareVersion', c_char * 32), + ("hwbcId", c_uint), + ("firmwareVersion", c_char * 32), ] + class c_nvmlValue_t(Union): _fields_ = [ - ('dVal', c_double), - ('uiVal', c_uint), - ('ulVal', c_ulong), - ('ullVal', c_ulonglong), - ('sllVal', c_longlong), - ('siVal', c_int), - ('usVal', c_ushort), + ("dVal", c_double), + ("uiVal", c_uint), + ("ulVal", c_ulong), + ("ullVal", c_ulonglong), + ("sllVal", c_longlong), + ("siVal", c_int), + ("usVal", c_ushort), ] + class c_nvmlSample_t(_PrintableStructure): _fields_ = [ - ('timeStamp', c_ulonglong), - ('sampleValue', c_nvmlValue_t), + ("timeStamp", c_ulonglong), + ("sampleValue", c_nvmlValue_t), ] + class c_nvmlViolationTime_t(_PrintableStructure): _fields_ = [ - ('referenceTime', c_ulonglong), - ('violationTime', c_ulonglong), + ("referenceTime", c_ulonglong), + ("violationTime", c_ulonglong), ] + class c_nvmlFieldValue_t(_PrintableStructure): _fields_ = [ - ('fieldId', c_uint32), - ('scopeId', c_uint32), - ('timestamp', c_int64), - ('latencyUsec', c_int64), - ('valueType', _nvmlValueType_t), - ('nvmlReturn', _nvmlReturn_t), - ('value', c_nvmlValue_t) + ("fieldId", c_uint32), + ("scopeId", c_uint32), + ("timestamp", c_int64), + ("latencyUsec", c_int64), + ("valueType", _nvmlValueType_t), + ("nvmlReturn", _nvmlReturn_t), + ("value", c_nvmlValue_t), ] + NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES = 23 -nvmlNvlinkSupportedBwModes_v1 = 0x100001c +nvmlNvlinkSupportedBwModes_v1 = 0x100001C + + class c_nvmlNvlinkSupportedBwModes_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('bwModes', c_uint8 * NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES), - ('totalBwModes', c_uint8) + ("version", c_uint), + ("bwModes", c_uint8 * NVML_NVLINK_TOTAL_SUPPORTED_BW_MODES), + ("totalBwModes", c_uint8), ] def __init__(self): - super(c_nvmlNvlinkSupportedBwModes_v1_t, self).__init__(version=nvmlNvlinkSupportedBwModes_v1) + super(c_nvmlNvlinkSupportedBwModes_v1_t, self).__init__( + version=nvmlNvlinkSupportedBwModes_v1 + ) + + +nvmlNvlinkGetBwMode_v1 = 0x100000C + -nvmlNvlinkGetBwMode_v1 = 0x100000c class c_nvmlNvlinkGetBwMode_v1_t(_PrintableStructure): - _fields_ = [ - ('version', c_uint), - ('bIsBest', c_uint), - ('bwMode', c_uint8) - ] + _fields_ = [("version", c_uint), ("bIsBest", c_uint), ("bwMode", c_uint8)] def __init__(self): super(c_nvmlNvlinkGetBwMode_v1_t, self).__init__(version=nvmlNvlinkGetBwMode_v1) -nvmlNvlinkSetBwMode_v1 = 0x100000c + +nvmlNvlinkSetBwMode_v1 = 0x100000C + + class c_nvmlNvlinkSetBwMode_v1_t(_PrintableStructure): - _fields_ = [ - ('version', c_uint), - ('bSetBest', c_uint), - ('bwMode', c_uint8) - ] + _fields_ = [("version", c_uint), ("bSetBest", c_uint), ("bwMode", c_uint8)] def __init__(self): super(c_nvmlNvlinkSetBwMode_v1_t, self).__init__(version=nvmlNvlinkSetBwMode_v1) + class c_nvmlVgpuHeterogeneousMode_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('mode', c_uint), + ("version", c_uint), + ("mode", c_uint), ] + VgpuHeterogeneousMode_v1 = 0x1000008 + class c_nvmlVgpuPlacementId_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('placementId', c_uint), + ("version", c_uint), + ("placementId", c_uint), ] + VgpuPlacementId_v1 = 0x1000008 + class c_nvmlVgpuPlacementList_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('count', c_uint), - ('placementSize', c_uint), - ('placementIds', POINTER(c_uint)), + ("version", c_uint), + ("count", c_uint), + ("placementSize", c_uint), + ("placementIds", POINTER(c_uint)), ] + VgpuPlacementList_v1 = 0x1000018 -NVML_VGPU_PGPU_HETEROGENEOUS_MODE = 0 -NVML_VGPU_PGPU_HOMOGENEOUS_MODE = 1 +NVML_VGPU_PGPU_HETEROGENEOUS_MODE = 0 +NVML_VGPU_PGPU_HOMOGENEOUS_MODE = 1 + class c_nvmlVgpuPlacementList_v2_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('placementSize', c_uint), - ('count', c_uint), - ('placementIds', POINTER(c_uint)), - ('mode', c_uint), + ("version", c_uint), + ("placementSize", c_uint), + ("count", c_uint), + ("placementIds", POINTER(c_uint)), + ("mode", c_uint), ] + VgpuPlacementList_v2 = 0x2000020 + class c_nvmlVgpuTypeBar1Info_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('bar1Size', c_ulonglong), + ("version", c_uint), + ("bar1Size", c_ulonglong), ] + VgpuTypeBar1Info_v1 = 0x1000010 + class c_nvmlVgpuInstanceUtilizationSample_t(_PrintableStructure): _fields_ = [ - ('vgpuInstance', _nvmlVgpuInstance_t), - ('timeStamp', c_ulonglong), - ('smUtil', c_nvmlValue_t), - ('memUtil', c_nvmlValue_t), - ('encUtil', c_nvmlValue_t), - ('decUtil', c_nvmlValue_t), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("timeStamp", c_ulonglong), + ("smUtil", c_nvmlValue_t), + ("memUtil", c_nvmlValue_t), + ("encUtil", c_nvmlValue_t), + ("decUtil", c_nvmlValue_t), ] + class c_nvmlVgpuInstanceUtilizationInfo_v1_t(_PrintableStructure): _fields_ = [ - ('timeStamp', c_ulonglong), - ('vgpuInstance', _nvmlVgpuInstance_t), - ('smUtil', c_nvmlValue_t), - ('memUtil', c_nvmlValue_t), - ('encUtil', c_nvmlValue_t), - ('decUtil', c_nvmlValue_t), - ('jpgUtil', c_nvmlValue_t), - ('ofaUtil', c_nvmlValue_t), + ("timeStamp", c_ulonglong), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("smUtil", c_nvmlValue_t), + ("memUtil", c_nvmlValue_t), + ("encUtil", c_nvmlValue_t), + ("decUtil", c_nvmlValue_t), + ("jpgUtil", c_nvmlValue_t), + ("ofaUtil", c_nvmlValue_t), ] + class c_nvmlVgpuInstancesUtilizationInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('sampleValType', _nvmlValueType_t), - ('vgpuInstanceCount', c_uint), - ('lastSeenTimeStamp', c_ulonglong), - ('vgpuUtilArray', POINTER(c_nvmlVgpuInstanceUtilizationInfo_v1_t)), + ("version", c_uint), + ("sampleValType", _nvmlValueType_t), + ("vgpuInstanceCount", c_uint), + ("lastSeenTimeStamp", c_ulonglong), + ("vgpuUtilArray", POINTER(c_nvmlVgpuInstanceUtilizationInfo_v1_t)), ] + VgpuInstancesUtilizationInfo_v1 = 0x01000020 + class c_nvmlVgpuProcessUtilizationSample_t(_PrintableStructure): _fields_ = [ - ('vgpuInstance', _nvmlVgpuInstance_t), - ('pid', c_uint), - ('processName', c_char * NVML_VGPU_NAME_BUFFER_SIZE), - ('timeStamp', c_ulonglong), - ('smUtil', c_uint), - ('memUtil', c_uint), - ('encUtil', c_uint), - ('decUtil', c_uint), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("pid", c_uint), + ("processName", c_char * NVML_VGPU_NAME_BUFFER_SIZE), + ("timeStamp", c_ulonglong), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), ] + class c_nvmlVgpuProcessUtilizationInfo_v1_t(_PrintableStructure): _fields_ = [ - ('processName', c_char * NVML_VGPU_NAME_BUFFER_SIZE), - ('timeStamp', c_ulonglong), - ('vgpuInstance', _nvmlVgpuInstance_t), - ('pid', c_uint), - ('smUtil', c_uint), - ('memUtil', c_uint), - ('encUtil', c_uint), - ('decUtil', c_uint), - ('jpgUtil', c_uint), - ('ofaUtil', c_uint), + ("processName", c_char * NVML_VGPU_NAME_BUFFER_SIZE), + ("timeStamp", c_ulonglong), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("pid", c_uint), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), + ("jpgUtil", c_uint), + ("ofaUtil", c_uint), ] + class c_nvmlVgpuProcessesUtilizationInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('vgpuProcessCount', c_uint), - ('lastSeenTimeStamp', c_ulonglong), - ('vgpuProcUtilArray', POINTER(c_nvmlVgpuProcessUtilizationInfo_v1_t)), + ("version", c_uint), + ("vgpuProcessCount", c_uint), + ("lastSeenTimeStamp", c_ulonglong), + ("vgpuProcUtilArray", POINTER(c_nvmlVgpuProcessUtilizationInfo_v1_t)), ] + VgpuProcessesUtilizationInfo_v1 = 0x01000018 + class nvmlVgpuRuntimeState_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('size', c_ulonglong), + ("version", c_uint), + ("size", c_ulonglong), ] + VgpuRuntimeState_v1 = 0x1000010 + class c_nvmlVgpuLicenseExpiry_t(_PrintableStructure): _fields_ = [ - ('year', c_uint32), - ('month', c_uint16), - ('day', c_uint16), - ('hour', c_uint16), - ('min', c_uint16), - ('sec', c_uint16), - ('status', c_uint8), + ("year", c_uint32), + ("month", c_uint16), + ("day", c_uint16), + ("hour", c_uint16), + ("min", c_uint16), + ("sec", c_uint16), + ("status", c_uint8), ] -NVML_GRID_LICENSE_STATE_UNKNOWN = 0 -NVML_GRID_LICENSE_STATE_UNINITIALIZED = 1 + +NVML_GRID_LICENSE_STATE_UNKNOWN = 0 +NVML_GRID_LICENSE_STATE_UNINITIALIZED = 1 NVML_GRID_LICENSE_STATE_UNLICENSED_UNRESTRICTED = 2 -NVML_GRID_LICENSE_STATE_UNLICENSED_RESTRICTED = 3 -NVML_GRID_LICENSE_STATE_UNLICENSED = 4 -NVML_GRID_LICENSE_STATE_LICENSED = 5 +NVML_GRID_LICENSE_STATE_UNLICENSED_RESTRICTED = 3 +NVML_GRID_LICENSE_STATE_UNLICENSED = 4 +NVML_GRID_LICENSE_STATE_LICENSED = 5 + class c_nvmlVgpuLicenseInfo_t(_PrintableStructure): _fields_ = [ - ('isLicensed', c_uint8), - ('licenseExpiry', c_nvmlVgpuLicenseExpiry_t), - ('currentState', c_uint), + ("isLicensed", c_uint8), + ("licenseExpiry", c_nvmlVgpuLicenseExpiry_t), + ("currentState", c_uint), ] + class c_nvmlEncoderSession_t(_PrintableStructure): _fields_ = [ - ('sessionId', c_uint), - ('pid', c_uint), - ('vgpuInstance', _nvmlVgpuInstance_t), - ('codecType', c_uint), - ('hResolution', c_uint), - ('vResolution', c_uint), - ('averageFps', c_uint), - ('encodeLatency', c_uint), + ("sessionId", c_uint), + ("pid", c_uint), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("codecType", c_uint), + ("hResolution", c_uint), + ("vResolution", c_uint), + ("averageFps", c_uint), + ("encodeLatency", c_uint), ] + class c_nvmlProcessUtilizationSample_t(_PrintableStructure): _fields_ = [ - ('pid', c_uint), - ('timeStamp', c_ulonglong), - ('smUtil', c_uint), - ('memUtil', c_uint), - ('encUtil', c_uint), - ('decUtil', c_uint), + ("pid", c_uint), + ("timeStamp", c_ulonglong), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), ] + class c_nvmlProcessUtilizationInfo_v1_t(_PrintableStructure): _fields_ = [ - ('timeStamp', c_ulonglong), - ('pid', c_uint), - ('smUtil', c_uint), - ('memUtil', c_uint), - ('encUtil', c_uint), - ('decUtil', c_uint), - ('jpgUtil', c_uint), - ('ofaUtil', c_uint), + ("timeStamp", c_ulonglong), + ("pid", c_uint), + ("smUtil", c_uint), + ("memUtil", c_uint), + ("encUtil", c_uint), + ("decUtil", c_uint), + ("jpgUtil", c_uint), + ("ofaUtil", c_uint), ] + class c_nvmlProcessesUtilizationInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('processSamplesCount', c_uint), - ('lastSeenTimeStamp', c_ulonglong), - ('procUtilArray', POINTER(c_nvmlProcessUtilizationInfo_v1_t)), + ("version", c_uint), + ("processSamplesCount", c_uint), + ("lastSeenTimeStamp", c_ulonglong), + ("procUtilArray", POINTER(c_nvmlProcessUtilizationInfo_v1_t)), ] + ProcessesUtilizationInfo_v1 = 0x01000018 + class c_nvmlGridLicenseExpiry_t(_PrintableStructure): _fields_ = [ - ('year', c_uint32), - ('month', c_uint16), - ('day', c_uint16), - ('hour', c_uint16), - ('min', c_uint16), - ('sec', c_uint16), - ('status', c_uint8), + ("year", c_uint32), + ("month", c_uint16), + ("day", c_uint16), + ("hour", c_uint16), + ("min", c_uint16), + ("sec", c_uint16), + ("status", c_uint8), ] + class c_nvmlGridLicensableFeature_v4_t(_PrintableStructure): _fields_ = [ - ('featureCode', _nvmlGridLicenseFeatureCode_t), - ('featureState', c_uint), - ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), - ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), - ('featureEnabled', c_uint), - ('licenseExpiry', c_nvmlGridLicenseExpiry_t), + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("featureEnabled", c_uint), + ("licenseExpiry", c_nvmlGridLicenseExpiry_t), ] + class c_nvmlGridLicensableFeatures_v4_t(_PrintableStructure): _fields_ = [ - ('isGridLicenseSupported', c_int), - ('licensableFeaturesCount', c_uint), - ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v4_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT), + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_v4_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), ] + class c_nvmlGridLicensableFeature_v3_t(_PrintableStructure): _fields_ = [ - ('featureCode', _nvmlGridLicenseFeatureCode_t), - ('featureState', c_uint), - ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), - ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), - ('featureEnabled', c_uint), + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("featureEnabled", c_uint), ] + class c_nvmlGridLicensableFeatures_v3_t(_PrintableStructure): _fields_ = [ - ('isGridLicenseSupported', c_int), - ('licensableFeaturesCount', c_uint), - ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v3_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT), + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_v3_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), ] + class c_nvmlGridLicensableFeature_v2_t(_PrintableStructure): _fields_ = [ - ('featureCode', _nvmlGridLicenseFeatureCode_t), - ('featureState', c_uint), - ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), - ('productName', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("productName", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), ] + class c_nvmlGridLicensableFeatures_v2_t(_PrintableStructure): _fields_ = [ - ('isGridLicenseSupported', c_int), - ('licensableFeaturesCount', c_uint), - ('gridLicensableFeatures', c_nvmlGridLicensableFeature_v2_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT), + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_v2_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), ] + class c_nvmlGridLicensableFeature_t(_PrintableStructure): _fields_ = [ - ('featureCode', _nvmlGridLicenseFeatureCode_t), - ('featureState', c_uint), - ('licenseInfo', c_char * NVML_GRID_LICENSE_BUFFER_SIZE), + ("featureCode", _nvmlGridLicenseFeatureCode_t), + ("featureState", c_uint), + ("licenseInfo", c_char * NVML_GRID_LICENSE_BUFFER_SIZE), ] + class c_nvmlGridLicensableFeatures_t(_PrintableStructure): _fields_ = [ - ('isGridLicenseSupported', c_int), - ('licensableFeaturesCount', c_uint), - ('gridLicensableFeatures', c_nvmlGridLicensableFeature_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT), + ("isGridLicenseSupported", c_int), + ("licensableFeaturesCount", c_uint), + ( + "gridLicensableFeatures", + c_nvmlGridLicensableFeature_t * NVML_GRID_LICENSE_FEATURE_MAX_COUNT, + ), ] + class c_nvmlMarginTemperature_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('marginTemperature', c_int), + ("version", c_uint), + ("marginTemperature", c_int), ] + nvmlMarginTemperature_v1 = 0x1000008 + ## Event structures class struct_c_nvmlEventSet_t(Structure): - pass # opaque handle + pass # opaque handle + + c_nvmlEventSet_t = POINTER(struct_c_nvmlEventSet_t) -nvmlEventTypeSingleBitEccError = 0x0000000000000001 -nvmlEventTypeDoubleBitEccError = 0x0000000000000002 -nvmlEventTypePState = 0x0000000000000004 -nvmlEventTypeXidCriticalError = 0x0000000000000008 -nvmlEventTypeClock = 0x0000000000000010 -nvmlEventTypePowerSourceChange = 0x0000000000000080 -nvmlEventMigConfigChange = 0x0000000000000100 +nvmlEventTypeSingleBitEccError = 0x0000000000000001 +nvmlEventTypeDoubleBitEccError = 0x0000000000000002 +nvmlEventTypePState = 0x0000000000000004 +nvmlEventTypeXidCriticalError = 0x0000000000000008 +nvmlEventTypeClock = 0x0000000000000010 +nvmlEventTypePowerSourceChange = 0x0000000000000080 +nvmlEventMigConfigChange = 0x0000000000000100 nvmlEventTypeSingleBitEccErrorStorm = 0x0000000000000200 -nvmlEventTypeDramRetirementEvent = 0x0000000000000400 -nvmlEventTypeDramRetirementFailure = 0x0000000000000800 -nvmlEventTypeNonFatalPoisonError = 0x0000000000001000 -nvmlEventTypeFatalPoisonError = 0x0000000000002000 -nvmlEventTypeGpuUnavailableError = 0x0000000000004000 -nvmlEventTypeGpuRecoveryAction = 0x0000000000008000 -nvmlEventTypeNone = 0x0000000000000000 -nvmlEventTypeAll = ( - nvmlEventTypeNone - | nvmlEventTypeSingleBitEccError - | nvmlEventTypeDoubleBitEccError - | nvmlEventTypePState - | nvmlEventTypeClock - | nvmlEventTypePowerSourceChange - | nvmlEventTypeXidCriticalError - | nvmlEventMigConfigChange - | nvmlEventTypeSingleBitEccErrorStorm - | nvmlEventTypeDramRetirementEvent - | nvmlEventTypeDramRetirementFailure - | nvmlEventTypeNonFatalPoisonError - | nvmlEventTypeFatalPoisonError - | nvmlEventTypeGpuUnavailableError - | nvmlEventTypeGpuRecoveryAction - ) +nvmlEventTypeDramRetirementEvent = 0x0000000000000400 +nvmlEventTypeDramRetirementFailure = 0x0000000000000800 +nvmlEventTypeNonFatalPoisonError = 0x0000000000001000 +nvmlEventTypeFatalPoisonError = 0x0000000000002000 +nvmlEventTypeGpuUnavailableError = 0x0000000000004000 +nvmlEventTypeGpuRecoveryAction = 0x0000000000008000 +nvmlEventTypeNone = 0x0000000000000000 +nvmlEventTypeAll = ( + nvmlEventTypeNone + | nvmlEventTypeSingleBitEccError + | nvmlEventTypeDoubleBitEccError + | nvmlEventTypePState + | nvmlEventTypeClock + | nvmlEventTypePowerSourceChange + | nvmlEventTypeXidCriticalError + | nvmlEventMigConfigChange + | nvmlEventTypeSingleBitEccErrorStorm + | nvmlEventTypeDramRetirementEvent + | nvmlEventTypeDramRetirementFailure + | nvmlEventTypeNonFatalPoisonError + | nvmlEventTypeFatalPoisonError + | nvmlEventTypeGpuUnavailableError + | nvmlEventTypeGpuRecoveryAction +) ## Clock Event Reasons defines -nvmlClocksEventReasonGpuIdle = 0x0000000000000001 +nvmlClocksEventReasonGpuIdle = 0x0000000000000001 nvmlClocksEventReasonApplicationsClocksSetting = 0x0000000000000002 -nvmlClocksEventReasonUserDefinedClocks = nvmlClocksEventReasonApplicationsClocksSetting # deprecated, use nvmlClocksEventReasonApplicationsClocksSetting -nvmlClocksEventReasonSwPowerCap = 0x0000000000000004 -nvmlClocksEventReasonHwSlowdown = 0x0000000000000008 -nvmlClocksEventReasonSyncBoost = 0x0000000000000010 -nvmlClocksEventReasonSwThermalSlowdown = 0x0000000000000020 -nvmlClocksEventReasonHwThermalSlowdown = 0x0000000000000040 +nvmlClocksEventReasonUserDefinedClocks = nvmlClocksEventReasonApplicationsClocksSetting # deprecated, use nvmlClocksEventReasonApplicationsClocksSetting +nvmlClocksEventReasonSwPowerCap = 0x0000000000000004 +nvmlClocksEventReasonHwSlowdown = 0x0000000000000008 +nvmlClocksEventReasonSyncBoost = 0x0000000000000010 +nvmlClocksEventReasonSwThermalSlowdown = 0x0000000000000020 +nvmlClocksEventReasonHwThermalSlowdown = 0x0000000000000040 nvmlClocksEventReasonHwPowerBrakeSlowdown = 0x0000000000000080 -nvmlClocksEventReasonDisplayClockSetting = 0x0000000000000100 -nvmlClocksEventReasonNone = 0x0000000000000000 -nvmlClocksEventReasonAll = ( - nvmlClocksEventReasonNone | - nvmlClocksEventReasonGpuIdle | - nvmlClocksEventReasonApplicationsClocksSetting | - nvmlClocksEventReasonSwPowerCap | - nvmlClocksEventReasonHwSlowdown | - nvmlClocksEventReasonSyncBoost | - nvmlClocksEventReasonSwThermalSlowdown | - nvmlClocksEventReasonHwThermalSlowdown | - nvmlClocksEventReasonHwPowerBrakeSlowdown | - nvmlClocksEventReasonDisplayClockSetting - ) +nvmlClocksEventReasonDisplayClockSetting = 0x0000000000000100 +nvmlClocksEventReasonNone = 0x0000000000000000 +nvmlClocksEventReasonAll = ( + nvmlClocksEventReasonNone + | nvmlClocksEventReasonGpuIdle + | nvmlClocksEventReasonApplicationsClocksSetting + | nvmlClocksEventReasonSwPowerCap + | nvmlClocksEventReasonHwSlowdown + | nvmlClocksEventReasonSyncBoost + | nvmlClocksEventReasonSwThermalSlowdown + | nvmlClocksEventReasonHwThermalSlowdown + | nvmlClocksEventReasonHwPowerBrakeSlowdown + | nvmlClocksEventReasonDisplayClockSetting +) ## Following have been deprecated -nvmlClocksThrottleReasonGpuIdle = 0x0000000000000001 +nvmlClocksThrottleReasonGpuIdle = 0x0000000000000001 nvmlClocksThrottleReasonApplicationsClocksSetting = 0x0000000000000002 -nvmlClocksThrottleReasonUserDefinedClocks = nvmlClocksThrottleReasonApplicationsClocksSetting # deprecated, use nvmlClocksThrottleReasonApplicationsClocksSetting -nvmlClocksThrottleReasonSwPowerCap = 0x0000000000000004 -nvmlClocksThrottleReasonHwSlowdown = 0x0000000000000008 -nvmlClocksThrottleReasonSyncBoost = 0x0000000000000010 -nvmlClocksThrottleReasonSwThermalSlowdown = 0x0000000000000020 -nvmlClocksThrottleReasonHwThermalSlowdown = 0x0000000000000040 +nvmlClocksThrottleReasonUserDefinedClocks = nvmlClocksThrottleReasonApplicationsClocksSetting # deprecated, use nvmlClocksThrottleReasonApplicationsClocksSetting +nvmlClocksThrottleReasonSwPowerCap = 0x0000000000000004 +nvmlClocksThrottleReasonHwSlowdown = 0x0000000000000008 +nvmlClocksThrottleReasonSyncBoost = 0x0000000000000010 +nvmlClocksThrottleReasonSwThermalSlowdown = 0x0000000000000020 +nvmlClocksThrottleReasonHwThermalSlowdown = 0x0000000000000040 nvmlClocksThrottleReasonHwPowerBrakeSlowdown = 0x0000000000000080 -nvmlClocksThrottleReasonDisplayClockSetting = 0x0000000000000100 -nvmlClocksThrottleReasonNone = 0x0000000000000000 -nvmlClocksThrottleReasonAll = ( - nvmlClocksThrottleReasonNone | - nvmlClocksThrottleReasonGpuIdle | - nvmlClocksThrottleReasonApplicationsClocksSetting | - nvmlClocksThrottleReasonSwPowerCap | - nvmlClocksThrottleReasonHwSlowdown | - nvmlClocksThrottleReasonSyncBoost | - nvmlClocksThrottleReasonSwThermalSlowdown | - nvmlClocksThrottleReasonHwThermalSlowdown | - nvmlClocksThrottleReasonHwPowerBrakeSlowdown | - nvmlClocksThrottleReasonDisplayClockSetting - ) +nvmlClocksThrottleReasonDisplayClockSetting = 0x0000000000000100 +nvmlClocksThrottleReasonNone = 0x0000000000000000 +nvmlClocksThrottleReasonAll = ( + nvmlClocksThrottleReasonNone + | nvmlClocksThrottleReasonGpuIdle + | nvmlClocksThrottleReasonApplicationsClocksSetting + | nvmlClocksThrottleReasonSwPowerCap + | nvmlClocksThrottleReasonHwSlowdown + | nvmlClocksThrottleReasonSyncBoost + | nvmlClocksThrottleReasonSwThermalSlowdown + | nvmlClocksThrottleReasonHwThermalSlowdown + | nvmlClocksThrottleReasonHwPowerBrakeSlowdown + | nvmlClocksThrottleReasonDisplayClockSetting +) + class c_nvmlEventData_t(_PrintableStructure): _fields_ = [ - ('device', c_nvmlDevice_t), - ('eventType', c_ulonglong), - ('eventData', c_ulonglong), - ('gpuInstanceId', c_uint), - ('computeInstanceId', c_uint) + ("device", c_nvmlDevice_t), + ("eventType", c_ulonglong), + ("eventData", c_ulonglong), + ("gpuInstanceId", c_uint), + ("computeInstanceId", c_uint), ] - _fmt_ = {'eventType': "0x%08X"} + _fmt_ = {"eventType": "0x%08X"} + class c_nvmlAccountingStats_t(_PrintableStructure): _fields_ = [ - ('gpuUtilization', c_uint), - ('memoryUtilization', c_uint), - ('maxMemoryUsage', c_ulonglong), - ('time', c_ulonglong), - ('startTime', c_ulonglong), - ('isRunning', c_uint), - ('reserved', c_uint * 5) + ("gpuUtilization", c_uint), + ("memoryUtilization", c_uint), + ("maxMemoryUsage", c_ulonglong), + ("time", c_ulonglong), + ("startTime", c_ulonglong), + ("isRunning", c_uint), + ("reserved", c_uint * 5), ] + class c_nvmlVgpuVersion_t(Structure): - _fields_ = [("minVersion", c_uint), - ("maxVersion", c_uint) - ] + _fields_ = [("minVersion", c_uint), ("maxVersion", c_uint)] + class c_nvmlVgpuMetadata_t(_PrintableStructure): - _fields_ = [("version", c_uint), - ("revision", c_uint), - ("guestInfoState", _nvmlVgpuGuestInfoState_t), - ("guestDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), - ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), - ("reserved", c_uint * 6), - ("vgpuVirtualizationCaps", c_uint), - ("guestVgpuVersion", c_uint), - ("opaqueDataSize", c_uint), - ("opaqueData", c_char * NVML_VGPU_METADATA_OPAQUE_DATA_SIZE) - ] + _fields_ = [ + ("version", c_uint), + ("revision", c_uint), + ("guestInfoState", _nvmlVgpuGuestInfoState_t), + ("guestDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ("reserved", c_uint * 6), + ("vgpuVirtualizationCaps", c_uint), + ("guestVgpuVersion", c_uint), + ("opaqueDataSize", c_uint), + ("opaqueData", c_char * NVML_VGPU_METADATA_OPAQUE_DATA_SIZE), + ] + class c_nvmlVgpuPgpuMetadata_t(_PrintableStructure): - _fields_ = [("version", c_uint), - ("revision", c_uint), - ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), - ("pgpuVirtualizationCaps", c_uint), - ("reserved", c_uint * 5), - ("hostSupportedVgpuRange", c_nvmlVgpuVersion_t), - ("opaqueDataSize", c_uint), - ("opaqueData", c_char * NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE) - ] + _fields_ = [ + ("version", c_uint), + ("revision", c_uint), + ("hostDriverVersion", c_char * NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE), + ("pgpuVirtualizationCaps", c_uint), + ("reserved", c_uint * 5), + ("hostSupportedVgpuRange", c_nvmlVgpuVersion_t), + ("opaqueDataSize", c_uint), + ("opaqueData", c_char * NVML_VGPU_PGPU_METADATA_OPAQUE_DATA_SIZE), + ] + class c_nvmlVgpuPgpuCompatibility_t(Structure): - _fields_ = [("vgpuVmCompatibility", _nvmlVgpuVmCompatibility_t), - ("compatibilityLimitCode", _nvmlVgpuPgpuCompatibilityLimitCode_t) - ] + _fields_ = [ + ("vgpuVmCompatibility", _nvmlVgpuVmCompatibility_t), + ("compatibilityLimitCode", _nvmlVgpuPgpuCompatibilityLimitCode_t), + ] + ## vGPU scheduler policy defines -NVML_VGPU_SCHEDULER_POLICY_UNKNOWN = 0 -NVML_VGPU_SCHEDULER_POLICY_BEST_EFFORT = 1 -NVML_VGPU_SCHEDULER_POLICY_EQUAL_SHARE = 2 -NVML_VGPU_SCHEDULER_POLICY_FIXED_SHARE = 3 +NVML_VGPU_SCHEDULER_POLICY_UNKNOWN = 0 +NVML_VGPU_SCHEDULER_POLICY_BEST_EFFORT = 1 +NVML_VGPU_SCHEDULER_POLICY_EQUAL_SHARE = 2 +NVML_VGPU_SCHEDULER_POLICY_FIXED_SHARE = 3 ## Supported vGPU scheduler policy count -NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT = 3 +NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT = 3 -NVML_SCHEDULER_SW_MAX_LOG_ENTRIES = 200 +NVML_SCHEDULER_SW_MAX_LOG_ENTRIES = 200 + +NVML_VGPU_SCHEDULER_ARR_DEFAULT = 0 +NVML_VGPU_SCHEDULER_ARR_DISABLE = 1 +NVML_VGPU_SCHEDULER_ARR_ENABLE = 2 -NVML_VGPU_SCHEDULER_ARR_DEFAULT = 0 -NVML_VGPU_SCHEDULER_ARR_DISABLE = 1 -NVML_VGPU_SCHEDULER_ARR_ENABLE = 2 class c_nvmlVgpuSchedDataWithARR_t(_PrintableStructure): _fields_ = [ - ('avgFactor', c_uint), - ('timeslice', c_uint), + ("avgFactor", c_uint), + ("timeslice", c_uint), ] + class c_nvmlVgpuSchedData_t(_PrintableStructure): _fields_ = [ - ('timeslice', c_uint), + ("timeslice", c_uint), ] + class c_nvmlVgpuSchedulerParams_t(Union): _fields_ = [ - ('vgpuSchedDataWithARR', c_nvmlVgpuSchedDataWithARR_t), - ('vgpuSchedData', c_nvmlVgpuSchedData_t), + ("vgpuSchedDataWithARR", c_nvmlVgpuSchedDataWithARR_t), + ("vgpuSchedData", c_nvmlVgpuSchedData_t), ] + class c_nvmlVgpuSchedulerLogEntry_t(_PrintableStructure): _fields_ = [ - ('timestamp', c_ulonglong), - ('timeRunTotal', c_ulonglong), - ('timeRun', c_ulonglong), - ('swRunlistId', c_uint), - ('targetTimeSlice', c_ulonglong), - ('cumulativePreemptionTime', c_ulonglong), + ("timestamp", c_ulonglong), + ("timeRunTotal", c_ulonglong), + ("timeRun", c_ulonglong), + ("swRunlistId", c_uint), + ("targetTimeSlice", c_ulonglong), + ("cumulativePreemptionTime", c_ulonglong), ] + class c_nvmlVgpuSchedulerLog_t(_PrintableStructure): _fields_ = [ - ('engineId', c_uint), - ('schedulerPolicy', c_uint), - ('arrMode', c_uint), - ('schedulerParams', c_nvmlVgpuSchedulerParams_t), - ('entriesCount', c_uint), - ('logEntries', c_nvmlVgpuSchedulerLogEntry_t * NVML_SCHEDULER_SW_MAX_LOG_ENTRIES), + ("engineId", c_uint), + ("schedulerPolicy", c_uint), + ("arrMode", c_uint), + ("schedulerParams", c_nvmlVgpuSchedulerParams_t), + ("entriesCount", c_uint), + ( + "logEntries", + c_nvmlVgpuSchedulerLogEntry_t * NVML_SCHEDULER_SW_MAX_LOG_ENTRIES, + ), ] + class c_nvmlVgpuSchedulerGetState_t(_PrintableStructure): _fields_ = [ - ('schedulerPolicy', c_uint), - ('arrMode', c_uint), - ('schedulerParams', c_nvmlVgpuSchedulerParams_t), + ("schedulerPolicy", c_uint), + ("arrMode", c_uint), + ("schedulerParams", c_nvmlVgpuSchedulerParams_t), ] + class c_nvmlVgpuSchedSetDataWithARR_t(_PrintableStructure): _fields_ = [ - ('avgFactor', c_uint), - ('frequency', c_uint), + ("avgFactor", c_uint), + ("frequency", c_uint), ] + class c_nvmlVgpuSchedSetData_t(_PrintableStructure): _fields_ = [ - ('timeslice', c_uint), + ("timeslice", c_uint), ] + class c_nvmlVgpuSchedulerSetParams_t(Union): _fields_ = [ - ('vgpuSchedDataWithARR', c_nvmlVgpuSchedSetDataWithARR_t), - ('vgpuSchedData', c_nvmlVgpuSchedSetData_t), + ("vgpuSchedDataWithARR", c_nvmlVgpuSchedSetDataWithARR_t), + ("vgpuSchedData", c_nvmlVgpuSchedSetData_t), ] + class c_nvmlVgpuSchedulerSetState_t(_PrintableStructure): _fields_ = [ - ('schedulerPolicy', c_uint), - ('enableARRMode', c_uint), - ('schedulerParams', c_nvmlVgpuSchedulerSetParams_t), + ("schedulerPolicy", c_uint), + ("enableARRMode", c_uint), + ("schedulerParams", c_nvmlVgpuSchedulerSetParams_t), ] + class c_nvmlVgpuSchedulerCapabilities_t(_PrintableStructure): _fields_ = [ - ('supportedSchedulers', c_uint * NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT), - ('maxTimeslice', c_uint), - ('minTimeslice', c_uint), - ('isArrModeSupported', c_uint), - ('maxFrequencyForARR', c_uint), - ('minFrequencyForARR', c_uint), - ('maxAvgFactorForARR', c_uint), - ('minAvgFactorForARR', c_uint), + ("supportedSchedulers", c_uint * NVML_SUPPORTED_VGPU_SCHEDULER_POLICY_COUNT), + ("maxTimeslice", c_uint), + ("minTimeslice", c_uint), + ("isArrModeSupported", c_uint), + ("maxFrequencyForARR", c_uint), + ("minFrequencyForARR", c_uint), + ("maxAvgFactorForARR", c_uint), + ("minAvgFactorForARR", c_uint), ] + class c_nvmlFBCStats_t(Structure): - _fields_ = [("sessionsCount", c_uint), - ("averageFPS", c_uint), - ("averageLatency", c_uint) - ] + _fields_ = [ + ("sessionsCount", c_uint), + ("averageFPS", c_uint), + ("averageLatency", c_uint), + ] + class c_nvmlFBCSession_t(_PrintableStructure): _fields_ = [ - ('sessionId', c_uint), - ('pid', c_uint), - ('vgpuInstance', _nvmlVgpuInstance_t), - ('displayOrdinal', c_uint), - ('sessionType', c_uint), - ('sessionFlags', c_uint), - ('hMaxResolution', c_uint), - ('vMaxResolution', c_uint), - ('hResolution', c_uint), - ('vResolution', c_uint), - ('averageFPS', c_uint), - ('averageLatency', c_uint), + ("sessionId", c_uint), + ("pid", c_uint), + ("vgpuInstance", _nvmlVgpuInstance_t), + ("displayOrdinal", c_uint), + ("sessionType", c_uint), + ("sessionFlags", c_uint), + ("hMaxResolution", c_uint), + ("vMaxResolution", c_uint), + ("hResolution", c_uint), + ("vResolution", c_uint), + ("averageFPS", c_uint), + ("averageLatency", c_uint), ] -NVML_DEVICE_MIG_DISABLE = 0x0 -NVML_DEVICE_MIG_ENABLE = 0x1 -NVML_GPU_INSTANCE_PROFILE_1_SLICE = 0x0 -NVML_GPU_INSTANCE_PROFILE_2_SLICE = 0x1 -NVML_GPU_INSTANCE_PROFILE_3_SLICE = 0x2 -NVML_GPU_INSTANCE_PROFILE_4_SLICE = 0x3 -NVML_GPU_INSTANCE_PROFILE_7_SLICE = 0x4 -NVML_GPU_INSTANCE_PROFILE_8_SLICE = 0x5 -NVML_GPU_INSTANCE_PROFILE_6_SLICE = 0x6 +NVML_DEVICE_MIG_DISABLE = 0x0 +NVML_DEVICE_MIG_ENABLE = 0x1 + +NVML_GPU_INSTANCE_PROFILE_1_SLICE = 0x0 +NVML_GPU_INSTANCE_PROFILE_2_SLICE = 0x1 +NVML_GPU_INSTANCE_PROFILE_3_SLICE = 0x2 +NVML_GPU_INSTANCE_PROFILE_4_SLICE = 0x3 +NVML_GPU_INSTANCE_PROFILE_7_SLICE = 0x4 +NVML_GPU_INSTANCE_PROFILE_8_SLICE = 0x5 +NVML_GPU_INSTANCE_PROFILE_6_SLICE = 0x6 NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7 NVML_GPU_INSTANCE_PROFILE_2_SLICE_REV1 = 0x8 NVML_GPU_INSTANCE_PROFILE_1_SLICE_REV2 = 0x9 -NVML_GPU_INSTANCE_PROFILE_1_SLICE_GFX = 0xA -NVML_GPU_INSTANCE_PROFILE_2_SLICE_GFX = 0xB -NVML_GPU_INSTANCE_PROFILE_4_SLICE_GFX = 0xC -NVML_GPU_INSTANCE_PROFILE_COUNT = 0xD +NVML_GPU_INSTANCE_PROFILE_1_SLICE_GFX = 0xA +NVML_GPU_INSTANCE_PROFILE_2_SLICE_GFX = 0xB +NVML_GPU_INSTANCE_PROFILE_4_SLICE_GFX = 0xC +NVML_GPU_INSTANCE_PROFILE_COUNT = 0xD + class c_nvmlGpuInstancePlacement_t(Structure): - _fields_ = [("start", c_uint), - ("size", c_uint) - ] + _fields_ = [("start", c_uint), ("size", c_uint)] + class c_nvmlGpuInstanceProfileInfo_t(Structure): - _fields_ = [("id", c_uint), - ("isP2pSupported", c_uint), - ("sliceCount", c_uint), - ("instanceCount", c_uint), - ("multiprocessorCount", c_uint), - ("copyEngineCount", c_uint), - ("decoderCount", c_uint), - ("encoderCount", c_uint), - ("jpegCount", c_uint), - ("ofaCount", c_uint), - ("memorySizeMB", c_ulonglong), - ] + _fields_ = [ + ("id", c_uint), + ("isP2pSupported", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("copyEngineCount", c_uint), + ("decoderCount", c_uint), + ("encoderCount", c_uint), + ("jpegCount", c_uint), + ("ofaCount", c_uint), + ("memorySizeMB", c_ulonglong), + ] + nvmlGpuInstanceProfileInfo_v2 = 0x02000098 + class c_nvmlGpuInstanceProfileInfo_v2_t(_PrintableStructure): - _fields_ = [("version", c_uint), - ("id", c_uint), - ("isP2pSupported", c_uint), - ("sliceCount", c_uint), - ("instanceCount", c_uint), - ("multiprocessorCount", c_uint), - ("copyEngineCount", c_uint), - ("decoderCount", c_uint), - ("encoderCount", c_uint), - ("jpegCount", c_uint), - ("ofaCount", c_uint), - ("memorySizeMB", c_ulonglong), - ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE) - ] - + _fields_ = [ + ("version", c_uint), + ("id", c_uint), + ("isP2pSupported", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("copyEngineCount", c_uint), + ("decoderCount", c_uint), + ("encoderCount", c_uint), + ("jpegCount", c_uint), + ("ofaCount", c_uint), + ("memorySizeMB", c_ulonglong), + ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE), + ] + def __init__(self): - super(c_nvmlGpuInstanceProfileInfo_v2_t, self).__init__(version=nvmlGpuInstanceProfileInfo_v2) + super(c_nvmlGpuInstanceProfileInfo_v2_t, self).__init__( + version=nvmlGpuInstanceProfileInfo_v2 + ) + class c_nvmlGpuInstanceInfo_t(Structure): - _fields_ = [("device", c_nvmlDevice_t), - ("id", c_uint), - ("profileId", c_uint), - ("placement", c_nvmlGpuInstancePlacement_t) - ] + _fields_ = [ + ("device", c_nvmlDevice_t), + ("id", c_uint), + ("profileId", c_uint), + ("placement", c_nvmlGpuInstancePlacement_t), + ] + class struct_c_nvmlGpuInstance_t(Structure): - pass # opaque handle + pass # opaque handle + + c_nvmlGpuInstance_t = POINTER(struct_c_nvmlGpuInstance_t) -NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE = 0x0 -NVML_COMPUTE_INSTANCE_PROFILE_2_SLICE = 0x1 -NVML_COMPUTE_INSTANCE_PROFILE_3_SLICE = 0x2 -NVML_COMPUTE_INSTANCE_PROFILE_4_SLICE = 0x3 -NVML_COMPUTE_INSTANCE_PROFILE_7_SLICE = 0x4 -NVML_COMPUTE_INSTANCE_PROFILE_8_SLICE = 0x5 -NVML_COMPUTE_INSTANCE_PROFILE_6_SLICE = 0x6 +NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE = 0x0 +NVML_COMPUTE_INSTANCE_PROFILE_2_SLICE = 0x1 +NVML_COMPUTE_INSTANCE_PROFILE_3_SLICE = 0x2 +NVML_COMPUTE_INSTANCE_PROFILE_4_SLICE = 0x3 +NVML_COMPUTE_INSTANCE_PROFILE_7_SLICE = 0x4 +NVML_COMPUTE_INSTANCE_PROFILE_8_SLICE = 0x5 +NVML_COMPUTE_INSTANCE_PROFILE_6_SLICE = 0x6 NVML_COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1 = 0x7 -NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 0x8 +NVML_COMPUTE_INSTANCE_PROFILE_COUNT = 0x8 NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_SHARED = 0x0 NVML_COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT = 0x1 + class c_nvmlComputeInstancePlacement_t(Structure): - _fields_ = [("start", c_uint), - ("size", c_uint) - ] + _fields_ = [("start", c_uint), ("size", c_uint)] + class c_nvmlComputeInstanceProfileInfo_t(Structure): - _fields_ = [("id", c_uint), - ("sliceCount", c_uint), - ("instanceCount", c_uint), - ("multiprocessorCount", c_uint), - ("sharedCopyEngineCount", c_uint), - ("sharedDecoderCount", c_uint), - ("sharedEncoderCount", c_uint), - ("sharedJpegCount", c_uint), - ("sharedOfaCount", c_uint) - ] + _fields_ = [ + ("id", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("sharedCopyEngineCount", c_uint), + ("sharedDecoderCount", c_uint), + ("sharedEncoderCount", c_uint), + ("sharedJpegCount", c_uint), + ("sharedOfaCount", c_uint), + ] + nvmlComputeInstanceProfileInfo_v2 = 0x02000088 + class c_nvmlComputeInstanceProfileInfo_v2_t(_PrintableStructure): - _fields_ = [("version", c_uint), - ("id", c_uint), - ("sliceCount", c_uint), - ("instanceCount", c_uint), - ("multiprocessorCount", c_uint), - ("sharedCopyEngineCount", c_uint), - ("sharedDecoderCount", c_uint), - ("sharedEncoderCount", c_uint), - ("sharedJpegCount", c_uint), - ("sharedOfaCount", c_uint), - ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE) - ] + _fields_ = [ + ("version", c_uint), + ("id", c_uint), + ("sliceCount", c_uint), + ("instanceCount", c_uint), + ("multiprocessorCount", c_uint), + ("sharedCopyEngineCount", c_uint), + ("sharedDecoderCount", c_uint), + ("sharedEncoderCount", c_uint), + ("sharedJpegCount", c_uint), + ("sharedOfaCount", c_uint), + ("name", c_char * NVML_DEVICE_NAME_V2_BUFFER_SIZE), + ] def __init__(self): - super(c_nvmlComputeInstanceProfileInfo_v2_t, self).__init__(version=nvmlComputeInstanceProfileInfo_v2) + super(c_nvmlComputeInstanceProfileInfo_v2_t, self).__init__( + version=nvmlComputeInstanceProfileInfo_v2 + ) + class c_nvmlComputeInstanceInfo_t(Structure): - _fields_ = [("device", c_nvmlDevice_t), - ("gpuInstance", c_nvmlGpuInstance_t), - ("id", c_uint), - ("profileId", c_uint), - ("placement", c_nvmlComputeInstancePlacement_t) - ] + _fields_ = [ + ("device", c_nvmlDevice_t), + ("gpuInstance", c_nvmlGpuInstance_t), + ("id", c_uint), + ("profileId", c_uint), + ("placement", c_nvmlComputeInstancePlacement_t), + ] + NVML_MAX_GPU_UTILIZATIONS = 8 -NVML_GPU_UTILIZATION_DOMAIN_GPU = 0 -NVML_GPU_UTILIZATION_DOMAIN_FB = 1 -NVML_GPU_UTILIZATION_DOMAIN_VID = 2 -NVML_GPU_UTILIZATION_DOMAIN_BUS = 3 +NVML_GPU_UTILIZATION_DOMAIN_GPU = 0 +NVML_GPU_UTILIZATION_DOMAIN_FB = 1 +NVML_GPU_UTILIZATION_DOMAIN_VID = 2 +NVML_GPU_UTILIZATION_DOMAIN_BUS = 3 + + class c_nvmlGpuDynamicPstatesUtilization_t(Structure): - _fields_ = [("bIsPresent", c_uint, 1), - ("percentage", c_uint), - ("incThreshold", c_uint), - ("decThreshold", c_uint)] + _fields_ = [ + ("bIsPresent", c_uint, 1), + ("percentage", c_uint), + ("incThreshold", c_uint), + ("decThreshold", c_uint), + ] + + class c_nvmlGpuDynamicPstatesInfo_t(Structure): - _fields_ = [("flags", c_uint), - ("utilization", c_nvmlGpuDynamicPstatesUtilization_t * NVML_MAX_GPU_UTILIZATIONS)] + _fields_ = [ + ("flags", c_uint), + ( + "utilization", + c_nvmlGpuDynamicPstatesUtilization_t * NVML_MAX_GPU_UTILIZATIONS, + ), + ] + NVML_MAX_THERMAL_SENSORS_PER_GPU = 3 -NVML_THERMAL_TARGET_NONE = 0 -NVML_THERMAL_TARGET_GPU = 1 -NVML_THERMAL_TARGET_MEMORY = 2 -NVML_THERMAL_TARGET_POWER_SUPPLY = 4 -NVML_THERMAL_TARGET_BOARD = 8 -NVML_THERMAL_TARGET_VCD_BOARD = 9 -NVML_THERMAL_TARGET_VCD_INLET = 10 -NVML_THERMAL_TARGET_VCD_OUTLET = 11 -NVML_THERMAL_TARGET_ALL = 15 -NVML_THERMAL_TARGET_UNKNOWN = -1 +NVML_THERMAL_TARGET_NONE = 0 +NVML_THERMAL_TARGET_GPU = 1 +NVML_THERMAL_TARGET_MEMORY = 2 +NVML_THERMAL_TARGET_POWER_SUPPLY = 4 +NVML_THERMAL_TARGET_BOARD = 8 +NVML_THERMAL_TARGET_VCD_BOARD = 9 +NVML_THERMAL_TARGET_VCD_INLET = 10 +NVML_THERMAL_TARGET_VCD_OUTLET = 11 +NVML_THERMAL_TARGET_ALL = 15 +NVML_THERMAL_TARGET_UNKNOWN = -1 -NVML_THERMAL_CONTROLLER_NONE = 0 -NVML_THERMAL_CONTROLLER_GPU_INTERNAL = 1 -NVML_THERMAL_CONTROLLER_ADM1032 = 2 -NVML_THERMAL_CONTROLLER_ADT7461 = 3 -NVML_THERMAL_CONTROLLER_MAX6649 = 4 -NVML_THERMAL_CONTROLLER_MAX1617 = 5 -NVML_THERMAL_CONTROLLER_LM99 = 6 -NVML_THERMAL_CONTROLLER_LM89 = 7 -NVML_THERMAL_CONTROLLER_LM64 = 8 -NVML_THERMAL_CONTROLLER_G781 = 9 -NVML_THERMAL_CONTROLLER_ADT7473 = 10 -NVML_THERMAL_CONTROLLER_SBMAX6649 = 11 -NVML_THERMAL_CONTROLLER_VBIOSEVT = 12 -NVML_THERMAL_CONTROLLER_OS = 13 +NVML_THERMAL_CONTROLLER_NONE = 0 +NVML_THERMAL_CONTROLLER_GPU_INTERNAL = 1 +NVML_THERMAL_CONTROLLER_ADM1032 = 2 +NVML_THERMAL_CONTROLLER_ADT7461 = 3 +NVML_THERMAL_CONTROLLER_MAX6649 = 4 +NVML_THERMAL_CONTROLLER_MAX1617 = 5 +NVML_THERMAL_CONTROLLER_LM99 = 6 +NVML_THERMAL_CONTROLLER_LM89 = 7 +NVML_THERMAL_CONTROLLER_LM64 = 8 +NVML_THERMAL_CONTROLLER_G781 = 9 +NVML_THERMAL_CONTROLLER_ADT7473 = 10 +NVML_THERMAL_CONTROLLER_SBMAX6649 = 11 +NVML_THERMAL_CONTROLLER_VBIOSEVT = 12 +NVML_THERMAL_CONTROLLER_OS = 13 NVML_THERMAL_CONTROLLER_NVSYSCON_CANOAS = 14 -NVML_THERMAL_CONTROLLER_NVSYSCON_E551 = 15 -NVML_THERMAL_CONTROLLER_MAX6649R = 16 -NVML_THERMAL_CONTROLLER_ADT7473S = 17 -NVML_THERMAL_CONTROLLER_UNKNOWN = -1 +NVML_THERMAL_CONTROLLER_NVSYSCON_E551 = 15 +NVML_THERMAL_CONTROLLER_MAX6649R = 16 +NVML_THERMAL_CONTROLLER_ADT7473S = 17 +NVML_THERMAL_CONTROLLER_UNKNOWN = -1 + class c_nvmlGpuThermalSensor_t(Structure): - _fields_ = [("controller", c_int), - ("defaultMinTemp", c_int), - ("defaultMaxTemp", c_int), - ("currentTemp", c_int), - ("target", c_int)] + _fields_ = [ + ("controller", c_int), + ("defaultMinTemp", c_int), + ("defaultMaxTemp", c_int), + ("currentTemp", c_int), + ("target", c_int), + ] + + class c_nvmlGpuThermalSettings_t(Structure): - _fields_ = [("count", c_uint), - ("sensor", c_nvmlGpuThermalSensor_t * NVML_MAX_THERMAL_SENSORS_PER_GPU)] + _fields_ = [ + ("count", c_uint), + ("sensor", c_nvmlGpuThermalSensor_t * NVML_MAX_THERMAL_SENSORS_PER_GPU), + ] + _nvmlCoolerControl_t = c_uint -NVML_THERMAL_COOLER_SIGNAL_NONE = 0 -NVML_THERMAL_COOLER_SIGNAL_TOGGLE = 1 -NVML_THERMAL_COOLER_SIGNAL_VARIABLE = 2 -NVML_THERMAL_COOLER_SIGNAL_COUNT = 3 +NVML_THERMAL_COOLER_SIGNAL_NONE = 0 +NVML_THERMAL_COOLER_SIGNAL_TOGGLE = 1 +NVML_THERMAL_COOLER_SIGNAL_VARIABLE = 2 +NVML_THERMAL_COOLER_SIGNAL_COUNT = 3 _nvmlCoolerTarget_t = c_uint -NVML_THERMAL_COOLER_TARGET_NONE = (1 << 0) -NVML_THERMAL_COOLER_TARGET_GPU = (1 << 1) -NVML_THERMAL_COOLER_TARGET_MEMORY = (1 << 2) -NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY = (1 << 3) -NVML_THERMAL_COOLER_TARGET_GPU_RELATED = (NVML_THERMAL_COOLER_TARGET_GPU | NVML_THERMAL_COOLER_TARGET_MEMORY | NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY) +NVML_THERMAL_COOLER_TARGET_NONE = 1 << 0 +NVML_THERMAL_COOLER_TARGET_GPU = 1 << 1 +NVML_THERMAL_COOLER_TARGET_MEMORY = 1 << 2 +NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY = 1 << 3 +NVML_THERMAL_COOLER_TARGET_GPU_RELATED = ( + NVML_THERMAL_COOLER_TARGET_GPU + | NVML_THERMAL_COOLER_TARGET_MEMORY + | NVML_THERMAL_COOLER_TARGET_POWER_SUPPLY +) + class c_nvmlCoolerInfo_t(_PrintableStructure): - _fields_ = [("version", c_uint), - ("index", c_uint), - ("coolerControlType", _nvmlCoolerControl_t), - ("coolerTarget", _nvmlCoolerTarget_t) - ] + _fields_ = [ + ("version", c_uint), + ("index", c_uint), + ("coolerControlType", _nvmlCoolerControl_t), + ("coolerTarget", _nvmlCoolerTarget_t), + ] + nvmlCoolerInfo_v1 = 0x1000010 + def nvmlDeviceGetCoolerInfo(handle): c_coolerInfo = c_nvmlCoolerInfo_t() c_coolerInfo.version = nvmlCoolerInfo_v1 @@ -2241,105 +2672,138 @@ def nvmlDeviceGetCoolerInfo(handle): _nvmlCheckReturn(ret) return [c_coolerInfo.coolerControlType, c_coolerInfo.coolerTarget] + class struct_c_nvmlComputeInstance_t(Structure): - pass # opaque handle + pass # opaque handle + + c_nvmlComputeInstance_t = POINTER(struct_c_nvmlComputeInstance_t) + class c_nvmlDeviceAttributes(Structure): - _fields_ = [("multiprocessorCount", c_uint), - ("sharedCopyEngineCount", c_uint), - ("sharedDecoderCount", c_uint), - ("sharedEncoderCount", c_uint), - ("sharedJpegCount", c_uint), - ("sharedOfaCount", c_uint), - ("gpuInstanceSliceCount", c_uint), - ("computeInstanceSliceCount", c_uint), - ("memorySizeMB", c_ulonglong), - ] + _fields_ = [ + ("multiprocessorCount", c_uint), + ("sharedCopyEngineCount", c_uint), + ("sharedDecoderCount", c_uint), + ("sharedEncoderCount", c_uint), + ("sharedJpegCount", c_uint), + ("sharedOfaCount", c_uint), + ("gpuInstanceSliceCount", c_uint), + ("computeInstanceSliceCount", c_uint), + ("memorySizeMB", c_ulonglong), + ] + class c_nvmlRowRemapperHistogramValues(Structure): - _fields_ = [("max", c_uint), - ("high", c_uint), - ("partial", c_uint), - ("low", c_uint), - ("none", c_uint) - ] + _fields_ = [ + ("max", c_uint), + ("high", c_uint), + ("partial", c_uint), + ("low", c_uint), + ("none", c_uint), + ] -NVML_GPU_CERT_CHAIN_SIZE = 0x1000 -NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE = 0x1400 -NVML_CC_GPU_CEC_NONCE_SIZE = 0x20 -NVML_CC_GPU_ATTESTATION_REPORT_SIZE = 0x2000 + +NVML_GPU_CERT_CHAIN_SIZE = 0x1000 +NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE = 0x1400 +NVML_CC_GPU_CEC_NONCE_SIZE = 0x20 +NVML_CC_GPU_ATTESTATION_REPORT_SIZE = 0x2000 NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE = 0x1000 NVML_CC_CEC_ATTESTATION_REPORT_NOT_PRESENT = 0 -NVML_CC_CEC_ATTESTATION_REPORT_PRESENT = 1 +NVML_CC_CEC_ATTESTATION_REPORT_PRESENT = 1 + class c_nvmlConfComputeSystemState_t(Structure): - _fields_ = [('environment', c_uint), - ('ccFeature', c_uint), - ('devToolsMode', c_uint), - ] + _fields_ = [ + ("environment", c_uint), + ("ccFeature", c_uint), + ("devToolsMode", c_uint), + ] + nvmlSystemConfComputeSettings_v1 = 0x1000014 + class c_nvmlSystemConfComputeSettings_v1_t(Structure): - _fields_ = [('version', c_uint), - ('environment', c_uint), - ('ccFeature', c_uint), - ('devToolsMode', c_uint), - ('multiGpuMode', c_uint), - ] + _fields_ = [ + ("version", c_uint), + ("environment", c_uint), + ("ccFeature", c_uint), + ("devToolsMode", c_uint), + ("multiGpuMode", c_uint), + ] + def __init__(self): - super(c_nvmlSystemConfComputeSettings_v1_t, self).__init__(version=nvmlSystemConfComputeSettings_v1) + super(c_nvmlSystemConfComputeSettings_v1_t, self).__init__( + version=nvmlSystemConfComputeSettings_v1 + ) + class c_nvmlConfComputeSystemCaps_t(Structure): - _fields_ = [('cpuCaps', c_uint), - ('gpusCaps', c_uint), - ] + _fields_ = [ + ("cpuCaps", c_uint), + ("gpusCaps", c_uint), + ] + class c_nvmlConfComputeMemSizeInfo_t(Structure): - _fields_ = [('protectedMemSizeKib', c_ulonglong), - ('unprotectedMemSizeKib', c_ulonglong), - ] + _fields_ = [ + ("protectedMemSizeKib", c_ulonglong), + ("unprotectedMemSizeKib", c_ulonglong), + ] + class c_nvmlConfComputeGpuCertificate_t(Structure): - _fields_ = [('certChainSize', c_uint), - ('attestationCertChainSize', c_uint), - ('certChain', c_uint8 * NVML_GPU_CERT_CHAIN_SIZE), - ('attestationCertChain', c_uint8 * NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE), - ] + _fields_ = [ + ("certChainSize", c_uint), + ("attestationCertChainSize", c_uint), + ("certChain", c_uint8 * NVML_GPU_CERT_CHAIN_SIZE), + ("attestationCertChain", c_uint8 * NVML_GPU_ATTESTATION_CERT_CHAIN_SIZE), + ] + class c_nvmlConfComputeGpuAttestationReport_t(Structure): - _fields_ = [('isCecAttestationReportPresent', c_uint), - ('attestationReportSize', c_uint), - ('cecAttestationReportSize', c_uint), - ('nonce', c_uint8 * NVML_CC_GPU_CEC_NONCE_SIZE), - ('attestationReport', c_uint8 * NVML_CC_GPU_ATTESTATION_REPORT_SIZE), - ('cecAttestationReport', c_uint8 * NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE), - ] + _fields_ = [ + ("isCecAttestationReportPresent", c_uint), + ("attestationReportSize", c_uint), + ("cecAttestationReportSize", c_uint), + ("nonce", c_uint8 * NVML_CC_GPU_CEC_NONCE_SIZE), + ("attestationReport", c_uint8 * NVML_CC_GPU_ATTESTATION_REPORT_SIZE), + ("cecAttestationReport", c_uint8 * NVML_CC_GPU_CEC_ATTESTATION_REPORT_SIZE), + ] + class c_nvmlConfComputeSetKeyRotationThresholdInfo_t(Structure): - _fields_ = [('version', c_uint), - ('maxAttackerAdvantage', c_ulong), - ] + _fields_ = [ + ("version", c_uint), + ("maxAttackerAdvantage", c_ulong), + ] + + ConfComputeSetKeyRotationThresholdInfo_v1 = 0x1000010 + class c_nvmlConfComputeGetKeyRotationThresholdInfo_t(Structure): - _fields_ = [('version', c_uint), - ('attackerAdvantage', c_ulong), - ] + _fields_ = [ + ("version", c_uint), + ("attackerAdvantage", c_ulong), + ] + + ConfComputeGetKeyRotationThresholdInfo_v1 = 0x1000010 ## string/bytes conversion for ease of use def convertStrBytes(func): - ''' + """ In python 3, strings are unicode instead of bytes, and need to be converted for ctypes Args from caller: (1, 'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF>) Args passed to function: (1, b'string', <__main__.c_nvmlDevice_t at 0xFFFFFFFF)> ---- Returned from function: b'returned string' Returned to caller: 'returned string' - ''' + """ + @wraps(func) def wrapper(*args, **kwargs): # encoding a str returns bytes in python 2 and 3 @@ -2358,18 +2822,23 @@ def convertStrBytes(func): return wrapper return func + def throwOnVersionMismatch(func): @wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except NVMLError_FunctionNotFound: - raise NVMLLibraryMismatchError("Unversioned function called and the " - "pyNVML version does not match the NVML lib version. " - "Either use matching pyNVML and NVML lib versions or " - "use a versioned function such as " + func.__name__ + "_v2") + raise NVMLLibraryMismatchError( + "Unversioned function called and the " + "pyNVML version does not match the NVML lib version. " + "Either use matching pyNVML and NVML lib versions or " + "use a versioned function such as " + func.__name__ + "_v2" + ) + return wrapper + ## C function wrappers ## def nvmlInitWithFlags(flags): _LoadNvmlLibrary() @@ -2388,44 +2857,57 @@ def nvmlInitWithFlags(flags): libLoadLock.release() return None + def nvmlInit(): nvmlInitWithFlags(0) return None + def _LoadNvmlLibrary(): - ''' + """ Load the library if it isn't loaded already - ''' + """ global nvmlLib - if (nvmlLib == None): + if nvmlLib == None: # lock to ensure only one caller loads the library libLoadLock.acquire() try: # ensure the library still isn't loaded - if (nvmlLib == None): + if nvmlLib == None: try: - if (sys.platform[:3] == "win"): + if sys.platform[:3] == "win": # cdecl calling convention try: # Check for nvml.dll in System32 first for DCH drivers - nvmlLib = CDLL(os.path.join(os.getenv("WINDIR", "C:/Windows"), "System32/nvml.dll")) + nvmlLib = CDLL( + os.path.join( + os.getenv("WINDIR", "C:/Windows"), + "System32/nvml.dll", + ) + ) except OSError as ose: # If nvml.dll is not found in System32, it should be in ProgramFiles # load nvml.dll from %ProgramFiles%/NVIDIA Corporation/NVSMI/nvml.dll - nvmlLib = CDLL(os.path.join(os.getenv("ProgramFiles", "C:/Program Files"), "NVIDIA Corporation/NVSMI/nvml.dll")) + nvmlLib = CDLL( + os.path.join( + os.getenv("ProgramFiles", "C:/Program Files"), + "NVIDIA Corporation/NVSMI/nvml.dll", + ) + ) else: # assume linux nvmlLib = CDLL("libnvidia-ml.so.1") except OSError as ose: _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND) - if (nvmlLib == None): + if nvmlLib == None: _nvmlCheckReturn(NVML_ERROR_LIBRARY_NOT_FOUND) finally: # lock is always freed libLoadLock.release() + def nvmlShutdown(): # # Leave the library loaded, but shutdown the interface @@ -2437,19 +2919,21 @@ def nvmlShutdown(): # Atomically update refcount global _nvmlLib_refcount libLoadLock.acquire() - if (0 < _nvmlLib_refcount): + if 0 < _nvmlLib_refcount: _nvmlLib_refcount -= 1 libLoadLock.release() return None + # Added in 2.285 @convertStrBytes def nvmlErrorString(result): fn = _nvmlGetFunctionPointer("nvmlErrorString") - fn.restype = c_char_p # otherwise return is an int + fn.restype = c_char_p # otherwise return is an int ret = fn(result) return ret + # Added in 2.285 @convertStrBytes def nvmlSystemGetNVMLVersion(): @@ -2459,6 +2943,7 @@ def nvmlSystemGetNVMLVersion(): _nvmlCheckReturn(ret) return c_version.value + def nvmlSystemGetCudaDriverVersion(): c_cuda_version = c_int() fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion") @@ -2466,6 +2951,7 @@ def nvmlSystemGetCudaDriverVersion(): _nvmlCheckReturn(ret) return c_cuda_version.value + def nvmlSystemGetCudaDriverVersion_v2(): c_cuda_version = c_int() fn = _nvmlGetFunctionPointer("nvmlSystemGetCudaDriverVersion_v2") @@ -2473,6 +2959,7 @@ def nvmlSystemGetCudaDriverVersion_v2(): _nvmlCheckReturn(ret) return c_cuda_version.value + # Added in 2.285 @convertStrBytes def nvmlSystemGetProcessName(pid): @@ -2482,6 +2969,7 @@ def nvmlSystemGetProcessName(pid): _nvmlCheckReturn(ret) return c_name.value + @convertStrBytes def nvmlSystemGetDriverVersion(): c_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE) @@ -2490,6 +2978,7 @@ def nvmlSystemGetDriverVersion(): _nvmlCheckReturn(ret) return c_version.value + # Added in 2.285 def nvmlSystemGetHicVersion(): c_count = c_uint(0) @@ -2500,12 +2989,11 @@ def nvmlSystemGetHicVersion(): ret = fn(byref(c_count), None) # this should only fail with insufficient size - if ((ret != NVML_SUCCESS) and - (ret != NVML_ERROR_INSUFFICIENT_SIZE)): + if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE): raise NVMLError(ret) # If there are no hics - if (c_count.value == 0): + if c_count.value == 0: return [] hic_array = c_nvmlHwbcEntry_t * c_count.value @@ -2514,14 +3002,16 @@ def nvmlSystemGetHicVersion(): _nvmlCheckReturn(ret) return hics + def nvmlSystemGetDriverBranch(): c_branchInfo = c_nvmlSystemDriverBranchInfo_v1_t(0) c_branchInfo.version = SystemDriverBranchInfo_v1 - fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverBranch") + fn = _nvmlGetFunctionPointer("nvmlSystemGetDriverBranch") ret = fn(byref(c_branchInfo), c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE)) _nvmlCheckReturn(ret) return c_branchInfo + ## Unit get functions def nvmlUnitGetCount(): c_count = c_uint() @@ -2530,6 +3020,7 @@ def nvmlUnitGetCount(): _nvmlCheckReturn(ret) return c_count.value + def nvmlUnitGetHandleByIndex(index): c_index = c_uint(index) unit = c_nvmlUnit_t() @@ -2538,6 +3029,7 @@ def nvmlUnitGetHandleByIndex(index): _nvmlCheckReturn(ret) return unit + def nvmlUnitGetUnitInfo(unit): c_info = c_nvmlUnitInfo_t() fn = _nvmlGetFunctionPointer("nvmlUnitGetUnitInfo") @@ -2545,13 +3037,15 @@ def nvmlUnitGetUnitInfo(unit): _nvmlCheckReturn(ret) return c_info + def nvmlUnitGetLedState(unit): - c_state = c_nvmlLedState_t() + c_state = c_nvmlLedState_t() fn = _nvmlGetFunctionPointer("nvmlUnitGetLedState") ret = fn(unit, byref(c_state)) _nvmlCheckReturn(ret) return c_state + def nvmlUnitGetPsuInfo(unit): c_info = c_nvmlPSUInfo_t() fn = _nvmlGetFunctionPointer("nvmlUnitGetPsuInfo") @@ -2559,6 +3053,7 @@ def nvmlUnitGetPsuInfo(unit): _nvmlCheckReturn(ret) return c_info + def nvmlUnitGetTemperature(unit, type): c_temp = c_uint() fn = _nvmlGetFunctionPointer("nvmlUnitGetTemperature") @@ -2566,6 +3061,7 @@ def nvmlUnitGetTemperature(unit, type): _nvmlCheckReturn(ret) return c_temp.value + def nvmlUnitGetFanSpeedInfo(unit): c_speeds = c_nvmlUnitFanSpeeds_t() fn = _nvmlGetFunctionPointer("nvmlUnitGetFanSpeedInfo") @@ -2573,17 +3069,19 @@ def nvmlUnitGetFanSpeedInfo(unit): _nvmlCheckReturn(ret) return c_speeds + # added to API def nvmlUnitGetDeviceCount(unit): c_count = c_uint(0) # query the unit to determine device count fn = _nvmlGetFunctionPointer("nvmlUnitGetDevices") ret = fn(unit, byref(c_count), None) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: ret = NVML_SUCCESS _nvmlCheckReturn(ret) return c_count.value + def nvmlUnitGetDevices(unit): c_count = c_uint(nvmlUnitGetDeviceCount(unit)) device_array = c_nvmlDevice_t * c_count.value @@ -2593,6 +3091,7 @@ def nvmlUnitGetDevices(unit): _nvmlCheckReturn(ret) return c_devices + ## Device get functions def nvmlDeviceGetCount(): c_count = c_uint() @@ -2601,6 +3100,7 @@ def nvmlDeviceGetCount(): _nvmlCheckReturn(ret) return c_count.value + def nvmlDeviceGetHandleByIndex(index): c_index = c_uint(index) device = c_nvmlDevice_t() @@ -2609,6 +3109,7 @@ def nvmlDeviceGetHandleByIndex(index): _nvmlCheckReturn(ret) return device + @convertStrBytes def nvmlDeviceGetHandleBySerial(serial): c_serial = c_char_p(serial) @@ -2618,6 +3119,7 @@ def nvmlDeviceGetHandleBySerial(serial): _nvmlCheckReturn(ret) return device + @convertStrBytes def nvmlDeviceGetHandleByUUID(uuid): c_uuid = c_char_p(uuid) @@ -2627,6 +3129,7 @@ def nvmlDeviceGetHandleByUUID(uuid): _nvmlCheckReturn(ret) return device + @convertStrBytes def nvmlDeviceGetHandleByPciBusId(pciBusId): c_busId = c_char_p(pciBusId) @@ -2636,6 +3139,7 @@ def nvmlDeviceGetHandleByPciBusId(pciBusId): _nvmlCheckReturn(ret) return device + @convertStrBytes def nvmlDeviceGetName(handle): c_name = create_string_buffer(NVML_DEVICE_NAME_V2_BUFFER_SIZE) @@ -2644,14 +3148,17 @@ def nvmlDeviceGetName(handle): _nvmlCheckReturn(ret) return c_name.value + class c_nvmlDevicePerfModes_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('str', c_char * NVML_PERF_MODES_BUFFER_SIZE), + ("version", c_uint), + ("str", c_char * NVML_PERF_MODES_BUFFER_SIZE), ] + nvmlDevicePerfModes_v1 = 0x1000804 + @convertStrBytes def nvmlDeviceGetPerformanceModes(handle): perfModes = c_nvmlDevicePerfModes_v1_t() @@ -2661,14 +3168,17 @@ def nvmlDeviceGetPerformanceModes(handle): _nvmlCheckReturn(ret) return perfModes.str + class c_nvmlDeviceCurrentClockFreqs_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('str', c_char * NVML_PERF_MODES_BUFFER_SIZE), + ("version", c_uint), + ("str", c_char * NVML_PERF_MODES_BUFFER_SIZE), ] + nvmlDeviceCurrentClockFreqs_v1 = 0x1000804 + @convertStrBytes def nvmlDeviceGetCurrentClockFreqs(handle): currentClockFreqs = c_nvmlDeviceCurrentClockFreqs_v1_t() @@ -2678,20 +3188,23 @@ def nvmlDeviceGetCurrentClockFreqs(handle): _nvmlCheckReturn(ret) return currentClockFreqs.str + def nvmlDeviceGetBoardId(handle): - c_id = c_uint(); + c_id = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetBoardId") ret = fn(handle, byref(c_id)) _nvmlCheckReturn(ret) return c_id.value + def nvmlDeviceGetMultiGpuBoard(handle): - c_multiGpu = c_uint(); + c_multiGpu = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMultiGpuBoard") ret = fn(handle, byref(c_multiGpu)) _nvmlCheckReturn(ret) return c_multiGpu.value + def nvmlDeviceGetBrand(handle): c_type = _nvmlBrandType_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetBrand") @@ -2699,6 +3212,7 @@ def nvmlDeviceGetBrand(handle): _nvmlCheckReturn(ret) return c_type.value + def nvmlDeviceGetC2cModeInfoV1(handle): c_info = c_nvmlC2cModeInfo_v1_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetC2cModeInfoV") @@ -2706,9 +3220,11 @@ def nvmlDeviceGetC2cModeInfoV1(handle): _nvmlCheckReturn(ret) return c_info + def nvmlDeviceGetC2cModeInfoV(handle): return nvmlDeviceGetC2cModeInfoV1(handle) + @convertStrBytes def nvmlDeviceGetBoardPartNumber(handle): c_part_number = create_string_buffer(NVML_DEVICE_PART_NUMBER_BUFFER_SIZE) @@ -2717,6 +3233,7 @@ def nvmlDeviceGetBoardPartNumber(handle): _nvmlCheckReturn(ret) return c_part_number.value + @convertStrBytes def nvmlDeviceGetSerial(handle): c_serial = create_string_buffer(NVML_DEVICE_SERIAL_BUFFER_SIZE) @@ -2725,6 +3242,7 @@ def nvmlDeviceGetSerial(handle): _nvmlCheckReturn(ret) return c_serial.value + def nvmlDeviceGetModuleId(handle, moduleId=c_uint()): isReference = type(moduleId) is not c_uint moduleIdRef = moduleId if isReference else byref(moduleId) @@ -2736,6 +3254,7 @@ def nvmlDeviceGetModuleId(handle, moduleId=c_uint()): _nvmlCheckReturn(ret) return moduleId.value + def nvmlDeviceGetMemoryAffinity(handle, nodeSetSize, scope): affinity_array = c_ulonglong * nodeSetSize c_affinity = affinity_array() @@ -2744,6 +3263,7 @@ def nvmlDeviceGetMemoryAffinity(handle, nodeSetSize, scope): _nvmlCheckReturn(ret) return c_affinity + def nvmlDeviceGetCpuAffinityWithinScope(handle, cpuSetSize, scope): affinity_array = c_ulonglong * cpuSetSize c_affinity = affinity_array() @@ -2752,6 +3272,7 @@ def nvmlDeviceGetCpuAffinityWithinScope(handle, cpuSetSize, scope): _nvmlCheckReturn(ret) return c_affinity + def nvmlDeviceGetCpuAffinity(handle, cpuSetSize): affinity_array = c_ulonglong * cpuSetSize c_affinity = affinity_array() @@ -2760,18 +3281,21 @@ def nvmlDeviceGetCpuAffinity(handle, cpuSetSize): _nvmlCheckReturn(ret) return c_affinity + def nvmlDeviceSetCpuAffinity(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceSetCpuAffinity") ret = fn(handle) _nvmlCheckReturn(ret) return None + def nvmlDeviceClearCpuAffinity(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceClearCpuAffinity") ret = fn(handle) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetNumaNodeId(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumaNodeId") node = c_int() @@ -2779,6 +3303,7 @@ def nvmlDeviceGetNumaNodeId(handle): _nvmlCheckReturn(ret) return node.value + def nvmlDeviceGetMinorNumber(handle): c_minor_number = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinorNumber") @@ -2786,6 +3311,7 @@ def nvmlDeviceGetMinorNumber(handle): _nvmlCheckReturn(ret) return c_minor_number.value + @convertStrBytes def nvmlDeviceGetUUID(handle): c_uuid = create_string_buffer(NVML_DEVICE_UUID_V2_BUFFER_SIZE) @@ -2794,15 +3320,21 @@ def nvmlDeviceGetUUID(handle): _nvmlCheckReturn(ret) return c_uuid.value + @convertStrBytes def nvmlDeviceGetInforomVersion(handle, infoRomObject): c_version = create_string_buffer(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE) fn = _nvmlGetFunctionPointer("nvmlDeviceGetInforomVersion") - ret = fn(handle, _nvmlInforomObject_t(infoRomObject), - c_version, c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE)) + ret = fn( + handle, + _nvmlInforomObject_t(infoRomObject), + c_version, + c_uint(NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE), + ) _nvmlCheckReturn(ret) return c_version.value + # Added in 4.304 @convertStrBytes def nvmlDeviceGetInforomImageVersion(handle): @@ -2812,6 +3344,7 @@ def nvmlDeviceGetInforomImageVersion(handle): _nvmlCheckReturn(ret) return c_version.value + # Added in 4.304 def nvmlDeviceGetInforomConfigurationChecksum(handle): c_checksum = c_uint() @@ -2820,6 +3353,7 @@ def nvmlDeviceGetInforomConfigurationChecksum(handle): _nvmlCheckReturn(ret) return c_checksum.value + # Added in 4.304 def nvmlDeviceValidateInforom(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceValidateInforom") @@ -2827,6 +3361,7 @@ def nvmlDeviceValidateInforom(handle): _nvmlCheckReturn(ret) return None + def nvmlDeviceGetLastBBXFlushTime(handle): c_timestamp = c_ulonglong() c_durationUs = c_ulong() @@ -2835,6 +3370,7 @@ def nvmlDeviceGetLastBBXFlushTime(handle): _nvmlCheckReturn(ret) return [c_timestamp.value, c_durationUs.value] + def nvmlDeviceGetDisplayMode(handle): c_mode = _nvmlEnableState_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayMode") @@ -2842,6 +3378,7 @@ def nvmlDeviceGetDisplayMode(handle): _nvmlCheckReturn(ret) return c_mode.value + def nvmlDeviceGetDisplayActive(handle): c_mode = _nvmlEnableState_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetDisplayActive") @@ -2857,12 +3394,14 @@ def nvmlDeviceGetPersistenceMode(handle): _nvmlCheckReturn(ret) return c_state.value + def nvmlDeviceGetPciInfoExt(handle, c_info): fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfoExt") ret = fn(handle, c_info) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetPciInfo_v3(handle): c_info = nvmlPciInfo_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPciInfo_v3") @@ -2870,9 +3409,11 @@ def nvmlDeviceGetPciInfo_v3(handle): _nvmlCheckReturn(ret) return c_info + def nvmlDeviceGetPciInfo(handle): return nvmlDeviceGetPciInfo_v3(handle) + def nvmlDeviceGetClockInfo(handle, type): c_clock = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockInfo") @@ -2880,6 +3421,7 @@ def nvmlDeviceGetClockInfo(handle, type): _nvmlCheckReturn(ret) return c_clock.value + # Added in 2.285 def nvmlDeviceGetMaxClockInfo(handle, type): c_clock = c_uint() @@ -2888,6 +3430,7 @@ def nvmlDeviceGetMaxClockInfo(handle, type): _nvmlCheckReturn(ret) return c_clock.value + # Added in 4.304 def nvmlDeviceGetApplicationsClock(handle, type): c_clock = c_uint() @@ -2896,6 +3439,7 @@ def nvmlDeviceGetApplicationsClock(handle, type): _nvmlCheckReturn(ret) return c_clock.value + def nvmlDeviceGetMaxCustomerBoostClock(handle, type): c_clock = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxCustomerBoostClock") @@ -2903,6 +3447,7 @@ def nvmlDeviceGetMaxCustomerBoostClock(handle, type): _nvmlCheckReturn(ret) return c_clock.value + def nvmlDeviceGetClock(handle, type, id): c_clock = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetClock") @@ -2910,6 +3455,7 @@ def nvmlDeviceGetClock(handle, type, id): _nvmlCheckReturn(ret) return c_clock.value + # Added in 5.319 def nvmlDeviceGetDefaultApplicationsClock(handle, type): c_clock = c_uint() @@ -2918,6 +3464,7 @@ def nvmlDeviceGetDefaultApplicationsClock(handle, type): _nvmlCheckReturn(ret) return c_clock.value + # Added in 4.304 def nvmlDeviceGetSupportedMemoryClocks(handle): # first call to get the size @@ -2925,10 +3472,10 @@ def nvmlDeviceGetSupportedMemoryClocks(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedMemoryClocks") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no clocks return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case clocks_array = c_uint * c_count.value c_clocks = clocks_array() @@ -2946,6 +3493,7 @@ def nvmlDeviceGetSupportedMemoryClocks(handle): # error case raise NVMLError(ret) + # Added in 4.304 def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz): # first call to get the size @@ -2953,10 +3501,10 @@ def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz): fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedGraphicsClocks") ret = fn(handle, c_uint(memoryClockMHz), byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no clocks return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case clocks_array = c_uint * c_count.value c_clocks = clocks_array() @@ -2974,6 +3522,7 @@ def nvmlDeviceGetSupportedGraphicsClocks(handle, memoryClockMHz): # error case raise NVMLError(ret) + def nvmlDeviceGetFanSpeed(handle): c_speed = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed") @@ -2981,6 +3530,7 @@ def nvmlDeviceGetFanSpeed(handle): _nvmlCheckReturn(ret) return c_speed.value + def nvmlDeviceGetFanSpeed_v2(handle, fan): c_speed = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetFanSpeed_v2") @@ -2988,15 +3538,18 @@ def nvmlDeviceGetFanSpeed_v2(handle, fan): _nvmlCheckReturn(ret) return c_speed.value + class c_nvmlFanSpeedInfo_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('fan', c_uint), - ('speed', c_uint), + ("version", c_uint), + ("fan", c_uint), + ("speed", c_uint), ] + nvmlFanSpeedInfo_v1 = 0x100000C + def nvmlDeviceGetFanSpeedRPM(handle): c_fanSpeed = c_nvmlFanSpeedInfo_t() c_fanSpeed.fan = 0 @@ -3006,6 +3559,7 @@ def nvmlDeviceGetFanSpeedRPM(handle): _nvmlCheckReturn(ret) return c_fanSpeed.speed + def nvmlDeviceGetTargetFanSpeed(handle, fan): c_speed = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetTargetFanSpeed") @@ -3013,6 +3567,7 @@ def nvmlDeviceGetTargetFanSpeed(handle, fan): _nvmlCheckReturn(ret) return c_speed.value + def nvmlDeviceGetNumFans(device): c_numFans = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumFans") @@ -3020,12 +3575,14 @@ def nvmlDeviceGetNumFans(device): _nvmlCheckReturn(ret) return c_numFans.value + def nvmlDeviceSetDefaultFanSpeed_v2(handle, index): - fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultFanSpeed_v2"); + fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultFanSpeed_v2") ret = fn(handle, index) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetMinMaxFanSpeed(handle, minSpeed=c_uint(), maxSpeed=c_uint()): isReference = (type(minSpeed) is not c_uint) or (type(maxSpeed) is not c_uint) minSpeedRef = minSpeed if isReference else byref(minSpeed) @@ -3035,6 +3592,7 @@ def nvmlDeviceGetMinMaxFanSpeed(handle, minSpeed=c_uint(), maxSpeed=c_uint()): _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else [minSpeed.value, maxSpeed.value] + def nvmlDeviceGetFanControlPolicy_v2(handle, fan, fanControlPolicy=c_uint()): isReference = type(fanControlPolicy) is not c_uint fanControlPolicyRef = fanControlPolicy if isReference else byref(fanControlPolicy) @@ -3043,35 +3601,42 @@ def nvmlDeviceGetFanControlPolicy_v2(handle, fan, fanControlPolicy=c_uint()): _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else fanControlPolicy.value + def nvmlDeviceSetFanControlPolicy(handle, fan, fanControlPolicy): fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanControlPolicy") ret = fn(handle, fan, _nvmlFanControlPolicy_t(fanControlPolicy)) _nvmlCheckReturn(ret) return NVML_SUCCESS + class c_nvmlTemperature_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('sensorType', _nvmlTemperatureSensors_t), - ('temperature', c_int), + ("version", c_uint), + ("sensorType", _nvmlTemperatureSensors_t), + ("temperature", c_int), ] + + nvmlTemperature_v1 = 0x100000C + def nvmlDeviceGetTemperatureV1(handle, sensor): c_temp = c_nvmlTemperature_v1_t() c_temp.version = nvmlTemperature_v1 - c_temp.sensorType = _nvmlTemperatureSensors_t(sensor) + c_temp.sensorType = _nvmlTemperatureSensors_t(sensor) fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureV") ret = fn(handle, byref(c_temp)) _nvmlCheckReturn(ret) return c_temp.temperature + def nvmlDeviceGetTemperatureV(handle, sensor, version=nvmlTemperature_v1): if version == nvmlTemperature_v1: return nvmlDeviceGetTemperatureV1(handle, sensor) else: raise NVMLError(NVML_ERROR_ARGUMENT_VERSION_MISMATCH) + # DEPRECATED use nvmlDeviceGetTemperatureV instead def nvmlDeviceGetTemperature(handle, sensor): c_temp = c_uint() @@ -3080,6 +3645,7 @@ def nvmlDeviceGetTemperature(handle, sensor): _nvmlCheckReturn(ret) return c_temp.value + def nvmlDeviceGetTemperatureThreshold(handle, threshold): c_temp = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetTemperatureThreshold") @@ -3087,6 +3653,7 @@ def nvmlDeviceGetTemperatureThreshold(handle, threshold): _nvmlCheckReturn(ret) return c_temp.value + def nvmlDeviceSetTemperatureThreshold(handle, threshold, temp): c_temp = c_uint() c_temp.value = temp @@ -3095,6 +3662,7 @@ def nvmlDeviceSetTemperatureThreshold(handle, threshold, temp): _nvmlCheckReturn(ret) return None + def nvmlDeviceGetMarginTemperature(handle): c_marginTempInfo = c_nvmlMarginTemperature_v1_t() c_marginTempInfo.version = nvmlMarginTemperature_v1 @@ -3103,6 +3671,7 @@ def nvmlDeviceGetMarginTemperature(handle): _nvmlCheckReturn(ret) return c_marginTempInfo.marginTemperature + # DEPRECATED use nvmlDeviceGetPerformanceState def nvmlDeviceGetPowerState(handle): c_pstate = _nvmlPstates_t() @@ -3111,6 +3680,7 @@ def nvmlDeviceGetPowerState(handle): _nvmlCheckReturn(ret) return c_pstate.value + def nvmlDeviceGetPerformanceState(handle): c_pstate = _nvmlPstates_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPerformanceState") @@ -3118,6 +3688,7 @@ def nvmlDeviceGetPerformanceState(handle): _nvmlCheckReturn(ret) return c_pstate.value + def nvmlDeviceGetPowerManagementMode(handle): c_pcapMode = _nvmlEnableState_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementMode") @@ -3125,6 +3696,7 @@ def nvmlDeviceGetPowerManagementMode(handle): _nvmlCheckReturn(ret) return c_pcapMode.value + def nvmlDeviceGetPowerManagementLimit(handle): c_limit = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerManagementLimit") @@ -3132,6 +3704,7 @@ def nvmlDeviceGetPowerManagementLimit(handle): _nvmlCheckReturn(ret) return c_limit.value + # Added in 4.304 def nvmlDeviceGetPowerManagementLimitConstraints(handle): c_minLimit = c_uint() @@ -3141,6 +3714,7 @@ def nvmlDeviceGetPowerManagementLimitConstraints(handle): _nvmlCheckReturn(ret) return [c_minLimit.value, c_maxLimit.value] + # Added in 4.304 def nvmlDeviceGetPowerManagementDefaultLimit(handle): c_limit = c_uint() @@ -3158,6 +3732,7 @@ def nvmlDeviceGetEnforcedPowerLimit(handle): _nvmlCheckReturn(ret) return c_limit.value + def nvmlDeviceGetPowerUsage(handle): c_watts = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerUsage") @@ -3165,6 +3740,7 @@ def nvmlDeviceGetPowerUsage(handle): _nvmlCheckReturn(ret) return c_watts.value + def nvmlDeviceGetTotalEnergyConsumption(handle): c_millijoules = c_uint64() fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEnergyConsumption") @@ -3172,6 +3748,7 @@ def nvmlDeviceGetTotalEnergyConsumption(handle): _nvmlCheckReturn(ret) return c_millijoules.value + # Added in 4.304 def nvmlDeviceGetGpuOperationMode(handle): c_currState = _nvmlGpuOperationMode_t() @@ -3181,14 +3758,17 @@ def nvmlDeviceGetGpuOperationMode(handle): _nvmlCheckReturn(ret) return [c_currState.value, c_pendingState.value] + # Added in 4.304 def nvmlDeviceGetCurrentGpuOperationMode(handle): return nvmlDeviceGetGpuOperationMode(handle)[0] + # Added in 4.304 def nvmlDeviceGetPendingGpuOperationMode(handle): return nvmlDeviceGetGpuOperationMode(handle)[1] + def nvmlDeviceGetMemoryInfo(handle, version=None): if not version: c_memory = c_nvmlMemory_t() @@ -3201,6 +3781,7 @@ def nvmlDeviceGetMemoryInfo(handle, version=None): _nvmlCheckReturn(ret) return c_memory + def nvmlDeviceGetBAR1MemoryInfo(handle): c_bar1_memory = c_nvmlBAR1Memory_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetBAR1MemoryInfo") @@ -3208,6 +3789,7 @@ def nvmlDeviceGetBAR1MemoryInfo(handle): _nvmlCheckReturn(ret) return c_bar1_memory + def nvmlDeviceGetComputeMode(handle): c_mode = _nvmlComputeMode_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeMode") @@ -3215,6 +3797,7 @@ def nvmlDeviceGetComputeMode(handle): _nvmlCheckReturn(ret) return c_mode.value + def nvmlDeviceGetCudaComputeCapability(handle): c_major = c_int() c_minor = c_int() @@ -3223,6 +3806,7 @@ def nvmlDeviceGetCudaComputeCapability(handle): _nvmlCheckReturn(ret) return (c_major.value, c_minor.value) + def nvmlDeviceGetEccMode(handle): c_currState = _nvmlEnableState_t() c_pendingState = _nvmlEnableState_t() @@ -3231,14 +3815,17 @@ def nvmlDeviceGetEccMode(handle): _nvmlCheckReturn(ret) return [c_currState.value, c_pendingState.value] + # added to API def nvmlDeviceGetCurrentEccMode(handle): return nvmlDeviceGetEccMode(handle)[0] + # added to API def nvmlDeviceGetPendingEccMode(handle): return nvmlDeviceGetEccMode(handle)[1] + def nvmlDeviceGetDefaultEccMode(handle): c_defaultState = _nvmlEnableState_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetDefaultEccMode") @@ -3246,35 +3833,49 @@ def nvmlDeviceGetDefaultEccMode(handle): _nvmlCheckReturn(ret) return [c_defaultState.value] + def nvmlDeviceGetTotalEccErrors(handle, errorType, counterType): c_count = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetTotalEccErrors") - ret = fn(handle, _nvmlMemoryErrorType_t(errorType), - _nvmlEccCounterType_t(counterType), byref(c_count)) + ret = fn( + handle, + _nvmlMemoryErrorType_t(errorType), + _nvmlEccCounterType_t(counterType), + byref(c_count), + ) _nvmlCheckReturn(ret) return c_count.value + # This is deprecated, instead use nvmlDeviceGetMemoryErrorCounter def nvmlDeviceGetDetailedEccErrors(handle, errorType, counterType): c_counts = c_nvmlEccErrorCounts_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetDetailedEccErrors") - ret = fn(handle, _nvmlMemoryErrorType_t(errorType), - _nvmlEccCounterType_t(counterType), byref(c_counts)) + ret = fn( + handle, + _nvmlMemoryErrorType_t(errorType), + _nvmlEccCounterType_t(counterType), + byref(c_counts), + ) _nvmlCheckReturn(ret) return c_counts + # Added in 4.304 def nvmlDeviceGetMemoryErrorCounter(handle, errorType, counterType, locationType): c_count = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryErrorCounter") - ret = fn(handle, - _nvmlMemoryErrorType_t(errorType), - _nvmlEccCounterType_t(counterType), - _nvmlMemoryLocation_t(locationType), - byref(c_count)) + ret = fn( + handle, + _nvmlMemoryErrorType_t(errorType), + _nvmlEccCounterType_t(counterType), + _nvmlMemoryLocation_t(locationType), + byref(c_count), + ) _nvmlCheckReturn(ret) return c_count.value + def nvmlDeviceGetUtilizationRates(handle): c_util = c_nvmlUtilization_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetUtilizationRates") @@ -3282,6 +3883,7 @@ def nvmlDeviceGetUtilizationRates(handle): _nvmlCheckReturn(ret) return c_util + def nvmlDeviceGetEncoderUtilization(handle): c_util = c_uint() c_samplingPeriod = c_uint() @@ -3290,6 +3892,7 @@ def nvmlDeviceGetEncoderUtilization(handle): _nvmlCheckReturn(ret) return [c_util.value, c_samplingPeriod.value] + def nvmlDeviceGetDecoderUtilization(handle): c_util = c_uint() c_samplingPeriod = c_uint() @@ -3298,6 +3901,7 @@ def nvmlDeviceGetDecoderUtilization(handle): _nvmlCheckReturn(ret) return [c_util.value, c_samplingPeriod.value] + def nvmlDeviceGetJpgUtilization(handle): c_util = c_uint() c_samplingPeriod = c_uint() @@ -3306,6 +3910,7 @@ def nvmlDeviceGetJpgUtilization(handle): _nvmlCheckReturn(ret) return [c_util.value, c_samplingPeriod.value] + def nvmlDeviceGetOfaUtilization(handle): c_util = c_uint() c_samplingPeriod = c_uint() @@ -3314,6 +3919,7 @@ def nvmlDeviceGetOfaUtilization(handle): _nvmlCheckReturn(ret) return [c_util.value, c_samplingPeriod.value] + def nvmlDeviceGetPcieReplayCounter(handle): c_replay = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieReplayCounter") @@ -3321,6 +3927,7 @@ def nvmlDeviceGetPcieReplayCounter(handle): _nvmlCheckReturn(ret) return c_replay.value + def nvmlDeviceGetDriverModel(handle): c_currModel = _nvmlDriverModel_t() c_pendingModel = _nvmlDriverModel_t() @@ -3329,14 +3936,17 @@ def nvmlDeviceGetDriverModel(handle): _nvmlCheckReturn(ret) return [c_currModel.value, c_pendingModel.value] + # added to API def nvmlDeviceGetCurrentDriverModel(handle): return nvmlDeviceGetDriverModel(handle)[0] + # added to API def nvmlDeviceGetPendingDriverModel(handle): return nvmlDeviceGetDriverModel(handle)[1] + # Added in 2.285 @convertStrBytes def nvmlDeviceGetVbiosVersion(handle): @@ -3346,16 +3956,17 @@ def nvmlDeviceGetVbiosVersion(handle): _nvmlCheckReturn(ret) return c_version.value + # Added in 2.285 def nvmlDeviceGetComputeRunningProcesses_v2(handle): # first call to get the size c_count = c_uint(0) fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses_v2") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case # oversize the array incase more processes are created c_count.value = c_count.value * 2 + 5 @@ -3368,7 +3979,7 @@ def nvmlDeviceGetComputeRunningProcesses_v2(handle): for i in range(c_count.value): # use an alternative struct for this object obj = nvmlStructToFriendlyObject(c_procs[i]) - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above obj.usedGpuMemory = None procs.append(obj) @@ -3377,6 +3988,7 @@ def nvmlDeviceGetComputeRunningProcesses_v2(handle): # error case raise NVMLError(ret) + # Added in 2.285 def nvmlDeviceGetComputeRunningProcesses_v3(handle): # first call to get the size @@ -3384,10 +3996,10 @@ def nvmlDeviceGetComputeRunningProcesses_v3(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeRunningProcesses_v3") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case # oversize the array incase more processes are created c_count.value = c_count.value * 2 + 5 @@ -3402,7 +4014,7 @@ def nvmlDeviceGetComputeRunningProcesses_v3(handle): for i in range(c_count.value): # use an alternative struct for this object obj = nvmlStructToFriendlyObject(c_procs[i]) - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above obj.usedGpuMemory = None procs.append(obj) @@ -3412,19 +4024,21 @@ def nvmlDeviceGetComputeRunningProcesses_v3(handle): # error case raise NVMLError(ret) + @throwOnVersionMismatch def nvmlDeviceGetComputeRunningProcesses(handle): return nvmlDeviceGetComputeRunningProcesses_v3(handle) + def nvmlDeviceGetGraphicsRunningProcesses_v2(handle): # first call to get the size c_count = c_uint(0) fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses_v2") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case # oversize the array incase more processes are created c_count.value = c_count.value * 2 + 5 @@ -3437,7 +4051,7 @@ def nvmlDeviceGetGraphicsRunningProcesses_v2(handle): for i in range(c_count.value): # use an alternative struct for this object obj = nvmlStructToFriendlyObject(c_procs[i]) - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above obj.usedGpuMemory = None procs.append(obj) @@ -3446,16 +4060,17 @@ def nvmlDeviceGetGraphicsRunningProcesses_v2(handle): # error case raise NVMLError(ret) + def nvmlDeviceGetGraphicsRunningProcesses_v3(handle): # first call to get the size c_count = c_uint(0) fn = _nvmlGetFunctionPointer("nvmlDeviceGetGraphicsRunningProcesses_v3") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case # oversize the array incase more processes are created c_count.value = c_count.value * 2 + 5 @@ -3470,7 +4085,7 @@ def nvmlDeviceGetGraphicsRunningProcesses_v3(handle): for i in range(c_count.value): # use an alternative struct for this object obj = nvmlStructToFriendlyObject(c_procs[i]) - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above obj.usedGpuMemory = None procs.append(obj) @@ -3480,24 +4095,27 @@ def nvmlDeviceGetGraphicsRunningProcesses_v3(handle): # error case raise NVMLError(ret) + @throwOnVersionMismatch def nvmlDeviceGetGraphicsRunningProcesses(handle): return nvmlDeviceGetGraphicsRunningProcesses_v3(handle) + @throwOnVersionMismatch def nvmlDeviceGetMPSComputeRunningProcesses(handle): return nvmlDeviceGetMPSComputeRunningProcesses_v3(handle) + def nvmlDeviceGetMPSComputeRunningProcesses_v2(handle): # first call to get the size c_count = c_uint(0) fn = _nvmlGetFunctionPointer("nvmlDeviceGetMPSComputeRunningProcesses_v2") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case # oversize the array incase more processes are created c_count.value = c_count.value * 2 + 5 @@ -3512,7 +4130,7 @@ def nvmlDeviceGetMPSComputeRunningProcesses_v2(handle): for i in range(c_count.value): # use an alternative struct for this object obj = nvmlStructToFriendlyObject(c_procs[i]) - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above obj.usedGpuMemory = None procs.append(obj) @@ -3522,16 +4140,17 @@ def nvmlDeviceGetMPSComputeRunningProcesses_v2(handle): # error case raise NVMLError(ret) + def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle): # first call to get the size c_count = c_uint(0) fn = _nvmlGetFunctionPointer("nvmlDeviceGetMPSComputeRunningProcesses_v3") ret = fn(handle, byref(c_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case # oversize the array in case more processes are created c_count.value = c_count.value * 2 + 5 @@ -3546,7 +4165,7 @@ def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle): for i in range(c_count.value): # use an alternative struct for this object obj = nvmlStructToFriendlyObject(c_procs[i]) - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above obj.usedGpuMemory = None procs.append(obj) @@ -3556,6 +4175,7 @@ def nvmlDeviceGetMPSComputeRunningProcesses_v3(handle): # error case raise NVMLError(ret) + def nvmlDeviceGetRunningProcessDetailList(handle, version, mode): c_processDetailList = c_nvmlProcessDetailList_t() c_processDetailList.version = version @@ -3565,12 +4185,14 @@ def nvmlDeviceGetRunningProcessDetailList(handle, version, mode): # first call to get the size ret = fn(handle, byref(c_processDetailList)) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no running processes return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: c_procs = c_nvmlProcessDetail_v1_t * c_processDetailList.numProcArrayEntries - c_processDetailList.procArray = cast((c_procs)(), POINTER(c_nvmlProcessDetail_v1_t)) + c_processDetailList.procArray = cast( + (c_procs)(), POINTER(c_nvmlProcessDetail_v1_t) + ) # make the call again ret = fn(handle, byref(c_processDetailList)) @@ -3580,9 +4202,9 @@ def nvmlDeviceGetRunningProcessDetailList(handle, version, mode): for i in range(c_processDetailList.numProcArrayEntries): # use an alternative struct for this object obj = c_processDetailList.procArray[i] - if (obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: obj.usedGpuMemory = None - if (obj.usedGpuCcProtectedMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if obj.usedGpuCcProtectedMemory == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: obj.usedGpuCcProtectedMemory = None procs.append(obj) @@ -3591,6 +4213,7 @@ def nvmlDeviceGetRunningProcessDetailList(handle, version, mode): # error case raise NVMLError(ret) + def nvmlDeviceGetAutoBoostedClocksEnabled(handle): c_isEnabled = _nvmlEnableState_t() c_defaultIsEnabled = _nvmlEnableState_t() @@ -3598,7 +4221,8 @@ def nvmlDeviceGetAutoBoostedClocksEnabled(handle): ret = fn(handle, byref(c_isEnabled), byref(c_defaultIsEnabled)) _nvmlCheckReturn(ret) return [c_isEnabled.value, c_defaultIsEnabled.value] - #Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + ## Set functions def nvmlUnitSetLedState(unit, color): @@ -3607,49 +4231,57 @@ def nvmlUnitSetLedState(unit, color): _nvmlCheckReturn(ret) return None + def nvmlDeviceSetPersistenceMode(handle, mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetPersistenceMode") ret = fn(handle, _nvmlEnableState_t(mode)) _nvmlCheckReturn(ret) return None + def nvmlDeviceSetComputeMode(handle, mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetComputeMode") ret = fn(handle, _nvmlComputeMode_t(mode)) _nvmlCheckReturn(ret) return None + def nvmlDeviceSetEccMode(handle, mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetEccMode") ret = fn(handle, _nvmlEnableState_t(mode)) _nvmlCheckReturn(ret) return None + def nvmlDeviceClearEccErrorCounts(handle, counterType): fn = _nvmlGetFunctionPointer("nvmlDeviceClearEccErrorCounts") ret = fn(handle, _nvmlEccCounterType_t(counterType)) _nvmlCheckReturn(ret) return None + def nvmlDeviceSetDriverModel(handle, model): fn = _nvmlGetFunctionPointer("nvmlDeviceSetDriverModel") ret = fn(handle, _nvmlDriverModel_t(model)) _nvmlCheckReturn(ret) return None + def nvmlDeviceSetAutoBoostedClocksEnabled(handle, enabled): fn = _nvmlGetFunctionPointer("nvmlDeviceSetAutoBoostedClocksEnabled") ret = fn(handle, _nvmlEnableState_t(enabled)) _nvmlCheckReturn(ret) return None - #Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + def nvmlDeviceSetDefaultAutoBoostedClocksEnabled(handle, enabled, flags): fn = _nvmlGetFunctionPointer("nvmlDeviceSetDefaultAutoBoostedClocksEnabled") ret = fn(handle, _nvmlEnableState_t(enabled), c_uint(flags)) _nvmlCheckReturn(ret) return None - #Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + # Throws NVML_ERROR_NOT_SUPPORTED if hardware doesn't support setting auto boosted clocks + def nvmlDeviceSetGpuLockedClocks(handle, minGpuClockMHz, maxGpuClockMHz): fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuLockedClocks") @@ -3657,24 +4289,28 @@ def nvmlDeviceSetGpuLockedClocks(handle, minGpuClockMHz, maxGpuClockMHz): _nvmlCheckReturn(ret) return None + def nvmlDeviceResetGpuLockedClocks(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceResetGpuLockedClocks") ret = fn(handle) _nvmlCheckReturn(ret) return None + def nvmlDeviceSetMemoryLockedClocks(handle, minMemClockMHz, maxMemClockMHz): fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemoryLockedClocks") ret = fn(handle, c_uint(minMemClockMHz), c_uint(maxMemClockMHz)) _nvmlCheckReturn(ret) return None + def nvmlDeviceResetMemoryLockedClocks(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceResetMemoryLockedClocks") ret = fn(handle) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetClkMonStatus(handle, c_clkMonInfo=nvmlClkMonStatus_t()): isReference = type(c_clkMonInfo) is not nvmlClkMonStatus_t c_clkMonInfoRef = c_clkMonInfo if isReference else byref(c_clkMonInfo) @@ -3683,6 +4319,7 @@ def nvmlDeviceGetClkMonStatus(handle, c_clkMonInfo=nvmlClkMonStatus_t()): _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else c_clkMonInfo + # Added in 4.304 def nvmlDeviceSetApplicationsClocks(handle, maxMemClockMHz, maxGraphicsClockMHz): fn = _nvmlGetFunctionPointer("nvmlDeviceSetApplicationsClocks") @@ -3690,6 +4327,7 @@ def nvmlDeviceSetApplicationsClocks(handle, maxMemClockMHz, maxGraphicsClockMHz) _nvmlCheckReturn(ret) return None + # Added in 4.304 def nvmlDeviceResetApplicationsClocks(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceResetApplicationsClocks") @@ -3697,6 +4335,7 @@ def nvmlDeviceResetApplicationsClocks(handle): _nvmlCheckReturn(ret) return None + # Added in 4.304 def nvmlDeviceSetPowerManagementLimit(handle, limit): fn = _nvmlGetFunctionPointer("nvmlDeviceSetPowerManagementLimit") @@ -3704,6 +4343,7 @@ def nvmlDeviceSetPowerManagementLimit(handle, limit): _nvmlCheckReturn(ret) return None + # Added in 4.304 def nvmlDeviceSetGpuOperationMode(handle, mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpuOperationMode") @@ -3711,6 +4351,7 @@ def nvmlDeviceSetGpuOperationMode(handle, mode): _nvmlCheckReturn(ret) return None + # Added in 2.285 def nvmlEventSetCreate(): fn = _nvmlGetFunctionPointer("nvmlEventSetCreate") @@ -3719,6 +4360,7 @@ def nvmlEventSetCreate(): _nvmlCheckReturn(ret) return eventSet + # Added in 2.285 def nvmlDeviceRegisterEvents(handle, eventTypes, eventSet): fn = _nvmlGetFunctionPointer("nvmlDeviceRegisterEvents") @@ -3726,6 +4368,7 @@ def nvmlDeviceRegisterEvents(handle, eventTypes, eventSet): _nvmlCheckReturn(ret) return None + # Added in 2.285 def nvmlDeviceGetSupportedEventTypes(handle): c_eventTypes = c_ulonglong() @@ -3734,6 +4377,7 @@ def nvmlDeviceGetSupportedEventTypes(handle): _nvmlCheckReturn(ret) return c_eventTypes.value + # raises NVML_ERROR_TIMEOUT exception on timeout def nvmlEventSetWait_v2(eventSet, timeoutms): fn = _nvmlGetFunctionPointer("nvmlEventSetWait_v2") @@ -3742,9 +4386,11 @@ def nvmlEventSetWait_v2(eventSet, timeoutms): _nvmlCheckReturn(ret) return data + def nvmlEventSetWait(eventSet, timeoutms): return nvmlEventSetWait_v2(eventSet, timeoutms) + # Added in 2.285 def nvmlEventSetFree(eventSet): fn = _nvmlGetFunctionPointer("nvmlEventSetFree") @@ -3752,13 +4398,15 @@ def nvmlEventSetFree(eventSet): _nvmlCheckReturn(ret) return None + # Added in 3.295 def nvmlDeviceOnSameBoard(handle1, handle2): fn = _nvmlGetFunctionPointer("nvmlDeviceOnSameBoard") onSameBoard = c_int() ret = fn(handle1, handle2, byref(onSameBoard)) _nvmlCheckReturn(ret) - return (onSameBoard.value != 0) + return onSameBoard.value != 0 + # Added in 3.295 def nvmlDeviceGetCurrPcieLinkGeneration(handle): @@ -3768,6 +4416,7 @@ def nvmlDeviceGetCurrPcieLinkGeneration(handle): _nvmlCheckReturn(ret) return gen.value + # Added in 3.295 def nvmlDeviceGetMaxPcieLinkGeneration(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkGeneration") @@ -3776,6 +4425,7 @@ def nvmlDeviceGetMaxPcieLinkGeneration(handle): _nvmlCheckReturn(ret) return gen.value + # Added in 3.295 def nvmlDeviceGetCurrPcieLinkWidth(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrPcieLinkWidth") @@ -3784,6 +4434,7 @@ def nvmlDeviceGetCurrPcieLinkWidth(handle): _nvmlCheckReturn(ret) return width.value + # Added in 3.295 def nvmlDeviceGetMaxPcieLinkWidth(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxPcieLinkWidth") @@ -3792,6 +4443,7 @@ def nvmlDeviceGetMaxPcieLinkWidth(handle): _nvmlCheckReturn(ret) return width.value + def nvmlDeviceGetGpuMaxPcieLinkGeneration(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuMaxPcieLinkGeneration") gen = c_uint() @@ -3799,36 +4451,41 @@ def nvmlDeviceGetGpuMaxPcieLinkGeneration(handle): _nvmlCheckReturn(ret) return gen.value + # Added in 4.304 def nvmlDeviceGetSupportedClocksThrottleReasons(handle): - c_reasons= c_ulonglong() + c_reasons = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksThrottleReasons") ret = fn(handle, byref(c_reasons)) _nvmlCheckReturn(ret) return c_reasons.value + def nvmlDeviceGetSupportedClocksEventReasons(handle): - c_reasons= c_ulonglong() + c_reasons = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedClocksEventReasons") ret = fn(handle, byref(c_reasons)) _nvmlCheckReturn(ret) return c_reasons.value + # Added in 4.304 def nvmlDeviceGetCurrentClocksThrottleReasons(handle): - c_reasons= c_ulonglong() + c_reasons = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksThrottleReasons") ret = fn(handle, byref(c_reasons)) _nvmlCheckReturn(ret) return c_reasons.value + def nvmlDeviceGetCurrentClocksEventReasons(handle): - c_reasons= c_ulonglong() + c_reasons = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetCurrentClocksEventReasons") ret = fn(handle, byref(c_reasons)) _nvmlCheckReturn(ret) return c_reasons.value + # Added in 5.319 def nvmlDeviceGetIndex(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetIndex") @@ -3837,6 +4494,7 @@ def nvmlDeviceGetIndex(handle): _nvmlCheckReturn(ret) return c_index.value + # Added in 5.319 def nvmlDeviceGetAccountingMode(handle): c_mode = _nvmlEnableState_t() @@ -3845,35 +4503,40 @@ def nvmlDeviceGetAccountingMode(handle): _nvmlCheckReturn(ret) return c_mode.value + def nvmlDeviceSetAccountingMode(handle, mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetAccountingMode") ret = fn(handle, _nvmlEnableState_t(mode)) _nvmlCheckReturn(ret) return None + def nvmlDeviceClearAccountingPids(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceClearAccountingPids") ret = fn(handle) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetAccountingStats(handle, pid): stats = c_nvmlAccountingStats_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingStats") ret = fn(handle, c_uint(pid), byref(stats)) _nvmlCheckReturn(ret) - if (stats.maxMemoryUsage == NVML_VALUE_NOT_AVAILABLE_ulonglong.value): + if stats.maxMemoryUsage == NVML_VALUE_NOT_AVAILABLE_ulonglong.value: # special case for WDDM on Windows, see comment above stats.maxMemoryUsage = None return stats + def nvmlDeviceGetAccountingPids(handle): count = c_uint(nvmlDeviceGetAccountingBufferSize(handle)) pids = (c_uint * count.value)() fn = _nvmlGetFunctionPointer("nvmlDeviceGetAccountingPids") ret = fn(handle, byref(count), pids) _nvmlCheckReturn(ret) - return list(map(int, pids[0:count.value])) + return list(map(int, pids[0 : count.value])) + def nvmlDeviceGetAccountingBufferSize(handle): bufferSize = c_uint() @@ -3882,6 +4545,7 @@ def nvmlDeviceGetAccountingBufferSize(handle): _nvmlCheckReturn(ret) return int(bufferSize.value) + def nvmlDeviceGetRetiredPages(device, sourceFilter): c_source = _nvmlPageRetirementCause_t(sourceFilter) c_count = c_uint(0) @@ -3891,8 +4555,7 @@ def nvmlDeviceGetRetiredPages(device, sourceFilter): ret = fn(device, c_source, byref(c_count), None) # this should only fail with insufficient size - if ((ret != NVML_SUCCESS) and - (ret != NVML_ERROR_INSUFFICIENT_SIZE)): + if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE): raise NVMLError(ret) # call again with a buffer @@ -3903,7 +4566,8 @@ def nvmlDeviceGetRetiredPages(device, sourceFilter): c_pages = page_array() ret = fn(device, c_source, byref(c_count), c_pages) _nvmlCheckReturn(ret) - return list(map(int, c_pages[0:c_count.value])) + return list(map(int, c_pages[0 : c_count.value])) + def nvmlDeviceGetRetiredPages_v2(device, sourceFilter): c_source = _nvmlPageRetirementCause_t(sourceFilter) @@ -3914,8 +4578,7 @@ def nvmlDeviceGetRetiredPages_v2(device, sourceFilter): ret = fn(device, c_source, byref(c_count), None) # this should only fail with insufficient size - if ((ret != NVML_SUCCESS) and - (ret != NVML_ERROR_INSUFFICIENT_SIZE)): + if (ret != NVML_SUCCESS) and (ret != NVML_ERROR_INSUFFICIENT_SIZE): raise NVMLError(ret) # call again with a buffer @@ -3928,7 +4591,11 @@ def nvmlDeviceGetRetiredPages_v2(device, sourceFilter): c_times = times_array() ret = fn(device, c_source, byref(c_count), c_pages, c_times) _nvmlCheckReturn(ret) - return [ { 'address': int(c_pages[i]), 'timestamp': int(c_times[i]) } for i in range(c_count.value) ]; + return [ + {"address": int(c_pages[i]), "timestamp": int(c_times[i])} + for i in range(c_count.value) + ] + def nvmlDeviceGetRetiredPagesPendingStatus(device): c_pending = _nvmlEnableState_t() @@ -3937,6 +4604,7 @@ def nvmlDeviceGetRetiredPagesPendingStatus(device): _nvmlCheckReturn(ret) return int(c_pending.value) + def nvmlDeviceGetAPIRestriction(device, apiType): c_permission = _nvmlEnableState_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetAPIRestriction") @@ -3944,12 +4612,14 @@ def nvmlDeviceGetAPIRestriction(device, apiType): _nvmlCheckReturn(ret) return int(c_permission.value) + def nvmlDeviceSetAPIRestriction(handle, apiType, isRestricted): fn = _nvmlGetFunctionPointer("nvmlDeviceSetAPIRestriction") ret = fn(handle, _nvmlRestrictedAPI_t(apiType), _nvmlEnableState_t(isRestricted)) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetBridgeChipInfo(handle): bridgeHierarchy = c_nvmlBridgeChipHierarchy_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetBridgeChipInfo") @@ -3957,6 +4627,7 @@ def nvmlDeviceGetBridgeChipInfo(handle): _nvmlCheckReturn(ret) return bridgeHierarchy + def nvmlDeviceGetSamples(device, sampling_type, timeStamp): c_sampling_type = _nvmlSamplingType_t(sampling_type) c_time_stamp = c_ulonglong(timeStamp) @@ -3965,17 +4636,32 @@ def nvmlDeviceGetSamples(device, sampling_type, timeStamp): fn = _nvmlGetFunctionPointer("nvmlDeviceGetSamples") ## First Call gets the size - ret = fn(device, c_sampling_type, c_time_stamp, byref(c_sample_value_type), byref(c_sample_count), None) + ret = fn( + device, + c_sampling_type, + c_time_stamp, + byref(c_sample_value_type), + byref(c_sample_count), + None, + ) # Stop if this fails - if (ret != NVML_SUCCESS): + if ret != NVML_SUCCESS: raise NVMLError(ret) sampleArray = c_sample_count.value * c_nvmlSample_t c_samples = sampleArray() - ret = fn(device, c_sampling_type, c_time_stamp, byref(c_sample_value_type), byref(c_sample_count), c_samples) + ret = fn( + device, + c_sampling_type, + c_time_stamp, + byref(c_sample_value_type), + byref(c_sample_count), + c_samples, + ) _nvmlCheckReturn(ret) - return (c_sample_value_type.value, c_samples[0:c_sample_count.value]) + return (c_sample_value_type.value, c_samples[0 : c_sample_count.value]) + def nvmlDeviceGetViolationStatus(device, perfPolicyType): c_perfPolicy_type = _nvmlPerfPolicyType_t(perfPolicyType) @@ -3987,6 +4673,7 @@ def nvmlDeviceGetViolationStatus(device, perfPolicyType): _nvmlCheckReturn(ret) return c_violTime + def nvmlDeviceGetPcieThroughput(device, counter): c_util = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieThroughput") @@ -3994,6 +4681,7 @@ def nvmlDeviceGetPcieThroughput(device, counter): _nvmlCheckReturn(ret) return c_util.value + def nvmlSystemGetTopologyGpuSet(cpuNumber): c_count = c_uint(0) fn = _nvmlGetFunctionPointer("nvmlSystemGetTopologyGpuSet") @@ -4008,7 +4696,8 @@ def nvmlSystemGetTopologyGpuSet(cpuNumber): c_devices = device_array() ret = fn(cpuNumber, byref(c_count), c_devices) _nvmlCheckReturn(ret) - return list(c_devices[0:c_count.value]) + return list(c_devices[0 : c_count.value]) + def nvmlDeviceGetTopologyNearestGpus(device, level): c_count = c_uint(0) @@ -4025,7 +4714,8 @@ def nvmlDeviceGetTopologyNearestGpus(device, level): c_devices = device_array() ret = fn(device, level, byref(c_count), c_devices) _nvmlCheckReturn(ret) - return list(c_devices[0:c_count.value]) + return list(c_devices[0 : c_count.value]) + def nvmlDeviceGetTopologyCommonAncestor(device1, device2): c_level = _nvmlGpuTopologyLevel_t() @@ -4034,6 +4724,7 @@ def nvmlDeviceGetTopologyCommonAncestor(device1, device2): _nvmlCheckReturn(ret) return c_level.value + def nvmlDeviceGetNvLinkUtilizationCounter(device, link, counter): c_rxcounter = c_ulonglong() c_txcounter = c_ulonglong() @@ -4042,24 +4733,28 @@ def nvmlDeviceGetNvLinkUtilizationCounter(device, link, counter): _nvmlCheckReturn(ret) return (c_rxcounter.value, c_txcounter.value) + def nvmlDeviceFreezeNvLinkUtilizationCounter(device, link, counter, freeze): fn = _nvmlGetFunctionPointer("nvmlDeviceFreezeNvLinkUtilizationCounter") ret = fn(device, link, counter, freeze) _nvmlCheckReturn(ret) return None + def nvmlDeviceResetNvLinkUtilizationCounter(device, link, counter): fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkUtilizationCounter") ret = fn(device, link, counter) _nvmlCheckReturn(ret) return None + def nvmlDeviceSetNvLinkUtilizationControl(device, link, counter, control, reset): fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvLinkUtilizationControl") ret = fn(device, link, counter, byref(control), reset) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetNvLinkUtilizationControl(device, link, counter): c_control = nvmlNvLinkUtilizationControl_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkUtilizationControl") @@ -4067,6 +4762,7 @@ def nvmlDeviceGetNvLinkUtilizationControl(device, link, counter): _nvmlCheckReturn(ret) return c_control + def nvmlDeviceGetNvLinkCapability(device, link, capability): c_capResult = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkCapability") @@ -4074,6 +4770,7 @@ def nvmlDeviceGetNvLinkCapability(device, link, capability): _nvmlCheckReturn(ret) return c_capResult.value + def nvmlDeviceGetNvLinkErrorCounter(device, link, counter): c_result = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkErrorCounter") @@ -4081,12 +4778,14 @@ def nvmlDeviceGetNvLinkErrorCounter(device, link, counter): _nvmlCheckReturn(ret) return c_result.value + def nvmlDeviceResetNvLinkErrorCounters(device, link): fn = _nvmlGetFunctionPointer("nvmlDeviceResetNvLinkErrorCounters") ret = fn(device, link) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetNvLinkRemotePciInfo(device, link): c_pci = nvmlPciInfo_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemotePciInfo_v2") @@ -4094,6 +4793,7 @@ def nvmlDeviceGetNvLinkRemotePciInfo(device, link): _nvmlCheckReturn(ret) return c_pci + def nvmlDeviceGetNvLinkRemoteDeviceType(handle, link): c_type = _nvmlNvLinkDeviceType_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkRemoteDeviceType") @@ -4101,6 +4801,7 @@ def nvmlDeviceGetNvLinkRemoteDeviceType(handle, link): _nvmlCheckReturn(ret) return c_type.value + def nvmlDeviceGetNvLinkState(device, link): c_isActive = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkState") @@ -4108,6 +4809,7 @@ def nvmlDeviceGetNvLinkState(device, link): _nvmlCheckReturn(ret) return c_isActive.value + def nvmlDeviceGetNvLinkVersion(device, link): c_version = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvLinkVersion") @@ -4115,12 +4817,14 @@ def nvmlDeviceGetNvLinkVersion(device, link): _nvmlCheckReturn(ret) return c_version.value + def nvmlDeviceModifyDrainState(pciInfo, newState): fn = _nvmlGetFunctionPointer("nvmlDeviceModifyDrainState") ret = fn(pointer(pciInfo), newState) _nvmlCheckReturn(ret) return None + def nvmlDeviceQueryDrainState(pciInfo): c_newState = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceQueryDrainState") @@ -4128,18 +4832,21 @@ def nvmlDeviceQueryDrainState(pciInfo): _nvmlCheckReturn(ret) return c_newState.value + def nvmlDeviceRemoveGpu(pciInfo): fn = _nvmlGetFunctionPointer("nvmlDeviceRemoveGpu") ret = fn(pointer(pciInfo)) _nvmlCheckReturn(ret) return None + def nvmlDeviceDiscoverGpus(pciInfo): fn = _nvmlGetFunctionPointer("nvmlDeviceDiscoverGpus") ret = fn(pointer(pciInfo)) _nvmlCheckReturn(ret) return None + def nvmlDeviceGetFieldValues(handle, fieldIds): values_arr = c_nvmlFieldValue_t * len(fieldIds) values = values_arr() @@ -4155,6 +4862,7 @@ def nvmlDeviceGetFieldValues(handle, fieldIds): _nvmlCheckReturn(ret) return values + def nvmlDeviceClearFieldValues(handle, fieldIds): values_arr = c_nvmlFieldValue_t * len(fieldIds) values = values_arr() @@ -4170,6 +4878,7 @@ def nvmlDeviceClearFieldValues(handle, fieldIds): _nvmlCheckReturn(ret) return values + def nvmlDeviceGetVirtualizationMode(handle): c_virtualization_mode = c_ulonglong() fn = _nvmlGetFunctionPointer("nvmlDeviceGetVirtualizationMode") @@ -4177,10 +4886,12 @@ def nvmlDeviceGetVirtualizationMode(handle): _nvmlCheckReturn(ret) return c_virtualization_mode.value + def nvmlDeviceSetVirtualizationMode(handle, virtualization_mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetVirtualizationMode") return fn(handle, virtualization_mode) + def nvmlDeviceGetVgpuHeterogeneousMode(handle): c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0) c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1 @@ -4189,6 +4900,7 @@ def nvmlDeviceGetVgpuHeterogeneousMode(handle): _nvmlCheckReturn(ret) return c_vgpuHeterogeneousMode.mode + def nvmlDeviceSetVgpuHeterogeneousMode(handle, heterogeneous_mode): c_vgpuHeterogeneousMode = c_nvmlVgpuHeterogeneousMode_v1_t(0) c_vgpuHeterogeneousMode.version = VgpuHeterogeneousMode_v1 @@ -4198,6 +4910,7 @@ def nvmlDeviceSetVgpuHeterogeneousMode(handle, heterogeneous_mode): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlVgpuInstanceGetPlacementId(vgpuInstance): c_placement = c_nvmlVgpuPlacementId_v1_t(0) c_placement.version = VgpuPlacementId_v1 @@ -4206,9 +4919,10 @@ def nvmlVgpuInstanceGetPlacementId(vgpuInstance): _nvmlCheckReturn(ret) return c_placement.placementId + def nvmlDeviceGetVgpuTypeSupportedPlacements(handle, vgpuTypeId, mode=0, version=1): c_max_instances = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") ret = fn(handle, vgpuTypeId, byref(c_max_instances)) _nvmlCheckReturn(ret) @@ -4225,14 +4939,15 @@ def nvmlDeviceGetVgpuTypeSupportedPlacements(handle, vgpuTypeId, mode=0, version c_placements = c_uint * c_max_instances.value c_vgpu_placements.placementIds = c_placements() - fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeSupportedPlacements") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeSupportedPlacements") ret = fn(handle, vgpuTypeId, byref(c_vgpu_placements)) _nvmlCheckReturn(ret) return c_vgpu_placements + def nvmlDeviceGetVgpuTypeCreatablePlacements(handle, vgpuTypeId, version=1): c_max_instances = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") ret = fn(handle, vgpuTypeId, byref(c_max_instances)) _nvmlCheckReturn(ret) @@ -4246,11 +4961,12 @@ def nvmlDeviceGetVgpuTypeCreatablePlacements(handle, vgpuTypeId, version=1): c_placements = c_uint * c_max_instances.value c_vgpu_placements.placementIds = c_placements() - fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeCreatablePlacements") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuTypeCreatablePlacements") ret = fn(handle, vgpuTypeId, byref(c_vgpu_placements)) _nvmlCheckReturn(ret) return c_vgpu_placements + def nvmlGetVgpuDriverCapabilities(capability): c_capResult = c_uint() fn = _nvmlGetFunctionPointer("nvmlGetVgpuDriverCapabilities") @@ -4258,6 +4974,7 @@ def nvmlGetVgpuDriverCapabilities(capability): _nvmlCheckReturn(ret) return c_capResult.value + def nvmlDeviceGetVgpuCapabilities(handle, capability): c_capResult = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuCapabilities") @@ -4265,23 +4982,25 @@ def nvmlDeviceGetVgpuCapabilities(handle, capability): _nvmlCheckReturn(ret) return c_capResult.value + def nvmlDeviceSetVgpuCapabilities(handle, capability, state): fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuCapabilities") ret = fn(handle, _nvmlDeviceVgpuCapability_t(capability), state) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetSupportedVgpus(handle): # first call to get the size c_vgpu_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedVgpus") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetSupportedVgpus") ret = fn(handle, byref(c_vgpu_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no supported vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value c_vgpu_type_ids = vgpu_type_ids_array() @@ -4297,17 +5016,18 @@ def nvmlDeviceGetSupportedVgpus(handle): # error case raise NVMLError(ret) + def nvmlDeviceGetCreatableVgpus(handle): # first call to get the size c_vgpu_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetCreatableVgpus") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetCreatableVgpus") ret = fn(handle, byref(c_vgpu_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no supported vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case vgpu_type_ids_array = _nvmlVgpuTypeId_t * c_vgpu_count.value c_vgpu_type_ids = vgpu_type_ids_array() @@ -4323,132 +5043,148 @@ def nvmlDeviceGetCreatableVgpus(handle): # error case raise NVMLError(ret) + def nvmlVgpuTypeGetGpuInstanceProfileId(vgpuTypeId): c_profile_id = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGpuInstanceProfileId") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGpuInstanceProfileId") ret = fn(vgpuTypeId, byref(c_profile_id)) _nvmlCheckReturn(ret) - return (c_profile_id.value) + return c_profile_id.value + @convertStrBytes def nvmlVgpuTypeGetClass(vgpuTypeId): c_class = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE) c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetClass") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetClass") ret = fn(vgpuTypeId, c_class, byref(c_buffer_size)) _nvmlCheckReturn(ret) return c_class.value + @convertStrBytes def nvmlVgpuTypeGetName(vgpuTypeId): c_name = create_string_buffer(NVML_DEVICE_NAME_BUFFER_SIZE) c_buffer_size = c_uint(NVML_DEVICE_NAME_BUFFER_SIZE) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetName") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetName") ret = fn(vgpuTypeId, c_name, byref(c_buffer_size)) _nvmlCheckReturn(ret) return c_name.value + def nvmlVgpuTypeGetDeviceID(vgpuTypeId): - c_device_id = c_ulonglong(0) + c_device_id = c_ulonglong(0) c_subsystem_id = c_ulonglong(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetDeviceID") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetDeviceID") ret = fn(vgpuTypeId, byref(c_device_id), byref(c_subsystem_id)) _nvmlCheckReturn(ret) return (c_device_id.value, c_subsystem_id.value) + def nvmlVgpuTypeGetFramebufferSize(vgpuTypeId): c_fb_size = c_ulonglong(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFramebufferSize") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFramebufferSize") ret = fn(vgpuTypeId, byref(c_fb_size)) _nvmlCheckReturn(ret) return c_fb_size.value + def nvmlVgpuTypeGetNumDisplayHeads(vgpuTypeId): c_num_heads = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetNumDisplayHeads") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetNumDisplayHeads") ret = fn(vgpuTypeId, byref(c_num_heads)) _nvmlCheckReturn(ret) return c_num_heads.value + def nvmlVgpuTypeGetResolution(vgpuTypeId): c_xdim = c_uint(0) c_ydim = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetResolution") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetResolution") ret = fn(vgpuTypeId, 0, byref(c_xdim), byref(c_ydim)) _nvmlCheckReturn(ret) return (c_xdim.value, c_ydim.value) + @convertStrBytes def nvmlVgpuTypeGetLicense(vgpuTypeId): c_license = create_string_buffer(NVML_GRID_LICENSE_BUFFER_SIZE) c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetLicense") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetLicense") ret = fn(vgpuTypeId, c_license, c_buffer_size) _nvmlCheckReturn(ret) return c_license.value + def nvmlVgpuTypeGetFrameRateLimit(vgpuTypeId): c_frl_config = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFrameRateLimit") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFrameRateLimit") ret = fn(vgpuTypeId, byref(c_frl_config)) _nvmlCheckReturn(ret) return c_frl_config.value + def nvmlVgpuTypeGetGspHeapSize(vgpuTypeId): c_gsp_heap = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGspHeapSize") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetGspHeapSize") ret = fn(vgpuTypeId, byref(c_gsp_heap)) _nvmlCheckReturn(ret) return c_gsp_heap.value + def nvmlVgpuTypeGetFbReservation(vgpuTypeId): c_fb_reservation = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFbReservation") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetFbReservation") ret = fn(vgpuTypeId, byref(c_fb_reservation)) _nvmlCheckReturn(ret) return c_fb_reservation.value + def nvmlVgpuInstanceGetRuntimeStateSize(vgpuInstance): c_runtime_state = nvmlVgpuRuntimeState_v1_t() c_runtime_state.version = VgpuRuntimeState_v1 - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetRuntimeStateSize") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetRuntimeStateSize") ret = fn(vgpuInstance, byref(c_runtime_state)) _nvmlCheckReturn(ret) return c_runtime_state + def nvmlVgpuTypeGetMaxInstances(handle, vgpuTypeId): c_max_instances = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstances") ret = fn(handle, vgpuTypeId, byref(c_max_instances)) _nvmlCheckReturn(ret) return c_max_instances.value + def nvmlVgpuTypeGetMaxInstancesPerVm(vgpuTypeId): c_max_instances_per_vm = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerVm") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetMaxInstancesPerVm") ret = fn(vgpuTypeId, byref(c_max_instances_per_vm)) _nvmlCheckReturn(ret) return c_max_instances_per_vm.value + def nvmlVgpuTypeGetBAR1Info(vgpuTypeId): c_bar1Info = c_nvmlVgpuTypeBar1Info_v1_t(0) c_bar1Info.version = VgpuTypeBar1Info_v1 - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetBAR1Info") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetBAR1Info") ret = fn(vgpuTypeId, byref(c_bar1Info)) _nvmlCheckReturn(ret) return c_bar1Info + def nvmlDeviceGetActiveVgpus(handle): # first call to get the size c_vgpu_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetActiveVgpus") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetActiveVgpus") ret = fn(handle, byref(c_vgpu_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no active vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case vgpu_instance_array = _nvmlVgpuInstance_t * c_vgpu_count.value c_vgpu_instances = vgpu_instance_array() @@ -4464,67 +5200,76 @@ def nvmlDeviceGetActiveVgpus(handle): # error case raise NVMLError(ret) + @convertStrBytes def nvmlVgpuInstanceGetVmID(vgpuInstance): c_vm_id = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE) c_buffer_size = c_uint(NVML_GRID_LICENSE_BUFFER_SIZE) - c_vm_id_type = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmID") + c_vm_id_type = c_uint(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmID") ret = fn(vgpuInstance, byref(c_vm_id), c_buffer_size, byref(c_vm_id_type)) _nvmlCheckReturn(ret) return (c_vm_id.value, c_vm_id_type.value) + @convertStrBytes def nvmlVgpuInstanceGetUUID(vgpuInstance): c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE) c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetUUID") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetUUID") ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size) _nvmlCheckReturn(ret) return c_uuid.value + @convertStrBytes def nvmlVgpuInstanceGetMdevUUID(vgpuInstance): c_uuid = create_string_buffer(NVML_DEVICE_UUID_BUFFER_SIZE) c_buffer_size = c_uint(NVML_DEVICE_UUID_BUFFER_SIZE) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMdevUUID") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMdevUUID") ret = fn(vgpuInstance, byref(c_uuid), c_buffer_size) _nvmlCheckReturn(ret) return c_uuid.value + @convertStrBytes def nvmlVgpuInstanceGetVmDriverVersion(vgpuInstance): c_driver_version = create_string_buffer(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE) c_buffer_size = c_uint(NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmDriverVersion") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetVmDriverVersion") ret = fn(vgpuInstance, byref(c_driver_version), c_buffer_size) _nvmlCheckReturn(ret) return c_driver_version.value + def nvmlVgpuInstanceGetLicenseStatus(vgpuInstance): c_license_status = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseStatus") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseStatus") ret = fn(vgpuInstance, byref(c_license_status)) _nvmlCheckReturn(ret) return c_license_status.value + def nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance): - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseInfo_v2") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetLicenseInfo_v2") c_license_info = c_nvmlVgpuLicenseInfo_t() ret = fn(vgpuInstance, byref(c_license_info)) _nvmlCheckReturn(ret) return c_license_info + def nvmlVgpuInstanceGetLicenseInfo(vgpuInstance): return nvmlVgpuInstanceGetLicenseInfo_v2(vgpuInstance) + def nvmlVgpuInstanceGetFrameRateLimit(vgpuInstance): c_frl = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFrameRateLimit") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFrameRateLimit") ret = fn(vgpuInstance, byref(c_frl)) _nvmlCheckReturn(ret) return c_frl.value + def nvmlVgpuInstanceGetEccMode(vgpuInstance): c_mode = _nvmlEnableState_t() fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEccMode") @@ -4532,79 +5277,98 @@ def nvmlVgpuInstanceGetEccMode(vgpuInstance): _nvmlCheckReturn(ret) return c_mode.value + def nvmlVgpuInstanceGetType(vgpuInstance): c_vgpu_type = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetType") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetType") ret = fn(vgpuInstance, byref(c_vgpu_type)) _nvmlCheckReturn(ret) return c_vgpu_type.value + def nvmlVgpuInstanceGetEncoderCapacity(vgpuInstance): c_encoder_capacity = c_ulonglong(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderCapacity") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderCapacity") ret = fn(vgpuInstance, byref(c_encoder_capacity)) _nvmlCheckReturn(ret) return c_encoder_capacity.value + def nvmlVgpuInstanceSetEncoderCapacity(vgpuInstance, encoder_capacity): - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceSetEncoderCapacity") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceSetEncoderCapacity") return fn(vgpuInstance, encoder_capacity) + def nvmlVgpuInstanceGetFbUsage(vgpuInstance): c_fb_usage = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFbUsage") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFbUsage") ret = fn(vgpuInstance, byref(c_fb_usage)) _nvmlCheckReturn(ret) return c_fb_usage.value + def nvmlVgpuTypeGetCapabilities(vgpuTypeId, capability): c_cap_result = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetCapabilities") + fn = _nvmlGetFunctionPointer("nvmlVgpuTypeGetCapabilities") ret = fn(vgpuTypeId, _nvmlVgpuCapability_t(capability), byref(c_cap_result)) _nvmlCheckReturn(ret) - return (c_cap_result.value) + return c_cap_result.value + def nvmlVgpuInstanceGetGpuInstanceId(vgpuInstance): c_id = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuInstanceId") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuInstanceId") ret = fn(vgpuInstance, byref(c_id)) _nvmlCheckReturn(ret) - return (c_id.value) + return c_id.value + @convertStrBytes def nvmlVgpuInstanceGetGpuPciId(vgpuInstance): c_vgpuPciId = create_string_buffer(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE) fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetGpuPciId") - ret = fn(vgpuInstance, c_vgpuPciId, byref(c_uint(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE))) + ret = fn( + vgpuInstance, c_vgpuPciId, byref(c_uint(NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE)) + ) _nvmlCheckReturn(ret) return c_vgpuPciId.value + def nvmlDeviceGetVgpuUtilization(handle, timeStamp): # first call to get the size c_vgpu_count = c_uint(0) c_time_stamp = c_ulonglong(timeStamp) c_sample_value_type = _nvmlValueType_t() - fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuUtilization") - ret = fn(handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), None) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuUtilization") + ret = fn( + handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), None + ) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no active vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case sampleArray = c_vgpu_count.value * c_nvmlVgpuInstanceUtilizationSample_t c_samples = sampleArray() # make the call again - ret = fn(handle, c_time_stamp, byref(c_sample_value_type), byref(c_vgpu_count), c_samples) + ret = fn( + handle, + c_time_stamp, + byref(c_sample_value_type), + byref(c_vgpu_count), + c_samples, + ) _nvmlCheckReturn(ret) - return c_samples[0:c_vgpu_count.value] + return c_samples[0 : c_vgpu_count.value] else: # error case raise NVMLError(ret) + def nvmlDeviceGetVgpuInstancesUtilizationInfo(handle, timeStamp): # first call to get the size c_time_stamp = c_ulonglong(timeStamp) @@ -4614,15 +5378,17 @@ def nvmlDeviceGetVgpuInstancesUtilizationInfo(handle, timeStamp): c_vgpuUtilInfo.vgpuInstanceCount = c_uint(0) c_vgpuUtilInfo.lastSeenTimeStamp = c_time_stamp - fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuInstancesUtilizationInfo") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuInstancesUtilizationInfo") ret = fn(handle, byref(c_vgpuUtilInfo)) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no active vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case - sampleArray = c_vgpuUtilInfo.vgpuInstanceCount * c_nvmlVgpuInstanceUtilizationInfo_v1_t + sampleArray = ( + c_vgpuUtilInfo.vgpuInstanceCount * c_nvmlVgpuInstanceUtilizationInfo_v1_t + ) c_samples = sampleArray() c_vgpuUtilInfo.vgpuUtilArray = c_samples @@ -4630,29 +5396,33 @@ def nvmlDeviceGetVgpuInstancesUtilizationInfo(handle, timeStamp): ret = fn(handle, byref(c_vgpuUtilInfo)) _nvmlCheckReturn(ret) - return c_samples[0:c_vgpuUtilInfo.vgpuInstanceCount] + return c_samples[0 : c_vgpuUtilInfo.vgpuInstanceCount] else: # error case raise NVMLError(ret) + def nvmlDeviceGetP2PStatus(device1, device2, p2pIndex): c_p2pstatus = _nvmlGpuP2PStatus_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetP2PStatus") - ret = fn(device1, device2,p2pIndex, byref(c_p2pstatus)) + ret = fn(device1, device2, p2pIndex, byref(c_p2pstatus)) _nvmlCheckReturn(ret) return c_p2pstatus.value + def nvmlDeviceGetGridLicensableFeatures_v4(handle): c_get_grid_licensable_features = c_nvmlGridLicensableFeatures_v4_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetGridLicensableFeatures_v4") ret = fn(handle, byref(c_get_grid_licensable_features)) _nvmlCheckReturn(ret) - return (c_get_grid_licensable_features) + return c_get_grid_licensable_features + def nvmlDeviceGetGridLicensableFeatures(handle): return nvmlDeviceGetGridLicensableFeatures_v4(handle) + def nvmlDeviceGetGspFirmwareVersion(handle, version=None): isUserDefined = version is not None if not isUserDefined: @@ -4662,6 +5432,7 @@ def nvmlDeviceGetGspFirmwareVersion(handle, version=None): _nvmlCheckReturn(ret) return NVML_SUCCESS if isUserDefined else version.value + def nvmlDeviceGetGspFirmwareMode(handle, isEnabled=c_uint(), defaultMode=c_uint()): isReference = type(isEnabled) is not c_uint isEnabledRef = isEnabled if isReference else byref(isEnabled) @@ -4671,6 +5442,7 @@ def nvmlDeviceGetGspFirmwareMode(handle, isEnabled=c_uint(), defaultMode=c_uint( _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else [isEnabled.value, defaultMode.value] + def nvmlDeviceGetEncoderCapacity(handle, encoderQueryType): c_encoder_capacity = c_ulonglong(0) c_encoderQuery_type = _nvmlEncoderQueryType_t(encoderQueryType) @@ -4680,18 +5452,19 @@ def nvmlDeviceGetEncoderCapacity(handle, encoderQueryType): _nvmlCheckReturn(ret) return c_encoder_capacity.value + def nvmlDeviceGetVgpuProcessUtilization(handle, timeStamp): # first call to get the size c_vgpu_count = c_uint(0) c_time_stamp = c_ulonglong(timeStamp) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessUtilization") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessUtilization") ret = fn(handle, c_time_stamp, byref(c_vgpu_count), None) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no active vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case sampleArray = c_vgpu_count.value * c_nvmlVgpuProcessUtilizationSample_t c_samples = sampleArray() @@ -4700,11 +5473,12 @@ def nvmlDeviceGetVgpuProcessUtilization(handle, timeStamp): ret = fn(handle, c_time_stamp, byref(c_vgpu_count), c_samples) _nvmlCheckReturn(ret) - return c_samples[0:c_vgpu_count.value] + return c_samples[0 : c_vgpu_count.value] else: # error case raise NVMLError(ret) + def nvmlDeviceGetVgpuProcessesUtilizationInfo(handle, timeStamp): # first call to get the size c_time_stamp = c_ulonglong(timeStamp) @@ -4713,15 +5487,17 @@ def nvmlDeviceGetVgpuProcessesUtilizationInfo(handle, timeStamp): c_vgpuProcUtilInfo.vgpuProcessCount = c_uint(0) c_vgpuProcUtilInfo.lastSeenTimeStamp = c_time_stamp - fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessesUtilizationInfo") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuProcessesUtilizationInfo") ret = fn(handle, byref(c_vgpuProcUtilInfo)) - if (ret == NVML_SUCCESS): + if ret == NVML_SUCCESS: # special case, no active vGPUs return [] - elif (ret == NVML_ERROR_INSUFFICIENT_SIZE): + elif ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case - sampleArray = c_vgpuProcUtilInfo.vgpuProcessCount * c_nvmlVgpuProcessUtilizationInfo_v1_t + sampleArray = ( + c_vgpuProcUtilInfo.vgpuProcessCount * c_nvmlVgpuProcessUtilizationInfo_v1_t + ) c_samples = sampleArray() c_vgpuProcUtilInfo.vgpuProcUtilArray = c_samples @@ -4729,11 +5505,12 @@ def nvmlDeviceGetVgpuProcessesUtilizationInfo(handle, timeStamp): ret = fn(handle, byref(c_vgpuProcUtilInfo)) _nvmlCheckReturn(ret) - return c_samples[0:c_vgpuProcUtilInfo.vgpuProcessCount] + return c_samples[0 : c_vgpuProcUtilInfo.vgpuProcessCount] else: # error case raise NVMLError(ret) + def nvmlDeviceGetEncoderStats(handle): c_encoderCount = c_ulonglong(0) c_encodeFps = c_ulonglong(0) @@ -4743,15 +5520,16 @@ def nvmlDeviceGetEncoderStats(handle): _nvmlCheckReturn(ret) return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value) + def nvmlDeviceGetEncoderSessions(handle): # first call to get the size c_session_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderSessions") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetEncoderSessions") ret = fn(handle, byref(c_session_count), None) - if (ret == NVML_SUCCESS): - if (c_session_count.value != 0): + if ret == NVML_SUCCESS: + if c_session_count.value != 0: # typical case session_array = c_nvmlEncoderSession_t * c_session_count.value c_sessions = session_array() @@ -4769,6 +5547,7 @@ def nvmlDeviceGetEncoderSessions(handle): # error case raise NVMLError(ret) + def nvmlDeviceGetFBCStats(handle): c_fbcStats = c_nvmlFBCStats_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCStats") @@ -4776,15 +5555,16 @@ def nvmlDeviceGetFBCStats(handle): _nvmlCheckReturn(ret) return c_fbcStats + def nvmlDeviceGetFBCSessions(handle): # first call to get the size c_session_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCSessions") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetFBCSessions") ret = fn(handle, byref(c_session_count), None) - if (ret == NVML_SUCCESS): - if (c_session_count.value != 0): + if ret == NVML_SUCCESS: + if c_session_count.value != 0: # typical case session_array = c_nvmlFBCSession_t * c_session_count.value c_sessions = session_array() @@ -4802,24 +5582,28 @@ def nvmlDeviceGetFBCSessions(handle): # error case raise NVMLError(ret) + def nvmlVgpuInstanceGetEncoderStats(vgpuInstance): - c_encoderCount = c_ulonglong(0) - c_encodeFps = c_ulonglong(0) - c_encoderLatency = c_ulonglong(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderStats") - ret = fn(vgpuInstance, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency)) + c_encoderCount = c_ulonglong(0) + c_encodeFps = c_ulonglong(0) + c_encoderLatency = c_ulonglong(0) + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderStats") + ret = fn( + vgpuInstance, byref(c_encoderCount), byref(c_encodeFps), byref(c_encoderLatency) + ) _nvmlCheckReturn(ret) return (c_encoderCount.value, c_encodeFps.value, c_encoderLatency.value) + def nvmlVgpuInstanceGetEncoderSessions(vgpuInstance): # first call to get the size c_session_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderSessions") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetEncoderSessions") ret = fn(vgpuInstance, byref(c_session_count), None) - if (ret == NVML_SUCCESS): - if (c_session_count.value != 0): + if ret == NVML_SUCCESS: + if c_session_count.value != 0: # typical case session_array = c_nvmlEncoderSession_t * c_session_count.value c_sessions = session_array() @@ -4837,6 +5621,7 @@ def nvmlVgpuInstanceGetEncoderSessions(vgpuInstance): # error case raise NVMLError(ret) + def nvmlVgpuInstanceGetFBCStats(vgpuInstance): c_fbcStats = c_nvmlFBCStats_t() fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCStats") @@ -4844,15 +5629,16 @@ def nvmlVgpuInstanceGetFBCStats(vgpuInstance): _nvmlCheckReturn(ret) return c_fbcStats + def nvmlVgpuInstanceGetFBCSessions(vgpuInstance): # first call to get the size c_session_count = c_uint(0) - fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCSessions") + fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetFBCSessions") ret = fn(vgpuInstance, byref(c_session_count), None) - if (ret == NVML_SUCCESS): - if (c_session_count.value != 0): + if ret == NVML_SUCCESS: + if c_session_count.value != 0: # typical case session_array = c_nvmlFBCSession_t * c_session_count.value c_sessions = session_array() @@ -4870,15 +5656,16 @@ def nvmlVgpuInstanceGetFBCSessions(vgpuInstance): # error case raise NVMLError(ret) + def nvmlDeviceGetProcessUtilization(handle, timeStamp): # first call to get the size c_count = c_uint(0) c_time_stamp = c_ulonglong(timeStamp) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessUtilization") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessUtilization") ret = fn(handle, None, byref(c_count), c_time_stamp) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case sampleArray = c_count.value * c_nvmlProcessUtilizationSample_t c_samples = sampleArray() @@ -4887,11 +5674,12 @@ def nvmlDeviceGetProcessUtilization(handle, timeStamp): ret = fn(handle, c_samples, byref(c_count), c_time_stamp) _nvmlCheckReturn(ret) - return c_samples[0:c_count.value] + return c_samples[0 : c_count.value] else: # error case raise NVMLError(ret) + def nvmlDeviceGetProcessesUtilizationInfo(handle, timeStamp): # first call to get the size c_time_stamp = c_ulonglong(timeStamp) @@ -4900,12 +5688,14 @@ def nvmlDeviceGetProcessesUtilizationInfo(handle, timeStamp): c_processesUtilInfo.processSamplesCount = c_uint(0) c_processesUtilInfo.lastSeenTimeStamp = c_time_stamp - fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessesUtilizationInfo") + fn = _nvmlGetFunctionPointer("nvmlDeviceGetProcessesUtilizationInfo") ret = fn(handle, byref(c_processesUtilInfo)) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: # typical case - sampleArray = c_processesUtilInfo.processSamplesCount * c_nvmlProcessUtilizationInfo_v1_t + sampleArray = ( + c_processesUtilInfo.processSamplesCount * c_nvmlProcessUtilizationInfo_v1_t + ) c_samples = sampleArray() c_processesUtilInfo.procUtilArray = c_samples @@ -4913,11 +5703,12 @@ def nvmlDeviceGetProcessesUtilizationInfo(handle, timeStamp): ret = fn(handle, byref(c_processesUtilInfo)) _nvmlCheckReturn(ret) - return c_samples[0:c_processesUtilInfo.processSamplesCount] + return c_samples[0 : c_processesUtilInfo.processSamplesCount] else: # error case raise NVMLError(ret) + def nvmlVgpuInstanceGetMetadata(vgpuInstance): fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetMetadata") c_vgpuMetadata = c_nvmlVgpuMetadata_t() @@ -4925,13 +5716,14 @@ def nvmlVgpuInstanceGetMetadata(vgpuInstance): # Make the first NVML API call to get the c_bufferSize value. # We have already allocated required buffer above. ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize)) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: ret = fn(vgpuInstance, byref(c_vgpuMetadata), byref(c_bufferSize)) _nvmlCheckReturn(ret) else: raise NVMLError(ret) return c_vgpuMetadata + def nvmlDeviceGetVgpuMetadata(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuMetadata") c_vgpuPgpuMetadata = c_nvmlVgpuPgpuMetadata_t() @@ -4939,13 +5731,14 @@ def nvmlDeviceGetVgpuMetadata(handle): # Make the first NVML API call to get the c_bufferSize value. # We have already allocated required buffer above. ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize)) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: ret = fn(handle, byref(c_vgpuPgpuMetadata), byref(c_bufferSize)) _nvmlCheckReturn(ret) else: raise NVMLError(ret) return c_vgpuPgpuMetadata + def nvmlGetVgpuCompatibility(vgpuMetadata, pgpuMetadata): fn = _nvmlGetFunctionPointer("nvmlGetVgpuCompatibility") c_vgpuPgpuCompatibility = c_nvmlVgpuPgpuCompatibility_t() @@ -4953,6 +5746,7 @@ def nvmlGetVgpuCompatibility(vgpuMetadata, pgpuMetadata): _nvmlCheckReturn(ret) return c_vgpuPgpuCompatibility + @convertStrBytes def nvmlDeviceGetPgpuMetadataString(handle): fn = _nvmlGetFunctionPointer("nvmlDeviceGetPgpuMetadataString") @@ -4961,13 +5755,14 @@ def nvmlDeviceGetPgpuMetadataString(handle): # Make the first NVML API call to get the c_bufferSize value. # We have already allocated required buffer above. ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize)) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: ret = fn(handle, byref(c_pgpuMetadata), byref(c_bufferSize)) _nvmlCheckReturn(ret) else: raise NVMLError(ret) return (c_pgpuMetadata.value, c_bufferSize.value) + def nvmlDeviceGetVgpuSchedulerLog(handle): c_vgpu_sched_log = c_nvmlVgpuSchedulerLog_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerLog") @@ -4975,6 +5770,7 @@ def nvmlDeviceGetVgpuSchedulerLog(handle): _nvmlCheckReturn(ret) return c_vgpu_sched_log + def nvmlDeviceGetVgpuSchedulerState(handle): c_vgpu_sched_state = c_nvmlVgpuSchedulerGetState_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerState") @@ -4982,6 +5778,7 @@ def nvmlDeviceGetVgpuSchedulerState(handle): _nvmlCheckReturn(ret) return c_vgpu_sched_state + def nvmlDeviceGetVgpuSchedulerCapabilities(handle): c_vgpu_sched_caps = c_nvmlVgpuSchedulerCapabilities_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetVgpuSchedulerCapabilities") @@ -4989,18 +5786,21 @@ def nvmlDeviceGetVgpuSchedulerCapabilities(handle): _nvmlCheckReturn(ret) return c_vgpu_sched_caps + def nvmlDeviceSetVgpuSchedulerState(handle, sched_state): fn = _nvmlGetFunctionPointer("nvmlDeviceSetVgpuSchedulerState") ret = fn(handle, byref(sched_state)) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlSetVgpuVersion(vgpuVersion): fn = _nvmlGetFunctionPointer("nvmlSetVgpuVersion") ret = fn(byref(vgpuVersion)) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlGetVgpuVersion(supported=None, current=None): isUserDefined = (supported is not None) or (current is not None) if not isUserDefined: @@ -5009,10 +5809,15 @@ def nvmlGetVgpuVersion(supported=None, current=None): fn = _nvmlGetFunctionPointer("nvmlGetVgpuVersion") ret = fn(byref(supported), byref(current)) _nvmlCheckReturn(ret) - return NVML_SUCCESS if isUserDefined else [(supported.minVersion, - supported.maxVersion), - (current.minVersion, - current.maxVersion)] + return ( + NVML_SUCCESS + if isUserDefined + else [ + (supported.minVersion, supported.maxVersion), + (current.minVersion, current.maxVersion), + ] + ) + def nvmlVgpuInstanceGetAccountingMode(vgpuInstance): c_mode = _nvmlEnableState_t() @@ -5021,11 +5826,12 @@ def nvmlVgpuInstanceGetAccountingMode(vgpuInstance): _nvmlCheckReturn(ret) return c_mode.value + def nvmlVgpuInstanceGetAccountingPids(vgpuInstance): c_pidCount = c_uint() fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingPids") ret = fn(vgpuInstance, byref(c_pidCount), None) - if (ret == NVML_ERROR_INSUFFICIENT_SIZE): + if ret == NVML_ERROR_INSUFFICIENT_SIZE: sampleArray = c_pidCount.value * c_uint c_pidArray = sampleArray() ret = fn(vgpuInstance, byref(c_pidCount), byref(c_pidArray)) @@ -5034,6 +5840,7 @@ def nvmlVgpuInstanceGetAccountingPids(vgpuInstance): raise NVMLError(ret) return (c_pidCount, c_pidArray) + def nvmlVgpuInstanceGetAccountingStats(vgpuInstance, pid): c_accountingStats = c_nvmlAccountingStats_t() fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceGetAccountingStats") @@ -5041,12 +5848,14 @@ def nvmlVgpuInstanceGetAccountingStats(vgpuInstance, pid): _nvmlCheckReturn(ret) return c_accountingStats + def nvmlVgpuInstanceClearAccountingPids(vgpuInstance): fn = _nvmlGetFunctionPointer("nvmlVgpuInstanceClearAccountingPids") ret = fn(vgpuInstance) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlGetExcludedDeviceCount(): c_count = c_uint() fn = _nvmlGetFunctionPointer("nvmlGetExcludedDeviceCount") @@ -5054,6 +5863,7 @@ def nvmlGetExcludedDeviceCount(): _nvmlCheckReturn(ret) return c_count.value + def nvmlGetExcludedDeviceInfoByIndex(index): c_index = c_uint(index) info = c_nvmlExcludedDeviceInfo_t() @@ -5062,6 +5872,7 @@ def nvmlGetExcludedDeviceInfoByIndex(index): _nvmlCheckReturn(ret) return info + def nvmlDeviceGetHostVgpuMode(handle): c_host_vgpu_mode = _nvmlHostVgpuMode_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetHostVgpuMode") @@ -5069,6 +5880,7 @@ def nvmlDeviceGetHostVgpuMode(handle): _nvmlCheckReturn(ret) return c_host_vgpu_mode.value + def nvmlDeviceSetMigMode(device, mode): c_activationStatus = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceSetMigMode") @@ -5076,6 +5888,7 @@ def nvmlDeviceSetMigMode(device, mode): _nvmlCheckReturn(ret) return c_activationStatus.value + def nvmlDeviceGetMigMode(device): c_currentMode = c_uint() c_pendingMode = c_uint() @@ -5084,6 +5897,7 @@ def nvmlDeviceGetMigMode(device): _nvmlCheckReturn(ret) return [c_currentMode.value, c_pendingMode.value] + def nvmlDeviceGetGpuInstanceProfileInfo(device, profile, version=2): if version == 2: c_info = c_nvmlGpuInstanceProfileInfo_v2_t() @@ -5097,9 +5911,11 @@ def nvmlDeviceGetGpuInstanceProfileInfo(device, profile, version=2): _nvmlCheckReturn(ret) return c_info + # Define function alias for the API exposed by NVML nvmlDeviceGetGpuInstanceProfileInfoV = nvmlDeviceGetGpuInstanceProfileInfo + def nvmlDeviceGetGpuInstanceRemainingCapacity(device, profileId): c_count = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceRemainingCapacity") @@ -5107,12 +5923,16 @@ def nvmlDeviceGetGpuInstanceRemainingCapacity(device, profileId): _nvmlCheckReturn(ret) return c_count.value -def nvmlDeviceGetGpuInstancePossiblePlacements(device, profileId, placementsRef, countRef): + +def nvmlDeviceGetGpuInstancePossiblePlacements( + device, profileId, placementsRef, countRef +): fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstancePossiblePlacements_v2") ret = fn(device, profileId, placementsRef, countRef) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceCreateGpuInstance(device, profileId): c_instance = c_nvmlGpuInstance_t() fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstance") @@ -5120,6 +5940,7 @@ def nvmlDeviceCreateGpuInstance(device, profileId): _nvmlCheckReturn(ret) return c_instance + def nvmlDeviceCreateGpuInstanceWithPlacement(device, profileId, placement): c_instance = c_nvmlGpuInstance_t() fn = _nvmlGetFunctionPointer("nvmlDeviceCreateGpuInstanceWithPlacement") @@ -5127,18 +5948,21 @@ def nvmlDeviceCreateGpuInstanceWithPlacement(device, profileId, placement): _nvmlCheckReturn(ret) return c_instance + def nvmlGpuInstanceDestroy(gpuInstance): fn = _nvmlGetFunctionPointer("nvmlGpuInstanceDestroy") ret = fn(gpuInstance) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetGpuInstances(device, profileId, gpuInstancesRef, countRef): fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstances") ret = fn(device, profileId, gpuInstancesRef, countRef) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetGpuInstanceById(device, gpuInstanceId): c_instance = c_nvmlGpuInstance_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceById") @@ -5146,6 +5970,7 @@ def nvmlDeviceGetGpuInstanceById(device, gpuInstanceId): _nvmlCheckReturn(ret) return c_instance + def nvmlGpuInstanceGetInfo(gpuInstance): c_info = c_nvmlGpuInstanceInfo_t() fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetInfo") @@ -5153,7 +5978,10 @@ def nvmlGpuInstanceGetInfo(gpuInstance): _nvmlCheckReturn(ret) return c_info -def nvmlGpuInstanceGetComputeInstanceProfileInfo(device, profile, engProfile, version=2): + +def nvmlGpuInstanceGetComputeInstanceProfileInfo( + device, profile, engProfile, version=2 +): if version == 2: c_info = c_nvmlComputeInstanceProfileInfo_v2_t() fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfoV") @@ -5161,13 +5989,17 @@ def nvmlGpuInstanceGetComputeInstanceProfileInfo(device, profile, engProfile, ve c_info = c_nvmlComputeInstanceProfileInfo_t() fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceProfileInfo") else: - raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND) + raise NVMLError(NVML_ERROR_FUNCTION_NOT_FOUND) ret = fn(device, profile, engProfile, byref(c_info)) _nvmlCheckReturn(ret) return c_info + # Define function alias for the API exposed by NVML -nvmlGpuInstanceGetComputeInstanceProfileInfoV = nvmlGpuInstanceGetComputeInstanceProfileInfo +nvmlGpuInstanceGetComputeInstanceProfileInfoV = ( + nvmlGpuInstanceGetComputeInstanceProfileInfo +) + def nvmlGpuInstanceGetComputeInstanceRemainingCapacity(gpuInstance, profileId): c_count = c_uint() @@ -5176,12 +6008,16 @@ def nvmlGpuInstanceGetComputeInstanceRemainingCapacity(gpuInstance, profileId): _nvmlCheckReturn(ret) return c_count.value -def nvmlGpuInstanceGetComputeInstancePossiblePlacements(gpuInstance, profileId, placementsRef, countRef): + +def nvmlGpuInstanceGetComputeInstancePossiblePlacements( + gpuInstance, profileId, placementsRef, countRef +): fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstancePossiblePlacements") ret = fn(gpuInstance, profileId, placementsRef, countRef) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlGpuInstanceCreateComputeInstance(gpuInstance, profileId): c_instance = c_nvmlComputeInstance_t() fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstance") @@ -5189,25 +6025,33 @@ def nvmlGpuInstanceCreateComputeInstance(gpuInstance, profileId): _nvmlCheckReturn(ret) return c_instance -def nvmlGpuInstanceCreateComputeInstanceWithPlacement(gpuInstance, profileId, placement): + +def nvmlGpuInstanceCreateComputeInstanceWithPlacement( + gpuInstance, profileId, placement +): c_instance = c_nvmlComputeInstance_t() fn = _nvmlGetFunctionPointer("nvmlGpuInstanceCreateComputeInstanceWithPlacement") ret = fn(gpuInstance, profileId, placement, byref(c_instance)) _nvmlCheckReturn(ret) return c_instance + def nvmlComputeInstanceDestroy(computeInstance): fn = _nvmlGetFunctionPointer("nvmlComputeInstanceDestroy") ret = fn(computeInstance) _nvmlCheckReturn(ret) return NVML_SUCCESS -def nvmlGpuInstanceGetComputeInstances(gpuInstance, profileId, computeInstancesRef, countRef): + +def nvmlGpuInstanceGetComputeInstances( + gpuInstance, profileId, computeInstancesRef, countRef +): fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstances") ret = fn(gpuInstance, profileId, computeInstancesRef, countRef) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlGpuInstanceGetComputeInstanceById(gpuInstance, computeInstanceId): c_instance = c_nvmlComputeInstance_t() fn = _nvmlGetFunctionPointer("nvmlGpuInstanceGetComputeInstanceById") @@ -5215,6 +6059,7 @@ def nvmlGpuInstanceGetComputeInstanceById(gpuInstance, computeInstanceId): _nvmlCheckReturn(ret) return c_instance + def nvmlComputeInstanceGetInfo_v2(computeInstance): c_info = c_nvmlComputeInstanceInfo_t() fn = _nvmlGetFunctionPointer("nvmlComputeInstanceGetInfo_v2") @@ -5222,9 +6067,11 @@ def nvmlComputeInstanceGetInfo_v2(computeInstance): _nvmlCheckReturn(ret) return c_info + def nvmlComputeInstanceGetInfo(computeInstance): return nvmlComputeInstanceGetInfo_v2(computeInstance) + def nvmlDeviceIsMigDeviceHandle(device): c_isMigDevice = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceIsMigDeviceHandle") @@ -5232,6 +6079,7 @@ def nvmlDeviceIsMigDeviceHandle(device): _nvmlCheckReturn(ret) return c_isMigDevice + def nvmlDeviceGetGpuInstanceId(device): c_gpuInstanceId = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuInstanceId") @@ -5239,6 +6087,7 @@ def nvmlDeviceGetGpuInstanceId(device): _nvmlCheckReturn(ret) return c_gpuInstanceId.value + def nvmlDeviceGetComputeInstanceId(device): c_computeInstanceId = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetComputeInstanceId") @@ -5246,6 +6095,7 @@ def nvmlDeviceGetComputeInstanceId(device): _nvmlCheckReturn(ret) return c_computeInstanceId.value + def nvmlDeviceGetMaxMigDeviceCount(device): c_count = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMaxMigDeviceCount") @@ -5253,6 +6103,7 @@ def nvmlDeviceGetMaxMigDeviceCount(device): _nvmlCheckReturn(ret) return c_count.value + def nvmlDeviceGetMigDeviceHandleByIndex(device, index): c_index = c_uint(index) migDevice = c_nvmlDevice_t() @@ -5261,6 +6112,7 @@ def nvmlDeviceGetMigDeviceHandleByIndex(device, index): _nvmlCheckReturn(ret) return migDevice + def nvmlDeviceGetDeviceHandleFromMigDeviceHandle(migDevice): device = c_nvmlDevice_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetDeviceHandleFromMigDeviceHandle") @@ -5268,6 +6120,7 @@ def nvmlDeviceGetDeviceHandleFromMigDeviceHandle(migDevice): _nvmlCheckReturn(ret) return device + def nvmlDeviceGetAttributes_v2(device): c_attrs = c_nvmlDeviceAttributes() fn = _nvmlGetFunctionPointer("nvmlDeviceGetAttributes_v2") @@ -5275,9 +6128,11 @@ def nvmlDeviceGetAttributes_v2(device): _nvmlCheckReturn(ret) return c_attrs + def nvmlDeviceGetAttributes(device): return nvmlDeviceGetAttributes_v2(device) + def nvmlDeviceGetRemappedRows(device): fn = _nvmlGetFunctionPointer("nvmlDeviceGetRemappedRows") c_corr = c_uint() @@ -5288,6 +6143,7 @@ def nvmlDeviceGetRemappedRows(device): _nvmlCheckReturn(ret) return (c_corr.value, c_unc.value, c_bpending.value, c_bfailure.value) + def nvmlDeviceGetRowRemapperHistogram(device): c_vals = c_nvmlRowRemapperHistogramValues() fn = _nvmlGetFunctionPointer("nvmlDeviceGetRowRemapperHistogram") @@ -5295,6 +6151,7 @@ def nvmlDeviceGetRowRemapperHistogram(device): _nvmlCheckReturn(ret) return c_vals + def nvmlDeviceGetArchitecture(device): arch = _nvmlDeviceArchitecture_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetArchitecture") @@ -5302,6 +6159,7 @@ def nvmlDeviceGetArchitecture(device): _nvmlCheckReturn(ret) return arch.value + def nvmlDeviceGetBusType(device): c_busType = _nvmlBusType_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetBusType") @@ -5309,6 +6167,7 @@ def nvmlDeviceGetBusType(device): _nvmlCheckReturn(ret) return c_busType.value + def nvmlDeviceGetIrqNum(device): c_irqNum = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetIrqNum") @@ -5316,6 +6175,7 @@ def nvmlDeviceGetIrqNum(device): _nvmlCheckReturn(ret) return c_irqNum.value + def nvmlDeviceGetNumGpuCores(device): c_numCores = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetNumGpuCores") @@ -5323,6 +6183,7 @@ def nvmlDeviceGetNumGpuCores(device): _nvmlCheckReturn(ret) return c_numCores.value + def nvmlDeviceGetPowerSource(device): c_powerSource = _nvmlPowerSource_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPowerSource") @@ -5330,6 +6191,7 @@ def nvmlDeviceGetPowerSource(device): _nvmlCheckReturn(ret) return c_powerSource.value + def nvmlDeviceGetMemoryBusWidth(device): c_memBusWidth = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemoryBusWidth") @@ -5337,6 +6199,7 @@ def nvmlDeviceGetMemoryBusWidth(device): _nvmlCheckReturn(ret) return c_memBusWidth.value + def nvmlDeviceGetPcieLinkMaxSpeed(device): c_speed = _nvmlPcieLinkMaxSpeed_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieLinkMaxSpeed") @@ -5344,6 +6207,7 @@ def nvmlDeviceGetPcieLinkMaxSpeed(device): _nvmlCheckReturn(ret) return c_speed.value + def nvmlDeviceGetAdaptiveClockInfoStatus(device): c_adaptiveClockInfoStatus = _nvmlAdaptiveClockInfoStatus_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetAdaptiveClockInfoStatus") @@ -5351,6 +6215,7 @@ def nvmlDeviceGetAdaptiveClockInfoStatus(device): _nvmlCheckReturn(ret) return c_adaptiveClockInfoStatus.value + def nvmlDeviceGetPcieSpeed(device): c_speed = c_uint() fn = _nvmlGetFunctionPointer("nvmlDeviceGetPcieSpeed") @@ -5358,64 +6223,87 @@ def nvmlDeviceGetPcieSpeed(device): _nvmlCheckReturn(ret) return c_speed.value -def nvmlDeviceGetDynamicPstatesInfo(device, c_dynamicpstatesinfo=c_nvmlGpuDynamicPstatesInfo_t()): - isReference = type(c_dynamicpstatesinfo) is not c_nvmlGpuDynamicPstatesInfo_t - dynamicpstatesinfoRef = c_dynamicpstatesinfo if isReference else byref(c_dynamicpstatesinfo) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetDynamicPstatesInfo"); +def nvmlDeviceGetDynamicPstatesInfo( + device, c_dynamicpstatesinfo=c_nvmlGpuDynamicPstatesInfo_t() +): + isReference = type(c_dynamicpstatesinfo) is not c_nvmlGpuDynamicPstatesInfo_t + dynamicpstatesinfoRef = ( + c_dynamicpstatesinfo if isReference else byref(c_dynamicpstatesinfo) + ) + + fn = _nvmlGetFunctionPointer("nvmlDeviceGetDynamicPstatesInfo") ret = fn(device, dynamicpstatesinfoRef) _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else c_dynamicpstatesinfo + def nvmlDeviceSetFanSpeed_v2(handle, index, speed): - fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanSpeed_v2"); + fn = _nvmlGetFunctionPointer("nvmlDeviceSetFanSpeed_v2") ret = fn(handle, index, speed) _nvmlCheckReturn(ret) return NVML_SUCCESS -def nvmlDeviceGetThermalSettings(device, sensorindex, c_thermalsettings=c_nvmlGpuThermalSettings_t()): + +def nvmlDeviceGetThermalSettings( + device, sensorindex, c_thermalsettings=c_nvmlGpuThermalSettings_t() +): isReference = type(c_thermalsettings) is not c_nvmlGpuThermalSettings_t thermalsettingsRef = c_thermalsettings if isReference else byref(c_thermalsettings) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetThermalSettings"); + fn = _nvmlGetFunctionPointer("nvmlDeviceGetThermalSettings") ret = fn(device, sensorindex, thermalsettingsRef) _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else c_thermalsettings.sensor[:] -def nvmlDeviceGetMinMaxClockOfPState(device, clockType, pstate, minClockMHz=c_uint(), maxClockMHz=c_uint()): + +def nvmlDeviceGetMinMaxClockOfPState( + device, clockType, pstate, minClockMHz=c_uint(), maxClockMHz=c_uint() +): isReference = (type(minClockMHz) is not c_uint) or (type(maxClockMHz) is not c_uint) minClockMHzRef = minClockMHz if isReference else byref(minClockMHz) maxClockMHzRef = maxClockMHz if isReference else byref(maxClockMHz) - fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxClockOfPState"); - ret = fn(device, _nvmlClockType_t(clockType), _nvmlClockType_t(pstate), minClockMHzRef, maxClockMHzRef) + fn = _nvmlGetFunctionPointer("nvmlDeviceGetMinMaxClockOfPState") + ret = fn( + device, + _nvmlClockType_t(clockType), + _nvmlClockType_t(pstate), + minClockMHzRef, + maxClockMHzRef, + ) _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else (minClockMHz.value, maxClockMHz.value) + class c_nvmlClockOffset_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('type', _nvmlClockType_t), - ('pstate', _nvmlPstates_t), - ('clockOffsetMHz', c_int), - ('minClockOffsetMHz', c_int), - ('maxClockOffsetMHz', c_int), + ("version", c_uint), + ("type", _nvmlClockType_t), + ("pstate", _nvmlPstates_t), + ("clockOffsetMHz", c_int), + ("minClockOffsetMHz", c_int), + ("maxClockOffsetMHz", c_int), ] + nvmlClockOffset_v1 = 0x1000018 + def nvmlDeviceGetClockOffsets(device, info): - fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockOffsets"); + fn = _nvmlGetFunctionPointer("nvmlDeviceGetClockOffsets") ret = fn(device, info) return NVML_SUCCESS + def nvmlDeviceSetClockOffsets(device, info): - fn = _nvmlGetFunctionPointer("nvmlDeviceSetClockOffsets"); + fn = _nvmlGetFunctionPointer("nvmlDeviceSetClockOffsets") ret = fn(device, info) return NVML_SUCCESS + def nvmlDeviceGetSupportedPerformanceStates(device): pstates = [] c_count = c_uint(NVML_MAX_GPU_PERF_PSTATES) - c_size = sizeof(c_uint)*c_count.value + c_size = sizeof(c_uint) * c_count.value # NOTE: use 'c_uint' to represent the size of the nvmlPstate_t enumeration. pstates_array = _nvmlPstates_t * c_count.value @@ -5431,6 +6319,7 @@ def nvmlDeviceGetSupportedPerformanceStates(device): return pstates + def nvmlDeviceGetGpcClkVfOffset(device): offset = c_int32() fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpcClkVfOffset") @@ -5438,6 +6327,7 @@ def nvmlDeviceGetGpcClkVfOffset(device): _nvmlCheckReturn(ret) return offset.value + def nvmlDeviceSetGpcClkVfOffset(device, offset): c_offset = c_int32(offset) fn = _nvmlGetFunctionPointer("nvmlDeviceSetGpcClkVfOffset") @@ -5445,6 +6335,7 @@ def nvmlDeviceSetGpcClkVfOffset(device, offset): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetGpcClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int()): isReference = (type(minOffset) is not c_int) or (type(maxOffset) is not c_int) minOffsetRef = minOffset if isReference else byref(minOffset) @@ -5454,6 +6345,7 @@ def nvmlDeviceGetGpcClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else (minOffset.value, maxOffset.value) + def nvmlDeviceGetMemClkVfOffset(device): offset = c_int32() fn = _nvmlGetFunctionPointer("nvmlDeviceGetMemClkVfOffset") @@ -5461,6 +6353,7 @@ def nvmlDeviceGetMemClkVfOffset(device): _nvmlCheckReturn(ret) return offset.value + def nvmlDeviceSetMemClkVfOffset(device, offset): c_offset = c_int32(offset) fn = _nvmlGetFunctionPointer("nvmlDeviceSetMemClkVfOffset") @@ -5468,6 +6361,7 @@ def nvmlDeviceSetMemClkVfOffset(device, offset): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetMemClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int()): isReference = (type(minOffset) is not c_int) or (type(maxOffset) is not c_int) minOffsetRef = minOffset if isReference else byref(minOffset) @@ -5478,6 +6372,7 @@ def nvmlDeviceGetMemClkMinMaxVfOffset(device, minOffset=c_int(), maxOffset=c_int _nvmlCheckReturn(ret) return NVML_SUCCESS if isReference else (minOffset.value, maxOffset.value) + def nvmlSystemSetConfComputeGpusReadyState(state): c_state = c_uint(state) fn = _nvmlGetFunctionPointer("nvmlSystemSetConfComputeGpusReadyState") @@ -5485,6 +6380,7 @@ def nvmlSystemSetConfComputeGpusReadyState(state): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlSystemGetConfComputeGpusReadyState(): c_state = c_uint() fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeGpusReadyState") @@ -5492,6 +6388,7 @@ def nvmlSystemGetConfComputeGpusReadyState(): _nvmlCheckReturn(ret) return c_state.value + def nvmlSystemGetConfComputeCapabilities(): c_ccSysCaps = c_nvmlConfComputeSystemCaps_t() fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeCapabilities") @@ -5499,6 +6396,7 @@ def nvmlSystemGetConfComputeCapabilities(): _nvmlCheckReturn(ret) return c_ccSysCaps + def nvmlSystemGetConfComputeState(): c_state = c_nvmlConfComputeSystemState_t() fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeState") @@ -5506,16 +6404,19 @@ def nvmlSystemGetConfComputeState(): _nvmlCheckReturn(ret) return c_state + def nvmlSystemGetConfComputeSettings(settings): fn = _nvmlGetFunctionPointer("nvmlSystemGetConfComputeSettings") return fn(settings) + def nvmlDeviceSetConfComputeUnprotectedMemSize(device, c_ccMemSize): fn = _nvmlGetFunctionPointer("nvmlDeviceSetConfComputeUnprotectedMemSize") ret = fn(device, c_ccMemSize) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetConfComputeMemSizeInfo(device): c_ccMemSize = c_nvmlConfComputeMemSizeInfo_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeMemSizeInfo") @@ -5523,6 +6424,7 @@ def nvmlDeviceGetConfComputeMemSizeInfo(device): _nvmlCheckReturn(ret) return c_ccMemSize + def nvmlDeviceGetConfComputeProtectedMemoryUsage(device): c_memory = c_nvmlMemory_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeProtectedMemoryUsage") @@ -5530,6 +6432,7 @@ def nvmlDeviceGetConfComputeProtectedMemoryUsage(device): _nvmlCheckReturn(ret) return c_memory + def nvmlDeviceGetConfComputeGpuCertificate(device): c_cert = c_nvmlConfComputeGpuCertificate_t() fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeGpuCertificate") @@ -5537,15 +6440,17 @@ def nvmlDeviceGetConfComputeGpuCertificate(device): _nvmlCheckReturn(ret) return c_cert + def nvmlDeviceGetConfComputeGpuAttestationReport(device, c_nonce): c_attestReport = c_nvmlConfComputeGpuAttestationReport_t() c_nonce_arr = (c_uint8 * len(c_nonce))(*(c_nonce)) - setattr(c_attestReport, 'nonce', c_nonce_arr) + setattr(c_attestReport, "nonce", c_nonce_arr) fn = _nvmlGetFunctionPointer("nvmlDeviceGetConfComputeGpuAttestationReport") ret = fn(device, byref(c_attestReport)) _nvmlCheckReturn(ret) return c_attestReport + def nvmlSystemSetConfComputeKeyRotationThresholdInfo(max_atk_adv): c_keyRotationThrInfo = c_nvmlConfComputeSetKeyRotationThresholdInfo_t(0) c_keyRotationThrInfo.version = ConfComputeSetKeyRotationThresholdInfo_v1 @@ -5555,6 +6460,7 @@ def nvmlSystemSetConfComputeKeyRotationThresholdInfo(max_atk_adv): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlSystemGetConfComputeKeyRotationThresholdInfo(): c_keyRotationThrInfo = c_nvmlConfComputeGetKeyRotationThresholdInfo_t(0) c_keyRotationThrInfo.version = ConfComputeGetKeyRotationThresholdInfo_v1 @@ -5563,98 +6469,182 @@ def nvmlSystemGetConfComputeKeyRotationThresholdInfo(): _nvmlCheckReturn(ret) return c_keyRotationThrInfo + ## GPM ## ######### ## Enums/defines #### GPM Metric Identifiers -NVML_GPM_METRIC_GRAPHICS_UTIL = 1 # Percentage of time any compute/graphics app was active on the GPU. 0.0 - 100.0 -NVML_GPM_METRIC_SM_UTIL = 2 # Percentage of SMs that were busy. 0.0 - 100.0 -NVML_GPM_METRIC_SM_OCCUPANCY = 3 # Percentage of warps that were active vs theoretical maximum. 0.0 - 100.0 -NVML_GPM_METRIC_INTEGER_UTIL = 4 # Percentage of time the GPU's SMs were doing integer operations. 0.0 - 100.0 -NVML_GPM_METRIC_ANY_TENSOR_UTIL = 5 # Percentage of time the GPU's SMs were doing ANY tensor operations. 0.0 - 100.0 -NVML_GPM_METRIC_DFMA_TENSOR_UTIL = 6 # Percentage of time the GPU's SMs were doing DFMA tensor operations. 0.0 - 100.0 -NVML_GPM_METRIC_HMMA_TENSOR_UTIL = 7 # Percentage of time the GPU's SMs were doing HMMA tensor operations. 0.0 - 100.0 -NVML_GPM_METRIC_IMMA_TENSOR_UTIL = 9 # Percentage of time the GPU's SMs were doing IMMA tensor operations. 0.0 - 100.0 -NVML_GPM_METRIC_DRAM_BW_UTIL = 10 # Percentage of DRAM bw used vs theoretical maximum. 0.0 - 100.0 -NVML_GPM_METRIC_FP64_UTIL = 11 # Percentage of time the GPU's SMs were doing non-tensor FP64 math. 0.0 - 100.0 -NVML_GPM_METRIC_FP32_UTIL = 12 # Percentage of time the GPU's SMs were doing non-tensor FP32 math. 0.0 - 100.0 -NVML_GPM_METRIC_FP16_UTIL = 13 # Percentage of time the GPU's SMs were doing non-tensor FP16 math. 0.0 - 100.0 -NVML_GPM_METRIC_PCIE_TX_PER_SEC = 20 # PCIe traffic from this GPU in MiB/sec -NVML_GPM_METRIC_PCIE_RX_PER_SEC = 21 # PCIe traffic to this GPU in MiB/sec -NVML_GPM_METRIC_NVDEC_0_UTIL = 30 # Percent utilization of NVDEC 0. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_1_UTIL = 31 # Percent utilization of NVDEC 1. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_2_UTIL = 32 # Percent utilization of NVDEC 2. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_3_UTIL = 33 # Percent utilization of NVDEC 3. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_4_UTIL = 34 # Percent utilization of NVDEC 4. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_5_UTIL = 35 # Percent utilization of NVDEC 5. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_6_UTIL = 36 # Percent utilization of NVDEC 6. 0.0 - 100.0 -NVML_GPM_METRIC_NVDEC_7_UTIL = 37 # Percent utilization of NVDEC 7. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_0_UTIL = 40 # Percent utilization of NVJPG 0. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_1_UTIL = 41 # Percent utilization of NVJPG 1. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_2_UTIL = 42 # Percent utilization of NVJPG 2. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_3_UTIL = 43 # Percent utilization of NVJPG 3. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_4_UTIL = 44 # Percent utilization of NVJPG 4. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_5_UTIL = 45 # Percent utilization of NVJPG 5. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_6_UTIL = 46 # Percent utilization of NVJPG 6. 0.0 - 100.0 -NVML_GPM_METRIC_NVJPG_7_UTIL = 47 # Percent utilization of NVJPG 7. 0.0 - 100.0 -NVML_GPM_METRIC_NVOFA_0_UTIL = 50 # Percent utilization of NVOFA 0. 0.0 - 100.0 -NVML_GPM_METRIC_NVOFA_1_UTIL = 51 # Percent utilization of NVOFA 1. 0.0 - 100.0 -NVML_GPM_METRIC_NVLINK_TOTAL_RX_PER_SEC = 60 # NvLink read bandwidth for all links in MiB/sec -NVML_GPM_METRIC_NVLINK_TOTAL_TX_PER_SEC = 61 # NvLink write bandwidth for all links in MiB/sec -NVML_GPM_METRIC_NVLINK_L0_RX_PER_SEC = 62 # NvLink read bandwidth for link 0 in MiB/sec -NVML_GPM_METRIC_NVLINK_L0_TX_PER_SEC = 63 # NvLink write bandwidth for link 0 in MiB/sec -NVML_GPM_METRIC_NVLINK_L1_RX_PER_SEC = 64 # NvLink read bandwidth for link 1 in MiB/sec -NVML_GPM_METRIC_NVLINK_L1_TX_PER_SEC = 65 # NvLink write bandwidth for link 1 in MiB/sec -NVML_GPM_METRIC_NVLINK_L2_RX_PER_SEC = 66 # NvLink read bandwidth for link 2 in MiB/sec -NVML_GPM_METRIC_NVLINK_L2_TX_PER_SEC = 67 # NvLink write bandwidth for link 2 in MiB/sec -NVML_GPM_METRIC_NVLINK_L3_RX_PER_SEC = 68 # NvLink read bandwidth for link 3 in MiB/sec -NVML_GPM_METRIC_NVLINK_L3_TX_PER_SEC = 69 # NvLink write bandwidth for link 3 in MiB/sec -NVML_GPM_METRIC_NVLINK_L4_RX_PER_SEC = 70 # NvLink read bandwidth for link 4 in MiB/sec -NVML_GPM_METRIC_NVLINK_L4_TX_PER_SEC = 71 # NvLink write bandwidth for link 4 in MiB/sec -NVML_GPM_METRIC_NVLINK_L5_RX_PER_SEC = 72 # NvLink read bandwidth for link 5 in MiB/sec -NVML_GPM_METRIC_NVLINK_L5_TX_PER_SEC = 73 # NvLink write bandwidth for link 5 in MiB/sec -NVML_GPM_METRIC_NVLINK_L6_RX_PER_SEC = 74 # NvLink read bandwidth for link 6 in MiB/sec -NVML_GPM_METRIC_NVLINK_L6_TX_PER_SEC = 75 # NvLink write bandwidth for link 6 in MiB/sec -NVML_GPM_METRIC_NVLINK_L7_RX_PER_SEC = 76 # NvLink read bandwidth for link 7 in MiB/sec -NVML_GPM_METRIC_NVLINK_L7_TX_PER_SEC = 77 # NvLink write bandwidth for link 7 in MiB/sec -NVML_GPM_METRIC_NVLINK_L8_RX_PER_SEC = 78 # NvLink read bandwidth for link 8 in MiB/sec -NVML_GPM_METRIC_NVLINK_L8_TX_PER_SEC = 79 # NvLink write bandwidth for link 8 in MiB/sec -NVML_GPM_METRIC_NVLINK_L9_RX_PER_SEC = 80 # NvLink read bandwidth for link 9 in MiB/sec -NVML_GPM_METRIC_NVLINK_L9_TX_PER_SEC = 81 # NvLink write bandwidth for link 9 in MiB/sec -NVML_GPM_METRIC_NVLINK_L10_RX_PER_SEC = 82 # NvLink read bandwidth for link 10 in MiB/sec -NVML_GPM_METRIC_NVLINK_L10_TX_PER_SEC = 83 # NvLink write bandwidth for link 10 in MiB/sec -NVML_GPM_METRIC_NVLINK_L11_RX_PER_SEC = 84 # NvLink read bandwidth for link 11 in MiB/sec -NVML_GPM_METRIC_NVLINK_L11_TX_PER_SEC = 85 # NvLink write bandwidth for link 11 in MiB/sec -NVML_GPM_METRIC_NVLINK_L12_RX_PER_SEC = 86 # NvLink read bandwidth for link 12 in MiB/sec -NVML_GPM_METRIC_NVLINK_L12_TX_PER_SEC = 87 # NvLink write bandwidth for link 12 in MiB/sec -NVML_GPM_METRIC_NVLINK_L13_RX_PER_SEC = 88 # NvLink read bandwidth for link 13 in MiB/sec -NVML_GPM_METRIC_NVLINK_L13_TX_PER_SEC = 89 # NvLink write bandwidth for link 13 in MiB/sec -NVML_GPM_METRIC_NVLINK_L14_RX_PER_SEC = 90 # NvLink read bandwidth for link 14 in MiB/sec -NVML_GPM_METRIC_NVLINK_L14_TX_PER_SEC = 91 # NvLink write bandwidth for link 14 in MiB/sec -NVML_GPM_METRIC_NVLINK_L15_RX_PER_SEC = 92 # NvLink read bandwidth for link 15 in MiB/sec -NVML_GPM_METRIC_NVLINK_L15_TX_PER_SEC = 93 # NvLink write bandwidth for link 15 in MiB/sec -NVML_GPM_METRIC_NVLINK_L16_RX_PER_SEC = 94 # NvLink read bandwidth for link 16 in MiB/sec -NVML_GPM_METRIC_NVLINK_L16_TX_PER_SEC = 95 # NvLink write bandwidth for link 16 in MiB/sec -NVML_GPM_METRIC_NVLINK_L17_RX_PER_SEC = 96 # NvLink read bandwidth for link 17 in MiB/sec -NVML_GPM_METRIC_NVLINK_L17_TX_PER_SEC = 97 # NvLink write bandwidth for link 17 in MiB/sec -NVML_GPM_METRIC_MAX = 98 +NVML_GPM_METRIC_GRAPHICS_UTIL = ( + 1 # Percentage of time any compute/graphics app was active on the GPU. 0.0 - 100.0 +) +NVML_GPM_METRIC_SM_UTIL = 2 # Percentage of SMs that were busy. 0.0 - 100.0 +NVML_GPM_METRIC_SM_OCCUPANCY = ( + 3 # Percentage of warps that were active vs theoretical maximum. 0.0 - 100.0 +) +NVML_GPM_METRIC_INTEGER_UTIL = ( + 4 # Percentage of time the GPU's SMs were doing integer operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_ANY_TENSOR_UTIL = ( + 5 # Percentage of time the GPU's SMs were doing ANY tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_DFMA_TENSOR_UTIL = ( + 6 # Percentage of time the GPU's SMs were doing DFMA tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_HMMA_TENSOR_UTIL = ( + 7 # Percentage of time the GPU's SMs were doing HMMA tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_IMMA_TENSOR_UTIL = ( + 9 # Percentage of time the GPU's SMs were doing IMMA tensor operations. 0.0 - 100.0 +) +NVML_GPM_METRIC_DRAM_BW_UTIL = ( + 10 # Percentage of DRAM bw used vs theoretical maximum. 0.0 - 100.0 +) +NVML_GPM_METRIC_FP64_UTIL = ( + 11 # Percentage of time the GPU's SMs were doing non-tensor FP64 math. 0.0 - 100.0 +) +NVML_GPM_METRIC_FP32_UTIL = ( + 12 # Percentage of time the GPU's SMs were doing non-tensor FP32 math. 0.0 - 100.0 +) +NVML_GPM_METRIC_FP16_UTIL = ( + 13 # Percentage of time the GPU's SMs were doing non-tensor FP16 math. 0.0 - 100.0 +) +NVML_GPM_METRIC_PCIE_TX_PER_SEC = 20 # PCIe traffic from this GPU in MiB/sec +NVML_GPM_METRIC_PCIE_RX_PER_SEC = 21 # PCIe traffic to this GPU in MiB/sec +NVML_GPM_METRIC_NVDEC_0_UTIL = 30 # Percent utilization of NVDEC 0. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_1_UTIL = 31 # Percent utilization of NVDEC 1. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_2_UTIL = 32 # Percent utilization of NVDEC 2. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_3_UTIL = 33 # Percent utilization of NVDEC 3. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_4_UTIL = 34 # Percent utilization of NVDEC 4. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_5_UTIL = 35 # Percent utilization of NVDEC 5. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_6_UTIL = 36 # Percent utilization of NVDEC 6. 0.0 - 100.0 +NVML_GPM_METRIC_NVDEC_7_UTIL = 37 # Percent utilization of NVDEC 7. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_0_UTIL = 40 # Percent utilization of NVJPG 0. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_1_UTIL = 41 # Percent utilization of NVJPG 1. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_2_UTIL = 42 # Percent utilization of NVJPG 2. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_3_UTIL = 43 # Percent utilization of NVJPG 3. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_4_UTIL = 44 # Percent utilization of NVJPG 4. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_5_UTIL = 45 # Percent utilization of NVJPG 5. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_6_UTIL = 46 # Percent utilization of NVJPG 6. 0.0 - 100.0 +NVML_GPM_METRIC_NVJPG_7_UTIL = 47 # Percent utilization of NVJPG 7. 0.0 - 100.0 +NVML_GPM_METRIC_NVOFA_0_UTIL = 50 # Percent utilization of NVOFA 0. 0.0 - 100.0 +NVML_GPM_METRIC_NVOFA_1_UTIL = 51 # Percent utilization of NVOFA 1. 0.0 - 100.0 +NVML_GPM_METRIC_NVLINK_TOTAL_RX_PER_SEC = ( + 60 # NvLink read bandwidth for all links in MiB/sec +) +NVML_GPM_METRIC_NVLINK_TOTAL_TX_PER_SEC = ( + 61 # NvLink write bandwidth for all links in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L0_RX_PER_SEC = 62 # NvLink read bandwidth for link 0 in MiB/sec +NVML_GPM_METRIC_NVLINK_L0_TX_PER_SEC = ( + 63 # NvLink write bandwidth for link 0 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L1_RX_PER_SEC = 64 # NvLink read bandwidth for link 1 in MiB/sec +NVML_GPM_METRIC_NVLINK_L1_TX_PER_SEC = ( + 65 # NvLink write bandwidth for link 1 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L2_RX_PER_SEC = 66 # NvLink read bandwidth for link 2 in MiB/sec +NVML_GPM_METRIC_NVLINK_L2_TX_PER_SEC = ( + 67 # NvLink write bandwidth for link 2 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L3_RX_PER_SEC = 68 # NvLink read bandwidth for link 3 in MiB/sec +NVML_GPM_METRIC_NVLINK_L3_TX_PER_SEC = ( + 69 # NvLink write bandwidth for link 3 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L4_RX_PER_SEC = 70 # NvLink read bandwidth for link 4 in MiB/sec +NVML_GPM_METRIC_NVLINK_L4_TX_PER_SEC = ( + 71 # NvLink write bandwidth for link 4 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L5_RX_PER_SEC = 72 # NvLink read bandwidth for link 5 in MiB/sec +NVML_GPM_METRIC_NVLINK_L5_TX_PER_SEC = ( + 73 # NvLink write bandwidth for link 5 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L6_RX_PER_SEC = 74 # NvLink read bandwidth for link 6 in MiB/sec +NVML_GPM_METRIC_NVLINK_L6_TX_PER_SEC = ( + 75 # NvLink write bandwidth for link 6 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L7_RX_PER_SEC = 76 # NvLink read bandwidth for link 7 in MiB/sec +NVML_GPM_METRIC_NVLINK_L7_TX_PER_SEC = ( + 77 # NvLink write bandwidth for link 7 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L8_RX_PER_SEC = 78 # NvLink read bandwidth for link 8 in MiB/sec +NVML_GPM_METRIC_NVLINK_L8_TX_PER_SEC = ( + 79 # NvLink write bandwidth for link 8 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L9_RX_PER_SEC = 80 # NvLink read bandwidth for link 9 in MiB/sec +NVML_GPM_METRIC_NVLINK_L9_TX_PER_SEC = ( + 81 # NvLink write bandwidth for link 9 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L10_RX_PER_SEC = ( + 82 # NvLink read bandwidth for link 10 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L10_TX_PER_SEC = ( + 83 # NvLink write bandwidth for link 10 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L11_RX_PER_SEC = ( + 84 # NvLink read bandwidth for link 11 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L11_TX_PER_SEC = ( + 85 # NvLink write bandwidth for link 11 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L12_RX_PER_SEC = ( + 86 # NvLink read bandwidth for link 12 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L12_TX_PER_SEC = ( + 87 # NvLink write bandwidth for link 12 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L13_RX_PER_SEC = ( + 88 # NvLink read bandwidth for link 13 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L13_TX_PER_SEC = ( + 89 # NvLink write bandwidth for link 13 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L14_RX_PER_SEC = ( + 90 # NvLink read bandwidth for link 14 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L14_TX_PER_SEC = ( + 91 # NvLink write bandwidth for link 14 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L15_RX_PER_SEC = ( + 92 # NvLink read bandwidth for link 15 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L15_TX_PER_SEC = ( + 93 # NvLink write bandwidth for link 15 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L16_RX_PER_SEC = ( + 94 # NvLink read bandwidth for link 16 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L16_TX_PER_SEC = ( + 95 # NvLink write bandwidth for link 16 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L17_RX_PER_SEC = ( + 96 # NvLink read bandwidth for link 17 in MiB/sec +) +NVML_GPM_METRIC_NVLINK_L17_TX_PER_SEC = ( + 97 # NvLink write bandwidth for link 17 in MiB/sec +) +NVML_GPM_METRIC_MAX = 98 ## Structs + class c_nvmlUnitInfo_t(_PrintableStructure): _fields_ = [ - ('name', c_char * 96), - ('id', c_char * 96), - ('serial', c_char * 96), - ('firmwareVersion', c_char * 96), + ("name", c_char * 96), + ("id", c_char * 96), + ("serial", c_char * 96), + ("firmwareVersion", c_char * 96), ] + class struct_c_nvmlGpmSample_t(Structure): - pass # opaque handle + pass # opaque handle + + c_nvmlGpmSample_t = POINTER(struct_c_nvmlGpmSample_t) + class c_metricInfo_t(Structure): _fields_ = [ ("shortName", c_char_p), @@ -5662,47 +6652,55 @@ class c_metricInfo_t(Structure): ("unit", c_char_p), ] + class c_nvmlGpmMetric_t(_PrintableStructure): _fields_ = [ - ('metricId', c_uint), - ('nvmlReturn', _nvmlReturn_t), - ('value', c_double), - ('metricInfo', c_metricInfo_t) + ("metricId", c_uint), + ("nvmlReturn", _nvmlReturn_t), + ("value", c_double), + ("metricInfo", c_metricInfo_t), ] + class c_nvmlGpmMetricsGet_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('numMetrics', c_uint), - ('sample1', c_nvmlGpmSample_t), - ('sample2', c_nvmlGpmSample_t), - ('metrics', c_nvmlGpmMetric_t * NVML_GPM_METRIC_MAX) + ("version", c_uint), + ("numMetrics", c_uint), + ("sample1", c_nvmlGpmSample_t), + ("sample2", c_nvmlGpmSample_t), + ("metrics", c_nvmlGpmMetric_t * NVML_GPM_METRIC_MAX), ] + NVML_GPM_METRICS_GET_VERSION = 1 + class c_nvmlGpmSupport_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('isSupportedDevice', c_uint), + ("version", c_uint), + ("isSupportedDevice", c_uint), ] + NVML_GPM_SUPPORT_VERSION = 1 ## Functions + def nvmlGpmMetricsGet(metricsGet): fn = _nvmlGetFunctionPointer("nvmlGpmMetricsGet") ret = fn(byref(metricsGet)) _nvmlCheckReturn(ret) return metricsGet + def nvmlGpmSampleFree(gpmSample): fn = _nvmlGetFunctionPointer("nvmlGpmSampleFree") ret = fn(gpmSample) _nvmlCheckReturn(ret) return + def nvmlGpmSampleAlloc(): gpmSample = c_nvmlGpmSample_t() fn = _nvmlGetFunctionPointer("nvmlGpmSampleAlloc") @@ -5710,18 +6708,21 @@ def nvmlGpmSampleAlloc(): _nvmlCheckReturn(ret) return gpmSample + def nvmlGpmSampleGet(device, gpmSample): fn = _nvmlGetFunctionPointer("nvmlGpmSampleGet") ret = fn(device, gpmSample) _nvmlCheckReturn(ret) return gpmSample + def nvmlGpmMigSampleGet(device, gpuInstanceId, gpmSample): fn = _nvmlGetFunctionPointer("nvmlGpmMigSampleGet") ret = fn(device, gpuInstanceId, gpmSample) _nvmlCheckReturn(ret) return gpmSample + def nvmlGpmQueryDeviceSupport(device): gpmSupport = c_nvmlGpmSupport_t() gpmSupport.version = NVML_GPM_SUPPORT_VERSION @@ -5730,6 +6731,7 @@ def nvmlGpmQueryDeviceSupport(device): _nvmlCheckReturn(ret) return gpmSupport + def nvmlGpmSetStreamingEnabled(device, state): c_state = c_uint(state) fn = _nvmlGetFunctionPointer("nvmlGpmSetStreamingEnabled") @@ -5737,6 +6739,7 @@ def nvmlGpmSetStreamingEnabled(device, state): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlGpmQueryIfStreamingEnabled(device): c_state = c_uint() fn = _nvmlGetFunctionPointer("nvmlGpmQueryIfStreamingEnabled") @@ -5744,21 +6747,24 @@ def nvmlGpmQueryIfStreamingEnabled(device): _nvmlCheckReturn(ret) return c_state.value + # Low Power Structure and Function -NVML_NVLINK_POWER_STATE_HIGH_SPEED = 0x0 -NVML_NVLINK_POWER_STATE_LOW = 0x1 +NVML_NVLINK_POWER_STATE_HIGH_SPEED = 0x0 +NVML_NVLINK_POWER_STATE_LOW = 0x1 -NVML_NVLINK_LOW_POWER_THRESHOLD_MIN = 0x1 -NVML_NVLINK_LOW_POWER_THRESHOLD_MAX = 0x1FFF -NVML_NVLINK_LOW_POWER_THRESHOLD_RESET = 0xFFFFFFFF +NVML_NVLINK_LOW_POWER_THRESHOLD_MIN = 0x1 +NVML_NVLINK_LOW_POWER_THRESHOLD_MAX = 0x1FFF +NVML_NVLINK_LOW_POWER_THRESHOLD_RESET = 0xFFFFFFFF NVML_NVLINK_LOW_POWER_THRESHOLD_DEFAULT = NVML_NVLINK_LOW_POWER_THRESHOLD_RESET + class c_nvmlNvLinkPowerThres_t(Structure): _fields_ = [ ("lowPwrThreshold", c_uint), ] + def nvmlDeviceSetNvLinkDeviceLowPowerThreshold(device, l1threshold): c_info = c_nvmlNvLinkPowerThres_t() c_info.lowPwrThreshold = l1threshold @@ -5767,48 +6773,52 @@ def nvmlDeviceSetNvLinkDeviceLowPowerThreshold(device, l1threshold): _nvmlCheckReturn(ret) return NVML_SUCCESS + NVML_GPU_FABRIC_UUID_LEN = 16 _nvmlGpuFabricState_t = c_uint NVML_GPU_FABRIC_STATE_NOT_SUPPORTED = 0 -NVML_GPU_FABRIC_STATE_NOT_STARTED = 1 -NVML_GPU_FABRIC_STATE_IN_PROGRESS = 2 -NVML_GPU_FABRIC_STATE_COMPLETED = 3 +NVML_GPU_FABRIC_STATE_NOT_STARTED = 1 +NVML_GPU_FABRIC_STATE_IN_PROGRESS = 2 +NVML_GPU_FABRIC_STATE_COMPLETED = 3 + class c_nvmlGpuFabricInfo_t(_PrintableStructure): _fields_ = [ ("clusterUuid", c_char * NVML_DEVICE_UUID_BUFFER_SIZE), ("status", _nvmlReturn_t), ("cliqueId", c_uint32), - ("state", _nvmlGpuFabricState_t) + ("state", _nvmlGpuFabricState_t), ] + NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_NOT_SUPPORTED = 0 -NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_TRUE = 1 -NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE = 2 -NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW = 0 -NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW = 0x11 +NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_DEGRADED_BW_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_DEGRADED_BW = 0 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_DEGRADED_BW = 0x11 -NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_NOT_SUPPORTED = 0 -NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_TRUE = 1 -NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE = 2 -NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY = 2 -NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY = 0x11 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_RECOVERY_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_RECOVERY = 2 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_RECOVERY = 0x11 -NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_NOT_SUPPORTED = 0 -NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_TRUE = 1 -NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE = 2 -NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY = 4 -NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY = 0x11 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_NOT_SUPPORTED = 0 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_ROUTE_UNHEALTHY_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ROUTE_UNHEALTHY = 4 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ROUTE_UNHEALTHY = 0x11 NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_NOT_SUPPORTED = 0 -NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_TRUE = 1 -NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_FALSE = 2 -NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY = 6 -NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY = 0x11 +NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_TRUE = 1 +NVML_GPU_FABRIC_HEALTH_MASK_ACCESS_TIMEOUT_RECOVERY_FALSE = 2 +NVML_GPU_FABRIC_HEALTH_MASK_SHIFT_ACCESS_TIMEOUT_RECOVERY = 6 +NVML_GPU_FABRIC_HEALTH_MASK_WIDTH_ACCESS_TIMEOUT_RECOVERY = 0x11 nvmlGpuFabricInfo_v2 = 0x02000024 + class c_nvmlGpuFabricInfoV_t(_PrintableStructure): _fields_ = [ ("version", c_uint), @@ -5816,33 +6826,37 @@ class c_nvmlGpuFabricInfoV_t(_PrintableStructure): ("status", _nvmlReturn_t), ("cliqueId", c_uint32), ("state", _nvmlGpuFabricState_t), - ("healthMask", c_uint32) + ("healthMask", c_uint32), ] def __init__(self): super(c_nvmlGpuFabricInfoV_t, self).__init__(version=nvmlGpuFabricInfo_v2) + def nvmlDeviceGetGpuFabricInfo(device, gpuFabricInfo): - fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfo"); + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfo") ret = fn(device, gpuFabricInfo) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetGpuFabricInfoV(device, gpuFabricInfo): - fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfoV"); + fn = _nvmlGetFunctionPointer("nvmlDeviceGetGpuFabricInfoV") ret = fn(device, gpuFabricInfo) _nvmlCheckReturn(ret) return NVML_SUCCESS + ###################### ## Enums/defines #### NVML GPU NVLINK BW MODE -NVML_GPU_NVLINK_BW_MODE_FULL = 0x0 -NVML_GPU_NVLINK_BW_MODE_OFF = 0x1 -NVML_GPU_NVLINK_BW_MODE_MIN = 0x2 -NVML_GPU_NVLINK_BW_MODE_HALF = 0x3 -NVML_GPU_NVLINK_BW_MODE_3QUARTER = 0x4 -NVML_GPU_NVLINK_BW_MODE_COUNT = 0x5 +NVML_GPU_NVLINK_BW_MODE_FULL = 0x0 +NVML_GPU_NVLINK_BW_MODE_OFF = 0x1 +NVML_GPU_NVLINK_BW_MODE_MIN = 0x2 +NVML_GPU_NVLINK_BW_MODE_HALF = 0x3 +NVML_GPU_NVLINK_BW_MODE_3QUARTER = 0x4 +NVML_GPU_NVLINK_BW_MODE_COUNT = 0x5 + def nvmlSystemSetNvlinkBwMode(mode): fn = _nvmlGetFunctionPointer("nvmlSystemSetNvlinkBwMode") @@ -5850,6 +6864,7 @@ def nvmlSystemSetNvlinkBwMode(mode): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlSystemGetNvlinkBwMode(): mode = c_uint() fn = _nvmlGetFunctionPointer("nvmlSystemGetNvlinkBwMode") @@ -5857,22 +6872,28 @@ def nvmlSystemGetNvlinkBwMode(): _nvmlCheckReturn(ret) return mode.value + _nvmlPowerScopeType_t = c_uint -NVML_POWER_SCOPE_GPU = 0 -NVML_POWER_SCOPE_MODULE = 1 -NVML_POWER_SCOPE_MEMORY = 2 +NVML_POWER_SCOPE_GPU = 0 +NVML_POWER_SCOPE_MODULE = 1 +NVML_POWER_SCOPE_MEMORY = 2 + class c_nvmlPowerValue_v2_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('powerScope', _nvmlPowerScopeType_t), - ('powerValueMw', c_uint), + ("version", c_uint), + ("powerScope", _nvmlPowerScopeType_t), + ("powerValueMw", c_uint), ] - _fmt_ = {'': "%d B"} + _fmt_ = {"": "%d B"} + nvmlPowerValue_v2 = 0x0200000C -def nvmlDeviceSetPowerManagementLimit_v2(device, powerScope, powerLimit, version=nvmlPowerValue_v2): + +def nvmlDeviceSetPowerManagementLimit_v2( + device, powerScope, powerLimit, version=nvmlPowerValue_v2 +): c_powerScope = _nvmlPowerScopeType_t(powerScope) c_powerValue = c_nvmlPowerValue_v2_t() c_powerValue.version = c_uint(version) @@ -5882,139 +6903,178 @@ def nvmlDeviceSetPowerManagementLimit_v2(device, powerScope, powerLimit, version ret = fn(device, byref(c_powerValue)) return NVML_SUCCESS + class c_nvmlEccSramErrorStatus_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('aggregateUncParity', c_ulonglong), - ('aggregateUncSecDed', c_ulonglong), - ('aggregateCor', c_ulonglong), - ('volatileUncParity', c_ulonglong), - ('volatileUncSecDed', c_ulonglong), - ('volatileCor', c_ulonglong), - ('aggregateUncBucketL2', c_ulonglong), - ('aggregateUncBucketSm', c_ulonglong), - ('aggregateUncBucketPcie', c_ulonglong), - ('aggregateUncBucketMcu', c_ulonglong), - ('aggregateUncBucketOther', c_ulonglong), - ('bThresholdExceeded', c_uint) + ("version", c_uint), + ("aggregateUncParity", c_ulonglong), + ("aggregateUncSecDed", c_ulonglong), + ("aggregateCor", c_ulonglong), + ("volatileUncParity", c_ulonglong), + ("volatileUncSecDed", c_ulonglong), + ("volatileCor", c_ulonglong), + ("aggregateUncBucketL2", c_ulonglong), + ("aggregateUncBucketSm", c_ulonglong), + ("aggregateUncBucketPcie", c_ulonglong), + ("aggregateUncBucketMcu", c_ulonglong), + ("aggregateUncBucketOther", c_ulonglong), + ("bThresholdExceeded", c_uint), ] def __init__(self): - super(c_nvmlEccSramErrorStatus_v1_t, self).__init__(version=nvmlEccSramErrorStatus_v1) + super(c_nvmlEccSramErrorStatus_v1_t, self).__init__( + version=nvmlEccSramErrorStatus_v1 + ) + nvmlEccSramErrorStatus_v1 = 0x1000068 + + def nvmlDeviceGetSramEccErrorStatus(device, status): fn = _nvmlGetFunctionPointer("nvmlDeviceGetSramEccErrorStatus") ret = fn(device, status) _nvmlCheckReturn(ret) return NVML_SUCCESS -NVML_DEV_CAP_EGM = (1 << 0) + +NVML_DEV_CAP_EGM = 1 << 0 nvmlDeviceCapabilities_v1 = 0x1000008 + class c_nvmlDeviceCapabilities_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('capMask', c_uint), + ("version", c_uint), + ("capMask", c_uint), ] def __init__(self): - super(c_nvmlDeviceCapabilities_v1_t, self).__init__(version=nvmlDeviceCapabilities_v1) + super(c_nvmlDeviceCapabilities_v1_t, self).__init__( + version=nvmlDeviceCapabilities_v1 + ) def nvmlDeviceGetCapabilities(device, caps): fn = _nvmlGetFunctionPointer("nvmlDeviceGetCapabilities") return fn(device, caps) + class c_nvmlPlatformInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('ibGuid', c_char * 16), - ('rackGuid', c_char * 16), - ('chassisPhysicalSlotNumber', c_char), - ('computeSlotIndex', c_char), - ('nodeIndex', c_char), - ('peerType', c_char), - ('moduleId', c_char) + ("version", c_uint), + ("ibGuid", c_char * 16), + ("rackGuid", c_char * 16), + ("chassisPhysicalSlotNumber", c_char), + ("computeSlotIndex", c_char), + ("nodeIndex", c_char), + ("peerType", c_char), + ("moduleId", c_char), ] def __init__(self): super(c_nvmlPlatformInfo_v1_t, self).__init__(version=nvmlPlatformInfo_v1) -nvmlPlatformInfo_v1 = 0x100002c + +nvmlPlatformInfo_v1 = 0x100002C + + def nvmlDeviceGetPlatformInfo(device, platformInfo): fn = _nvmlGetFunctionPointer("nvmlDeviceGetPlatformInfo") ret = fn(device, platformInfo) _nvmlCheckReturn(ret) return NVML_SUCCESS + class c_nvmlMask255_t(_PrintableStructure): _fields_ = [ - ('mask', c_uint * 8), + ("mask", c_uint * 8), ] -NVML_WORKLOAD_POWER_MAX_PROFILES = 255 -NVML_POWER_PROFILE_MAX_P = 0 -NVML_POWER_PROFILE_MAX_Q = 1 -NVML_POWER_PROFILE_COMPUTE = 2 -NVML_POWER_PROFILE_MEMORY_BOUND = 3 -NVML_POWER_PROFILE_NETWORK = 4 -NVML_POWER_PROFILE_BALANCED = 5 -NVML_POWER_PROFILE_LLM_INFERENCE = 6 -NVML_POWER_PROFILE_LLM_TRAINING = 7 -NVML_POWER_PROFILE_RBM = 8 -NVML_POWER_PROFILE_DCPCIE = 9 -NVML_POWER_PROFILE_HMMA_SPARSE = 10 -NVML_POWER_PROFILE_HMMA_DENSE = 11 -NVML_POWER_PROFILE_SYNC_BALANCED = 12 -NVML_POWER_PROFILE_HPC = 13 -NVML_POWER_PROFILE_MIG = 14 -NVML_POWER_PROFILE_MAX = 15 -nvmlWorkloadPowerProfileInfo_v1 = 0x100002c +NVML_WORKLOAD_POWER_MAX_PROFILES = 255 +NVML_POWER_PROFILE_MAX_P = 0 +NVML_POWER_PROFILE_MAX_Q = 1 +NVML_POWER_PROFILE_COMPUTE = 2 +NVML_POWER_PROFILE_MEMORY_BOUND = 3 +NVML_POWER_PROFILE_NETWORK = 4 +NVML_POWER_PROFILE_BALANCED = 5 +NVML_POWER_PROFILE_LLM_INFERENCE = 6 +NVML_POWER_PROFILE_LLM_TRAINING = 7 +NVML_POWER_PROFILE_RBM = 8 +NVML_POWER_PROFILE_DCPCIE = 9 +NVML_POWER_PROFILE_HMMA_SPARSE = 10 +NVML_POWER_PROFILE_HMMA_DENSE = 11 +NVML_POWER_PROFILE_SYNC_BALANCED = 12 +NVML_POWER_PROFILE_HPC = 13 +NVML_POWER_PROFILE_MIG = 14 +NVML_POWER_PROFILE_MAX = 15 + +nvmlWorkloadPowerProfileInfo_v1 = 0x100002C + + class c_nvmlWorkloadPowerProfileInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('profileId', c_uint), - ('priority', c_uint), - ('conflictingmask', c_nvmlMask255_t) + ("version", c_uint), + ("profileId", c_uint), + ("priority", c_uint), + ("conflictingmask", c_nvmlMask255_t), ] def __init__(self): - super(c_nvmlWorkloadPowerProfileInfo_v1_t, self).__init__(version=nvmlWorkloadPowerProfileInfo_v1) + super(c_nvmlWorkloadPowerProfileInfo_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileInfo_v1 + ) + + +nvmlWorkloadPowerProfileProfilesInfo_v1 = 0x1002BF8 + -nvmlWorkloadPowerProfileProfilesInfo_v1 = 0x1002bf8 class c_nvmlWorkloadPowerProfileProfilesInfo_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('perfProfilesMask', c_nvmlMask255_t), - ('perfProfile', c_nvmlWorkloadPowerProfileInfo_v1_t * NVML_WORKLOAD_POWER_MAX_PROFILES) + ("version", c_uint), + ("perfProfilesMask", c_nvmlMask255_t), + ( + "perfProfile", + c_nvmlWorkloadPowerProfileInfo_v1_t * NVML_WORKLOAD_POWER_MAX_PROFILES, + ), ] def __init__(self): - super(c_nvmlWorkloadPowerProfileProfilesInfo_v1_t, self).__init__(version=nvmlWorkloadPowerProfileProfilesInfo_v1) + super(c_nvmlWorkloadPowerProfileProfilesInfo_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileProfilesInfo_v1 + ) + nvmlWorkloadPowerProfileCurrentProfiles_v1 = 0x1000064 + + class c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('perfProfilesMask', c_nvmlMask255_t), - ('requestedProfilesMask', c_nvmlMask255_t), - ('enforcedProfilesMask', c_nvmlMask255_t) + ("version", c_uint), + ("perfProfilesMask", c_nvmlMask255_t), + ("requestedProfilesMask", c_nvmlMask255_t), + ("enforcedProfilesMask", c_nvmlMask255_t), ] def __init__(self): - super(c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t, self).__init__(version=nvmlWorkloadPowerProfileCurrentProfiles_v1) + super(c_nvmlWorkloadPowerProfileCurrentProfiles_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileCurrentProfiles_v1 + ) + nvmlWorkloadPowerProfileRequestedProfiles_v1 = 0x1000024 + + class c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('requestedProfilesMask', c_nvmlMask255_t), + ("version", c_uint), + ("requestedProfilesMask", c_nvmlMask255_t), ] def __init__(self): - super(c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t, self).__init__(version=nvmlWorkloadPowerProfileRequestedProfiles_v1) + super(c_nvmlWorkloadPowerProfileRequestedProfiles_v1_t, self).__init__( + version=nvmlWorkloadPowerProfileRequestedProfiles_v1 + ) + def nvmlDeviceWorkloadPowerProfileGetProfilesInfo(device, profilesInfo): fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileGetProfilesInfo") @@ -6022,52 +7082,63 @@ def nvmlDeviceWorkloadPowerProfileGetProfilesInfo(device, profilesInfo): _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceWorkloadPowerProfileGetCurrentProfiles(device, currentProfiles): fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileGetCurrentProfiles") ret = fn(device, currentProfiles) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceWorkloadPowerProfileSetRequestedProfiles(device, requestedProfiles): fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileSetRequestedProfiles") ret = fn(device, requestedProfiles) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceWorkloadPowerProfileClearRequestedProfiles(device, requestedProfiles): fn = _nvmlGetFunctionPointer("nvmlDeviceWorkloadPowerProfileClearRequestedProfiles") ret = fn(device, requestedProfiles) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetNvlinkSupportedBwModes(device, supportedBwModes): fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvlinkSupportedBwModes") ret = fn(device, supportedBwModes) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceGetNvlinkBwMode(device, getBwMode): fn = _nvmlGetFunctionPointer("nvmlDeviceGetNvlinkBwMode") ret = fn(device, getBwMode) _nvmlCheckReturn(ret) return NVML_SUCCESS + def nvmlDeviceSetNvlinkBwMode(device, setBwMode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetNvlinkBwMode") ret = fn(device, setBwMode) _nvmlCheckReturn(ret) return NVML_SUCCESS + nvmlDramEncryptionInfo_v1 = 0x01000008 + class c_nvmlDramEncryptionInfo_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('encryptionState', _nvmlEnableState_t), + ("version", c_uint), + ("encryptionState", _nvmlEnableState_t), ] def __init__(self): - super(c_nvmlDramEncryptionInfo_t, self).__init__(version=nvmlDramEncryptionInfo_v1) + super(c_nvmlDramEncryptionInfo_t, self).__init__( + version=nvmlDramEncryptionInfo_v1 + ) + def nvmlDeviceGetDramEncryptionMode(handle): c_currState = c_nvmlDramEncryptionInfo_t() @@ -6077,64 +7148,79 @@ def nvmlDeviceGetDramEncryptionMode(handle): _nvmlCheckReturn(ret) return [c_currState.encryptionState, c_pendingState.encryptionState] + # added to API def nvmlDeviceGetCurrentDramEncryptionMode(handle): return nvmlDeviceGetDramEncryptionMode(handle)[0] + # added to API def nvmlDeviceGetPendingDramEncryptionMode(handle): return nvmlDeviceGetDramEncryptionMode(handle)[1] + def nvmlDeviceSetDramEncryptionMode(handle, mode): fn = _nvmlGetFunctionPointer("nvmlDeviceSetDramEncryptionMode") c_dramEncryptionMode = c_nvmlDramEncryptionInfo_t() - c_dramEncryptionMode.encryptionState = mode; + c_dramEncryptionMode.encryptionState = mode ret = fn(handle, byref(c_dramEncryptionMode)) _nvmlCheckReturn(ret) return None + # Power Smoothing defines -NVML_POWER_SMOOTHING_MAX_NUM_PROFILES = 5 -NVML_POWER_SMOOTHING_ADMIN_OVERRIDE_NOT_SET = 0xFFFFFFFF -NVML_POWER_SMOOTHING_PROFILE_PARAM_PERCENT_TMP_FLOOR = 0 -NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_UP_RATE = 1 -NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_RATE = 2 +NVML_POWER_SMOOTHING_MAX_NUM_PROFILES = 5 +NVML_POWER_SMOOTHING_ADMIN_OVERRIDE_NOT_SET = 0xFFFFFFFF +NVML_POWER_SMOOTHING_PROFILE_PARAM_PERCENT_TMP_FLOOR = 0 +NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_UP_RATE = 1 +NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_RATE = 2 NVML_POWER_SMOOTHING_PROFILE_PARAM_RAMP_DOWN_HYSTERESIS = 3 -nvmlPowerSmoothingState_v1=0x1000008 +nvmlPowerSmoothingState_v1 = 0x1000008 + + class c_nvmlPowerSmoothingState_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('state', c_uint), + ("version", c_uint), + ("state", c_uint), ] def __init__(self): - super(c_nvmlPowerSmoothingState_v1_t, self).__init__(version=nvmlPowerSmoothingState_v1) + super(c_nvmlPowerSmoothingState_v1_t, self).__init__( + version=nvmlPowerSmoothingState_v1 + ) + + +nvmlPowerSmoothingProfile_v1 = 0x1000018 + -nvmlPowerSmoothingProfile_v1=0x1000018 class c_nvmlPowerSmoothingProfile_v1_t(_PrintableStructure): _fields_ = [ - ('version', c_uint), - ('profileId', c_uint), - ('paramId', c_uint), - ('value', c_double), + ("version", c_uint), + ("profileId", c_uint), + ("paramId", c_uint), + ("value", c_double), ] def __init__(self): - super(c_nvmlPowerSmoothingProfile_v1_t, self).__init__(version=nvmlPowerSmoothingProfile_v1) + super(c_nvmlPowerSmoothingProfile_v1_t, self).__init__( + version=nvmlPowerSmoothingProfile_v1 + ) + def nvmlDevicePowerSmoothingActivatePresetProfile(device, profile): fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingActivatePresetProfile") ret = fn(device, profile) _nvmlCheckReturn(ret) + def nvmlDevicePowerSmoothingUpdatePresetProfileParam(device, profile): fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingUpdatePresetProfileParam") ret = fn(device, profile) _nvmlCheckReturn(ret) + def nvmlDevicePowerSmoothingSetState(device, state): fn = _nvmlGetFunctionPointer("nvmlDevicePowerSmoothingSetState") ret = fn(device, state) _nvmlCheckReturn(ret) - diff --git a/vllm/tilelang_utils/__init__.py b/vllm/tilelang_utils/__init__.py index 803409b1ec..990c2fca3e 100644 --- a/vllm/tilelang_utils/__init__.py +++ b/vllm/tilelang_utils/__init__.py @@ -31,6 +31,7 @@ def _ensure_tilelang_imported() -> None: Raises: ImportError: If TileLang is not installed. + """ global T, tilelang diff --git a/vllm/tokenizers/deepseek_v32.py b/vllm/tokenizers/deepseek_v32.py index 0e9aaa2cae..3f0fff938a 100644 --- a/vllm/tokenizers/deepseek_v32.py +++ b/vllm/tokenizers/deepseek_v32.py @@ -13,9 +13,7 @@ from .protocol import TokenizerLike def get_deepseek_v32_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: - """ - Wraps a tokenizer to use the custom DeepSeek V3.2 chat template encoding. - """ + """Wraps a tokenizer to use the custom DeepSeek V3.2 chat template encoding.""" dsv32_tokenizer = copy.copy(tokenizer) added_vocab = tokenizer.get_added_vocab() diff --git a/vllm/tokenizers/deepseek_v4.py b/vllm/tokenizers/deepseek_v4.py index 2a0db215b8..a5d5020e94 100644 --- a/vllm/tokenizers/deepseek_v4.py +++ b/vllm/tokenizers/deepseek_v4.py @@ -13,9 +13,7 @@ from .protocol import TokenizerLike def get_deepseek_v4_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: - """ - Wraps a tokenizer to use the custom DeepSeek V4 chat template encoding. - """ + """Wraps a tokenizer to use the custom DeepSeek V4 chat template encoding.""" dsv4_tokenizer = copy.copy(tokenizer) added_vocab = tokenizer.get_added_vocab() diff --git a/vllm/tokenizers/detokenizer_utils.py b/vllm/tokenizers/detokenizer_utils.py index 516f1b70e7..35e73bc410 100644 --- a/vllm/tokenizers/detokenizer_utils.py +++ b/vllm/tokenizers/detokenizer_utils.py @@ -205,6 +205,7 @@ def detokenize_incrementally( skip_special_tokens: Whether to skip special tokens. spaces_between_special_tokens: Whether to add spaces between special tokens. + """ new_token_id = all_input_ids[-1] # This is the first iteration for this sequence diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index 3d35952c5f..a091d8e105 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -23,8 +23,7 @@ class ThreadSafeHFTokenizerMixin: def maybe_make_thread_pool(tokenizer: _T, copies: int = 1): - """ - If `tokenizer` is a `TokenizersBackend`, modify the tokenizer + """If `tokenizer` is a `TokenizersBackend`, modify the tokenizer in-place to make the public interface thread-safe by routing calls through a deep-copied tokenizer pool. @@ -105,8 +104,7 @@ def maybe_make_thread_pool(tokenizer: _T, copies: int = 1): def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: - """ - By default, transformers will recompute multiple tokenizer properties + """By default, transformers will recompute multiple tokenizer properties each time they are called, leading to a significant slowdown. This proxy caches these properties for faster access. """ diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 3ee35365e4..3847afa9b2 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -40,8 +40,7 @@ logger = init_logger(__name__) class ToolParser: - """ - Abstract ToolParser class that should not be used directly. Provided + """Abstract ToolParser class that should not be used directly. Provided properties and methods should be used in derived classes. """ @@ -186,8 +185,7 @@ class ToolParser: def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Static method that should be implemented for extracting tool calls from + """Static method that should be implemented for extracting tool calls from a complete model-generated string. Used for non-streaming responses where we have the entire model response available before sending to the client. @@ -207,8 +205,7 @@ class ToolParser: delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> DeltaMessage | None: - """ - Instance method that should be implemented for extracting tool calls + """Instance method that should be implemented for extracting tool calls from an incomplete response; for use when handling tool calls and streaming. Has to be an instance method because it requires state - the current tokens/diffs, but also the information about what has @@ -220,8 +217,7 @@ class ToolParser: class ToolParserManager: - """ - Central registry for ToolParser implementations. + """Central registry for ToolParser implementations. Supports two modes: - Eager (immediate) registration via `register_module` @@ -233,8 +229,7 @@ class ToolParserManager: @classmethod def get_tool_parser(cls, name: str) -> type[ToolParser]: - """ - Retrieve a registered or lazily registered ToolParser class. + """Retrieve a registered or lazily registered ToolParser class. If the parser is lazily registered, it will be imported and cached on first access. @@ -301,8 +296,7 @@ class ToolParserManager: @classmethod def register_lazy_module(cls, name: str, module_path: str, class_name: str) -> None: - """ - Register a lazy module mapping. + """Register a lazy module mapping. Example: ToolParserManager.register_lazy_module( @@ -310,6 +304,7 @@ class ToolParserManager: module_path="vllm.tool_parsers.kimi_k2_parser", class_name="KimiK2ToolParser", ) + """ cls.lazy_parsers[name] = (module_path, class_name) @@ -320,8 +315,7 @@ class ToolParserManager: force: bool = True, module: type[ToolParser] | None = None, ) -> type[ToolParser] | Callable[[type[ToolParser]], type[ToolParser]]: - """ - Register module immediately or lazily (as a decorator). + """Register module immediately or lazily (as a decorator). Usage: @ToolParserManager.register_module("kimi_k2") diff --git a/vllm/tool_parsers/apertus_tool_parser.py b/vllm/tool_parsers/apertus_tool_parser.py index f23b8601aa..6825b3ef39 100644 --- a/vllm/tool_parsers/apertus_tool_parser.py +++ b/vllm/tool_parsers/apertus_tool_parser.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Tool call parser for Apertus models. +"""Tool call parser for Apertus models. Extracts tool calls from the format: <|tools_prefix|>[{"function_name": {"arg1": "value1", ...}}, ...]<|tools_suffix|> @@ -45,8 +44,7 @@ TOOL_CALLS_SUFFIX = "<|tools_suffix|>" class ApertusToolParser(ToolParser): - """ - Tool call parser for Apertus models. + """Tool call parser for Apertus models. Handles the extraction of tool calls from text in both non-streaming (complete string) and streaming (chunked token) environments. @@ -68,11 +66,11 @@ class ApertusToolParser(ToolParser): "get_weather" >>> info.tool_calls[0].function.arguments '{"city": "Paris"}' + """ def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - """ - Initializes the ApertusToolParser. + """Initializes the ApertusToolParser. Args: tokenizer: The model's tokenizer. @@ -82,6 +80,7 @@ class ApertusToolParser(ToolParser): Raises: ValueError: If the `model_tokenizer` is not successfully passed to the base class. + """ super().__init__(tokenizer, tools) @@ -101,8 +100,7 @@ class ApertusToolParser(ToolParser): self._reset_streaming_state() def _reset_streaming_state(self) -> None: - """ - Resets all streaming state variables for a new completion request. + """Resets all streaming state variables for a new completion request. This clears the delta text buffer and resets the pointers used to track the currently streaming tool index and arguments. Called implicitly @@ -116,8 +114,7 @@ class ApertusToolParser(ToolParser): def adjust_request( self, request: ChatCompletionRequest | ResponsesRequest ) -> ChatCompletionRequest | ResponsesRequest: - """ - Adjusts the generation request to ensure special tool tokens are not skipped. + """Adjusts the generation request to ensure special tool tokens are not skipped. Forces `skip_special_tokens=False` if tools are actively being evaluated, ensuring the tools special tokens are surfaced to the engine for parsing. @@ -127,6 +124,7 @@ class ApertusToolParser(ToolParser): Returns: The potentially modified chat completion request. + """ request = super().adjust_request(request) if request.tools and request.tool_choice != "none": @@ -134,8 +132,7 @@ class ApertusToolParser(ToolParser): return request def _buffer_delta_text(self, delta_text: str) -> str: - """ - Buffers incoming delta chunks to prevent + """Buffers incoming delta chunks to prevent fragmentation of multi-token special tags. If a chunk ends with a partial match of @@ -155,6 +152,7 @@ class ApertusToolParser(ToolParser): "Let me check " # "<|tool" is buffered internally >>> parser._buffer_delta_text("s_prefix|>" \ "<|tools_prefix|>" # Buffer released on completion + """ self.buffered_delta_text += delta_text text = self.buffered_delta_text @@ -178,8 +176,7 @@ class ApertusToolParser(ToolParser): model_output: str, request: ChatCompletionRequest, ) -> ExtractedToolCallInformation: - """ - Extracts tool calls from a completely generated model response (Non-Streaming). + """Extract tool calls from a complete model response (non-streaming). Args: model_output: The full completion string generated by the model. @@ -200,6 +197,7 @@ class ApertusToolParser(ToolParser): 'Let me see.' >>> info.tool_calls[0].function.name 'get_weather' + """ match = self.tool_call_regex.search(model_output) if not match: @@ -280,8 +278,7 @@ class ApertusToolParser(ToolParser): delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> DeltaMessage | None: - """ - Handles streaming chunks + """Handles streaming chunks. Args: previous_text: The complete model text generated prior to this chunk. @@ -305,6 +302,7 @@ class ApertusToolParser(ToolParser): ... ) >>> msg.tool_calls[0].function.arguments 'ation": "Paris"}' + """ delta_text = self._buffer_delta_text(delta_text) if not delta_text: @@ -323,8 +321,7 @@ class ApertusToolParser(ToolParser): def _extract_streaming( self, current_text: str, delta_text: str ) -> DeltaMessage | None: - """ - Core streaming logic. + """Core streaming logic. Separates visible chat text from JSON blocks and computes diffs. Args: @@ -333,6 +330,7 @@ class ApertusToolParser(ToolParser): Returns: A `DeltaMessage` containing the `content` delta and/or `tool_calls` delta. + """ prefix_idx = current_text.rfind(TOOL_CALLS_PREFIX) suffix_idx = current_text.rfind(TOOL_CALLS_SUFFIX) @@ -404,8 +402,7 @@ class ApertusToolParser(ToolParser): def _parse_and_diff_json( self, json_str: str, is_final: bool ) -> list[DeltaToolCall]: - """ - Parses an isolated, potentially incomplete streaming JSON array and returns + """Parses an isolated, potentially incomplete streaming JSON array and returns newly accumulated tool call diffs. Args: @@ -417,6 +414,7 @@ class ApertusToolParser(ToolParser): A list of `DeltaToolCall` items representing string diffs in function arguments to stream back to the client. + """ try: parsed, _ = partial_json_loads(json_str, Allow.ALL) @@ -460,8 +458,7 @@ class ApertusToolParser(ToolParser): def _emit_tool_name( self, parsed: list, index: int, tool_calls: list[DeltaToolCall] ) -> None: - """ - Extracts and emits the function name mapped to a new tool call ID. + """Extracts and emits the function name mapped to a new tool call ID. Args: parsed: The partially parsed JSON list containing tool dictionaries. @@ -472,6 +469,7 @@ class ApertusToolParser(ToolParser): Appends `DeltaToolCall(index=0, function=DeltaFunctionCall(name="get_weather", ...))` to the `tool_calls` list and marks the name as sent. + """ obj = parsed[index] if isinstance(obj, dict) and obj: @@ -491,8 +489,7 @@ class ApertusToolParser(ToolParser): def _get_tool_diff( self, parsed: list, index: int, is_final: bool ) -> DeltaToolCall | None: - """ - Calculates the exact string difference to safely append new tool parameters. + """Calculates the exact string difference to safely append new tool parameters. This ensures characters like `{`, `}`, and `"` don't jump around unevenly in the UI frontend while streaming incomplete JSON arguments. @@ -513,6 +510,7 @@ class ApertusToolParser(ToolParser): >>> # Returns diff (closing bracket suppressed until final): >>> parser._get_tool_diff(parsed, index=0, is_final=False) DeltaToolCall(index=0, function=DeltaFunctionCall(arguments='s')) + """ obj = parsed[index] if not isinstance(obj, dict) or not obj: diff --git a/vllm/tool_parsers/ernie45_tool_parser.py b/vllm/tool_parsers/ernie45_tool_parser.py index 74624b1404..3c654b4489 100644 --- a/vllm/tool_parsers/ernie45_tool_parser.py +++ b/vllm/tool_parsers/ernie45_tool_parser.py @@ -29,8 +29,7 @@ logger = init_logger(__name__) class Ernie45ToolParser(ToolParser): def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - """ - Ernie thinking model format: + """Ernie thinking model format: abc\n\n\n\n\ndef\n\n """ super().__init__(tokenizer, tools) diff --git a/vllm/tool_parsers/functiongemma_tool_parser.py b/vllm/tool_parsers/functiongemma_tool_parser.py index a975f68368..708ecb1dd9 100644 --- a/vllm/tool_parsers/functiongemma_tool_parser.py +++ b/vllm/tool_parsers/functiongemma_tool_parser.py @@ -27,8 +27,7 @@ logger = init_logger(__name__) class FunctionGemmaToolParser(ToolParser): - """ - Tool parser for Google's FunctionGemma model (google/functiongemma-270m-it). + """Tool parser for Google's FunctionGemma model (google/functiongemma-270m-it). Handles the FunctionGemma function call format: call:func_name{param:value} diff --git a/vllm/tool_parsers/gemma4_utils.py b/vllm/tool_parsers/gemma4_utils.py index d9aad254c7..33ad20b193 100644 --- a/vllm/tool_parsers/gemma4_utils.py +++ b/vllm/tool_parsers/gemma4_utils.py @@ -52,6 +52,7 @@ def _parse_tool_arguments(args_str: str) -> dict[str, str]: Returns: Dictionary of argument name → string value. + """ if not args_str or not args_str.strip(): return {} @@ -95,6 +96,7 @@ def parse_tool_calls(text: str, *, strict: bool = False) -> list[dict]: >>> tool_calls = parse_tool_calls(output) >>> for tc in tool_calls: ... print(f"Call: {tc['name']}({tc['arguments']})") + """ results = [] @@ -150,6 +152,7 @@ def has_tool_response_tag(text: str) -> bool: >>> if not has_tool_response_tag(model_output): ... # Model used instead — inject <|tool_response> manually ... next_prompt = "<|tool_response>" + tool_result + """ stripped = text.rstrip() return stripped.endswith(_TOOL_RESPONSE_START_TAG) diff --git a/vllm/tool_parsers/gptoss_tool_parser.py b/vllm/tool_parsers/gptoss_tool_parser.py index e147ff1c74..80881f69a5 100644 --- a/vllm/tool_parsers/gptoss_tool_parser.py +++ b/vllm/tool_parsers/gptoss_tool_parser.py @@ -15,8 +15,7 @@ if TYPE_CHECKING: class GptOssToolParser(ToolParser): - """ - Stub tool parser for gpt-oss/harmony models. + """Stub tool parser for gpt-oss/harmony models. All output parsing is handled by HarmonyParser. This stub exists as a capability declaration via HarmonyParser.tool_parser_cls. diff --git a/vllm/tool_parsers/granite_20b_fc_tool_parser.py b/vllm/tool_parsers/granite_20b_fc_tool_parser.py index 432f94b61e..4c6c0a1b0e 100644 --- a/vllm/tool_parsers/granite_20b_fc_tool_parser.py +++ b/vllm/tool_parsers/granite_20b_fc_tool_parser.py @@ -38,8 +38,7 @@ logger = init_logger(__name__) class Granite20bFCToolParser(ToolParser): - """ - Tool call parser for the granite-20b-functioncalling model intended + """Tool call parser for the granite-20b-functioncalling model intended for use with the examples/tool_chat_template_granite20b_fc.jinja template. diff --git a/vllm/tool_parsers/granite_tool_parser.py b/vllm/tool_parsers/granite_tool_parser.py index c1d852dfc3..1e51c3227e 100644 --- a/vllm/tool_parsers/granite_tool_parser.py +++ b/vllm/tool_parsers/granite_tool_parser.py @@ -36,8 +36,7 @@ logger = init_logger(__name__) class GraniteToolParser(ToolParser): - """ - Tool call parser for the granite 3.0 models. Intended + """Tool call parser for the granite 3.0 models. Intended for use with the examples/tool_chat_template_granite.jinja template. diff --git a/vllm/tool_parsers/hunyuan_a13b_tool_parser.py b/vllm/tool_parsers/hunyuan_a13b_tool_parser.py index 88b2a20d3a..90e291084f 100644 --- a/vllm/tool_parsers/hunyuan_a13b_tool_parser.py +++ b/vllm/tool_parsers/hunyuan_a13b_tool_parser.py @@ -100,9 +100,7 @@ class HunyuanA13BToolParser(ToolParser): def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Extract tool calls from a complete model output. - """ + """Extract tool calls from a complete model output.""" try: # Preprocess the model output content, potential_tool_calls = self.preprocess_model_output(model_output) @@ -177,10 +175,7 @@ class HunyuanA13BToolParser(ToolParser): delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> DeltaMessage | None: - """ - Extract tool calls for streaming mode. - """ - + """Extract tool calls for streaming mode.""" start_idx = consume_space(0, current_text) if current_text[start_idx:].startswith(self.bot_string): start_idx = consume_space(start_idx + len(self.bot_string), current_text) diff --git a/vllm/tool_parsers/hy_v3_tool_parser.py b/vllm/tool_parsers/hy_v3_tool_parser.py index d73473673f..0effb377d0 100644 --- a/vllm/tool_parsers/hy_v3_tool_parser.py +++ b/vllm/tool_parsers/hy_v3_tool_parser.py @@ -181,6 +181,7 @@ class HYV3ToolParser(ToolParser): "-3" → int(-3) Return None on failure. + """ try: if "." in value or "e" in value or "E" in value: diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py index 7294631626..c2f82fc4a7 100644 --- a/vllm/tool_parsers/hy_v4_tool_parser.py +++ b/vllm/tool_parsers/hy_v4_tool_parser.py @@ -101,8 +101,8 @@ def detect_token_suffix(tokenizer: TokenizerLike) -> str: RuntimeError: The tokenizer declares the structural tokens through ``model_specific_special_tokens``, which transformers 5 no longer round-trips. - """ + """ import transformers if int(transformers.__version__.split(".")[0]) >= 5: @@ -611,6 +611,7 @@ class HYV4ToolExtractor: Returns: A streaming delta carrying content and/or the tool calls drained from the buffer, or None when nothing can be emitted yet. + """ content_delta: str | None = None tool_calls: list[StreamToolCall] = [] @@ -1011,6 +1012,7 @@ class HYV4ToolParser(ToolParser): Returns: The structural tag, or None when structural tagging does not apply. + """ if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING: return None @@ -1070,6 +1072,7 @@ class HYV4ToolParser(ToolParser): Returns: True when the streaming parser must use the string-marker path. + """ structured_outputs = getattr(request, "structured_outputs", None) return ( diff --git a/vllm/tool_parsers/lfm2_tool_parser.py b/vllm/tool_parsers/lfm2_tool_parser.py index 6f4db492be..2a0bab07e2 100644 --- a/vllm/tool_parsers/lfm2_tool_parser.py +++ b/vllm/tool_parsers/lfm2_tool_parser.py @@ -42,8 +42,7 @@ TOOL_CALL_END = "<|tool_call_end|>" class Lfm2ToolParser(ToolParser): - """ - Tool call parser for LiquidAI LFM2/LFM2.5 models that produce pythonic + """Tool call parser for LiquidAI LFM2/LFM2.5 models that produce pythonic tool calls wrapped in <|tool_call_start|> and <|tool_call_end|> tokens. Example model output: diff --git a/vllm/tool_parsers/llama4_pythonic_tool_parser.py b/vllm/tool_parsers/llama4_pythonic_tool_parser.py index 18c697f048..fdffb48ff7 100644 --- a/vllm/tool_parsers/llama4_pythonic_tool_parser.py +++ b/vllm/tool_parsers/llama4_pythonic_tool_parser.py @@ -31,8 +31,7 @@ logger = init_logger(__name__) class Llama4PythonicToolParser(ToolParser): - """ - Toolcall parser for Llama4 that produce tool calls in a pythonic style + """Toolcall parser for Llama4 that produce tool calls in a pythonic style Use --enable-auto-tool-choice --tool-call-parser llama4_pythonic """ @@ -67,10 +66,7 @@ class Llama4PythonicToolParser(ToolParser): def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Extract the tool calls from a complete model response. - """ - + """Extract the tool calls from a complete model response.""" # remove <|python_start|> and <|python_end|> # as Llama 4 model sometime will output those tokens if model_output.startswith("<|python_start|>"): diff --git a/vllm/tool_parsers/llama_tool_parser.py b/vllm/tool_parsers/llama_tool_parser.py index 5ad7b1cbdd..93b0980c37 100644 --- a/vllm/tool_parsers/llama_tool_parser.py +++ b/vllm/tool_parsers/llama_tool_parser.py @@ -37,8 +37,7 @@ logger = init_logger(__name__) class Llama3JsonToolParser(ToolParser): - """ - Tool call parser for Llama 3.x and 4 models intended for use with the + """Tool call parser for Llama 3.x and 4 models intended for use with the examples/tool_chat_template_llama.jinja template. Used when --enable-auto-tool-choice --tool-call-parser llama3_json or @@ -77,8 +76,7 @@ class Llama3JsonToolParser(ToolParser): def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Extract the tool calls from a complete model response. + """Extract the tool calls from a complete model response. Only extracts JSON content and ignores any surrounding plain text. Supports both single JSON and multiple JSONs separated by semicolons. """ diff --git a/vllm/tool_parsers/olmo3_tool_parser.py b/vllm/tool_parsers/olmo3_tool_parser.py index 49a8716533..cb7bc632ba 100644 --- a/vllm/tool_parsers/olmo3_tool_parser.py +++ b/vllm/tool_parsers/olmo3_tool_parser.py @@ -31,8 +31,7 @@ logger = init_logger(__name__) class Olmo3PythonicToolParser(ToolParser): - """ - Tool call parser for Olmo 3 models that produce tool calls as + """Tool call parser for Olmo 3 models that produce tool calls as newline-separated pythonic strings. Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set Code copied from pythonic_tool_parser.py and updated to handle @@ -71,9 +70,7 @@ class Olmo3PythonicToolParser(ToolParser): def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Extract the tool calls from a complete model response. - """ + """Extract the tool calls from a complete model response.""" original_model_output = model_output # Remove xml tags. match = re.search( diff --git a/vllm/tool_parsers/phi4mini_tool_parser.py b/vllm/tool_parsers/phi4mini_tool_parser.py index f5b794f0f6..820e797f4d 100644 --- a/vllm/tool_parsers/phi4mini_tool_parser.py +++ b/vllm/tool_parsers/phi4mini_tool_parser.py @@ -28,8 +28,7 @@ logger = init_logger(__name__) class Phi4MiniJsonToolParser(ToolParser): - """ - Tool call parser for phi-4-mini models intended for use with the + """Tool call parser for phi-4-mini models intended for use with the examples/tool_chat_template_llama.jinja template. Used when --enable-auto-tool-choice --tool-call-parser phi4_mini_json @@ -55,9 +54,7 @@ class Phi4MiniJsonToolParser(ToolParser): def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Extract the tool calls from a complete model response. - """ + """Extract the tool calls from a complete model response.""" logger.debug("Model output: %s", model_output) pattern = r"functools\[(.*?)\]" diff --git a/vllm/tool_parsers/poolside_v1_tool_parser.py b/vllm/tool_parsers/poolside_v1_tool_parser.py index 693580e85a..1780367606 100644 --- a/vllm/tool_parsers/poolside_v1_tool_parser.py +++ b/vllm/tool_parsers/poolside_v1_tool_parser.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GLM-4 Tool Call Parser with incremental string streaming support. +"""GLM-4 Tool Call Parser with incremental string streaming support. This parser fixes the streaming issue reported in Issue #32829 where long string parameters (e.g., file content with 4000+ characters of code) are buffered until diff --git a/vllm/tool_parsers/pythonic_tool_parser.py b/vllm/tool_parsers/pythonic_tool_parser.py index cc584d3d7c..76b263cfc5 100644 --- a/vllm/tool_parsers/pythonic_tool_parser.py +++ b/vllm/tool_parsers/pythonic_tool_parser.py @@ -31,8 +31,7 @@ logger = init_logger(__name__) class PythonicToolParser(ToolParser): - """ - Tool call parser for models that produce tool calls in a pythonic style, + """Tool call parser for models that produce tool calls in a pythonic style, such as Llama 3.2 and Llama 4 models. Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set @@ -69,9 +68,7 @@ class PythonicToolParser(ToolParser): def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: - """ - Extract the tool calls from a complete model response. - """ + """Extract the tool calls from a complete model response.""" is_tool_call_pattern = False try: is_tool_call_pattern = ( diff --git a/vllm/tool_parsers/step3_tool_parser.py b/vllm/tool_parsers/step3_tool_parser.py index 929e0c19c6..514945cd0d 100644 --- a/vllm/tool_parsers/step3_tool_parser.py +++ b/vllm/tool_parsers/step3_tool_parser.py @@ -32,8 +32,7 @@ logger = init_logger(__name__) class Step3ToolParser(ToolParser): - """ - Tool parser for a model that uses a specific XML-like format for tool calls. + """Tool parser for a model that uses a specific XML-like format for tool calls. This version uses a robust, stateful, cursor-based streaming parser and consolidates tool arguments into a single message. """ diff --git a/vllm/tool_parsers/step3p5_tool_parser.py b/vllm/tool_parsers/step3p5_tool_parser.py index 74fcb56ec4..2639d2f204 100644 --- a/vllm/tool_parsers/step3p5_tool_parser.py +++ b/vllm/tool_parsers/step3p5_tool_parser.py @@ -28,8 +28,7 @@ logger = init_logger(__name__) class StreamingXMLToolCallParser: - """ - Simplified streaming XML tool call parser + """Simplified streaming XML tool call parser Supports streaming input, parsing, and output """ @@ -46,8 +45,7 @@ class StreamingXMLToolCallParser: self.parameter_end_token: str = "" def reset_streaming_state(self): - """Reset streaming parsing state""" - + """Reset streaming parsing state.""" self.deltas = [] # state for streaming self.tool_call_index = 0 @@ -80,8 +78,7 @@ class StreamingXMLToolCallParser: self.setup_parser() def parse_single_streaming_chunks(self, xml_chunk: str) -> DeltaMessage: - """ - Parse single streaming XML chunk and return Delta response + """Parse single streaming XML chunk and return Delta response This is the actual streaming interface that receives chunks one by one and maintains internal state @@ -90,6 +87,7 @@ class StreamingXMLToolCallParser: Returns: DeltaMessage: Contains delta information generated by this chunk, returns empty response if no complete elements + """ # Record delta count before processing initial_delta_count = len(self.deltas) @@ -218,8 +216,7 @@ class StreamingXMLToolCallParser: return DeltaMessage(content=None) def _escape_xml_special_chars(self, text: str) -> str: - """ - Escape XML special characters + """Escape XML special characters Args: text: Original text Returns: @@ -239,11 +236,11 @@ class StreamingXMLToolCallParser: return text def _process_complete_xml_elements(self) -> bool: - """ - Process complete XML elements in buffer + """Process complete XML elements in buffer. Returns: bool: Whether complete elements were found and processed + """ found_any = False @@ -320,8 +317,7 @@ class StreamingXMLToolCallParser: return found_any def _fix_incomplete_tag_in_chunk(self, chunk: str) -> str: - """ - Fallback: fix incomplete ) Examples: , @@ -386,8 +382,7 @@ class StreamingXMLToolCallParser: return chunk def _fix_missing_equals_in_function_tag(self, chunk: str) -> str: - """ - Fix missing = in function tags: or + """Fix missing = in function tags: or Examples: -> -> @@ -425,7 +420,7 @@ class StreamingXMLToolCallParser: return chunk def _validate_function_name(self, func_name: str) -> bool: - """Check if function name exists in tool definitions""" + """Check if function name exists in tool definitions.""" if not self.tools: return False @@ -442,7 +437,7 @@ class StreamingXMLToolCallParser: return False def _validate_parameter_name(self, param_name: str) -> bool: - """Check if parameter exists in current function's tool definition""" + """Check if parameter exists in current function's tool definition.""" if not self.tools or not self.current_function_name: return True @@ -465,16 +460,15 @@ class StreamingXMLToolCallParser: return True def _should_skip_element(self, element: str) -> bool: - """ - Determine whether an element should be skipped + """Determine whether an element should be skipped. Args: element: Element to evaluate Returns: bool: True means should skip, False means should process - """ + """ # If it's a tool_call XML tag, don't skip if ( element.startswith(self.tool_call_start_token) @@ -501,8 +495,7 @@ class StreamingXMLToolCallParser: return not element def _find_next_complete_element(self, start_pos: int) -> tuple[str | None, int]: - """ - Find next complete XML element from specified position + """Find next complete XML element from specified position. Args: start_pos: Position to start searching @@ -510,6 +503,7 @@ class StreamingXMLToolCallParser: Returns: (Complete element string, element end position), returns (None, start_pos) if no complete element found + """ buffer = self.streaming_buffer[start_pos:] @@ -588,8 +582,7 @@ class StreamingXMLToolCallParser: return remaining, start_pos + len(remaining) def _merge_new_deltas_to_single_response(self, initial_count: int) -> DeltaMessage: - """ - Merge newly generated deltas from this processing + """Merge newly generated deltas from this processing into a single DeltaMessage Args: @@ -597,6 +590,7 @@ class StreamingXMLToolCallParser: Returns: Merged DeltaMessage containing all newly generated delta information + """ if len(self.deltas) <= initial_count: return DeltaMessage(content=None) @@ -652,8 +646,7 @@ class StreamingXMLToolCallParser: ) def _preprocess_xml_chunk(self, chunk: str) -> str: - """ - Preprocess XML chunk, handle non-standard formats, + """Preprocess XML chunk, handle non-standard formats, and escape special characters Args: @@ -661,8 +654,8 @@ class StreamingXMLToolCallParser: Returns: Processed XML chunk - """ + """ # Check if this is a tool_call related element is_tool_call = False if chunk.startswith(self.tool_call_start_token) or chunk.startswith( @@ -785,7 +778,7 @@ class StreamingXMLToolCallParser: return processed def _emit_delta(self, delta: DeltaMessage): - """Emit Delta response (streaming output)""" + """Emit Delta response (streaming output).""" self.deltas.append(delta) def _auto_close_open_parameter_if_needed(self, incoming_tag: str | None = None): @@ -814,8 +807,7 @@ class StreamingXMLToolCallParser: self._end_element("tool_call") def _start_element(self, name: str, attrs: dict[str, str]): - """Handle XML start element events""" - + """Handle XML start element events.""" if name == "root": return @@ -904,7 +896,7 @@ class StreamingXMLToolCallParser: self.current_param_is_first = False def _char_data(self, data: str): - """Handle XML character data events""" + """Handle XML character data events.""" if data and self.current_param_name: # If preprocessing stage determines deferred parsing is needed, # only cache character data, no streaming output @@ -979,8 +971,7 @@ class StreamingXMLToolCallParser: self._emit_delta(delta) def _end_element(self, name: str): - """Handle XML end element events""" - + """Handle XML end element events.""" if name == "root": return @@ -1154,18 +1145,18 @@ class StreamingXMLToolCallParser: self._reset_xml_parser_after_tool_call() def setup_parser(self): - """Set up XML parser event handlers""" + """Set up XML parser event handlers.""" self.parser.buffer_text = True self.parser.StartElementHandler = self._start_element self.parser.EndElementHandler = self._end_element self.parser.CharacterDataHandler = self._char_data def set_tools(self, tools: list[Tool] | None): - """Set tool configuration information""" + """Set tool configuration information.""" self.tools = tools def _extract_function_name(self, name: str, attrs: dict[str, str]) -> str | None: - """Extract function name from various formats""" + """Extract function name from various formats.""" if attrs and "name" in attrs: return attrs["name"] @@ -1177,7 +1168,7 @@ class StreamingXMLToolCallParser: return None def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> str | None: - """Extract parameter name from various formats""" + """Extract parameter name from various formats.""" if attrs and "name" in attrs: return attrs["name"] @@ -1195,6 +1186,7 @@ class StreamingXMLToolCallParser: Returns: Parameter type + """ if not self.tools or not self.current_function_name: return "string" @@ -1235,6 +1227,7 @@ class StreamingXMLToolCallParser: Returns: Repaired parameter type + """ if ( param_type in ["string", "str", "text", "varchar", "char", "enum"] @@ -1264,6 +1257,7 @@ class StreamingXMLToolCallParser: Returns: Converted value + """ if param_value.lower() == "null": return None @@ -1315,6 +1309,7 @@ class StreamingXMLToolCallParser: Returns: Converted string for streaming output + """ # Check if value is empty, but exclude numeric 0 if converted_value is None or converted_value == "": @@ -1331,11 +1326,9 @@ class StreamingXMLToolCallParser: return converted_value def _reset_xml_parser_after_tool_call(self): - """ - Each tool_call is treated as a separate XML document, + """Each tool_call is treated as a separate XML document, so we need to reset the parser after each tool_call. """ - # recreate XML parser self.parser = ParserCreate() self.setup_parser() @@ -1513,7 +1506,5 @@ class Step3p5ToolParser(ToolParser): return result def parser_should_check_for_unstreamed_tool_arg_tokens(self) -> bool: - """ - Skip the remaining_call calculation in serving_chat - """ + """Skip the remaining_call calculation in serving_chat.""" return False diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 989250e2ea..295d5844b4 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -113,7 +113,6 @@ def get_model_structural_tag( token_suffix: str = "", ) -> StructuralTag | None: """Build a structural tag with xgrammar's builtin model templates.""" - if not tools or tool_choice == "none": return None @@ -158,7 +157,6 @@ def _dump_tool_for_xgrammar( tool: ChatCompletionToolsParam | ResponsesTool, ) -> dict[str, Any]: """Convert tool objects to xgrammar's Chat Completions tool protocol.""" - if isinstance(tool, FunctionTool): function: dict[str, Any] = {"name": tool.name} if tool.description is not None: @@ -178,7 +176,6 @@ def _dump_tool_choice_for_xgrammar( tool_choice: ToolChoice, ) -> dict[str, Any] | str | None: """Convert tool_choice objects to xgrammar's expected protocol.""" - if tool_choice is None: return None @@ -783,6 +780,7 @@ def get_hy_v4_structural_tag( leading colon (e.g. ``":6124c78e"``), or ``""`` when the checkpoint uses unsuffixed tokens. The HYV4 tool parser reads it off the tokenizer vocab and passes it to ``get_model_structural_tag``. + """ del builtin_tools diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py index d39c83969f..d2c8c6f412 100644 --- a/vllm/tool_parsers/utils.py +++ b/vllm/tool_parsers/utils.py @@ -56,8 +56,7 @@ def partial_tag_overlap(text: str, tag: str) -> int: def find_common_prefix(s1: str, s2: str) -> str: - """ - Finds a common prefix that is shared between two strings, if there is one. + """Finds a common prefix that is shared between two strings, if there is one. Order of arguments is NOT important. This function is provided as a UTILITY for extracting information from JSON @@ -79,8 +78,7 @@ def find_common_prefix(s1: str, s2: str) -> str: def find_common_suffix(s1: str, s2: str) -> str: - """ - Finds a common suffix shared between two strings, if there is one. Order of + """Finds a common suffix shared between two strings, if there is one. Order of arguments is NOT important. Stops when the suffix ends OR it hits an alphanumeric character @@ -97,8 +95,7 @@ def find_common_suffix(s1: str, s2: str) -> str: def extract_intermediate_diff(curr: str, old: str) -> str: - """ - Given two strings, extract the difference in the middle between two strings + """Given two strings, extract the difference in the middle between two strings that are known to have a common prefix and/or suffix. This function is provided as a UTILITY for extracting information from JSON @@ -450,6 +447,7 @@ def get_parameter_value(val: ast.expr) -> Any: Raises: UnexpectedAstError: If the AST node is not a supported literal type. + """ if isinstance(val, ast.Constant): if val.value is None or isinstance(val.value, (str, int, float)): @@ -528,6 +526,7 @@ def _ast_callable_dotted_name(node: ast.expr) -> str: Raises: UnexpectedAstError: If the chain does not bottom out in an ``ast.Name`` (e.g. subscript or call expression as receiver). + """ parts: list[str] = [] current: ast.expr = node @@ -550,6 +549,7 @@ def handle_single_tool(call: ast.Call) -> ToolCall: Raises: UnexpectedAstError: If the call target is neither a simple name nor a chain of attribute accesses bottoming out in a name. + """ if not isinstance(call.func, (ast.Name, ast.Attribute)): logger.warning( @@ -921,6 +921,7 @@ def make_valid_python(text: str) -> tuple[str, str] | None: Raises: UnexpectedAstError: If mismatched brackets or parentheses are detected. + """ bracket_stack: list[str] = [] for index, char in enumerate(text): @@ -1088,6 +1089,7 @@ def coerce_to_schema_type(value: str, schema_type: str | list[str]) -> Any: value: The raw string value from the model output. schema_type: One or more JSON Schema type strings (e.g. ``"string"`` or ``["string", "null"]``). + """ if isinstance(schema_type, str): schema_type = [schema_type] @@ -1176,6 +1178,7 @@ def compute_tool_delta( Returns: A DeltaToolCall with only the new argument characters, or None if there is no difference from what was previously sent. + """ new_call_args = new_call.function.arguments if withheld_suffix: diff --git a/vllm/tracing/__init__.py b/vllm/tracing/__init__.py index 4f025970ec..9725b57ba9 100644 --- a/vllm/tracing/__init__.py +++ b/vllm/tracing/__init__.py @@ -94,9 +94,7 @@ def instrument( attributes: dict[str, str] | None = None, record_exception: bool = True, ): - """ - Generic decorator to instrument functions. - """ + """Generic decorator to instrument functions.""" if obj is None: return functools.partial( instrument, @@ -135,6 +133,7 @@ def instrument_manual( attributes: Optional dict of span attributes. context: Optional trace context (e.g., from extract_trace_context). kind: Optional SpanKind (e.g., SpanKind.SERVER). + """ is_available, _, _, _, manual_instrument_fn = _REGISTERED_TRACING_BACKENDS["otel"] if is_available(): @@ -146,8 +145,7 @@ def instrument_manual( def is_tracing_available() -> bool: - """ - Returns True if any tracing backend (OTel, Profiler, etc.) is available. + """Returns True if any tracing backend (OTel, Profiler, etc.) is available. Use this to guard expensive tracing logic in the main code. """ check_available = [ diff --git a/vllm/tracing/otel.py b/vllm/tracing/otel.py index ac06ae9725..36c189b883 100644 --- a/vllm/tracing/otel.py +++ b/vllm/tracing/otel.py @@ -107,9 +107,7 @@ def init_otel_worker_tracer( process_kind: str, process_name: str, ) -> Tracer: - """ - Backend-specific initialization for OpenTelemetry in a worker process. - """ + """Backend-specific initialization for OpenTelemetry in a worker process.""" # Initialize the tracer if an OTLP endpoint is configured. # The endpoint is propagated via environment variable from the main process. otlp_endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") @@ -133,7 +131,6 @@ def extract_trace_context(headers: Mapping[str, str] | None) -> Context | None: def instrument_otel(func, span_name, attributes, record_exception): """Internal wrapper logic for sync and async functions.""" - # Pre-calculate static code attributes once (these don't change) code_attrs = { LoadingSpanAttributes.CODE_FUNCTION: func.__qualname__, @@ -214,8 +211,7 @@ def manual_instrument_otel( def _get_smart_context() -> Context | None: - """ - Determines the parent context. + """Determines the parent context. 1. If a Span is already active in this process, use it. 2. If not, extract from os.environ, handling the case-sensitivity mismatch. """ @@ -239,8 +235,7 @@ def _get_smart_context() -> Context | None: @contextmanager def propagate_trace_to_env(): - """ - Temporarily injects the current OTel context into os.environ. + """Temporarily injects the current OTel context into os.environ. This ensures that any subprocesses (like vLLM workers) spawned within this context inherit the correct traceparent. """ diff --git a/vllm/tracing/utils.py b/vllm/tracing/utils.py index 0e11f850b4..138b801f51 100644 --- a/vllm/tracing/utils.py +++ b/vllm/tracing/utils.py @@ -13,8 +13,7 @@ TRACE_HEADERS = ["traceparent", "tracestate"] class SpanAttributes: - """ - Standard attributes for spans. + """Standard attributes for spans. These are largely based on OpenTelemetry Semantic Conventions but are defined here as constants so they can be used by any backend or logger. @@ -60,8 +59,7 @@ def contains_trace_headers(headers: Mapping[str, str]) -> bool: def extract_trace_headers(headers: Mapping[str, str]) -> Mapping[str, str]: - """ - Extract only trace-related headers from a larger header dictionary. + """Extract only trace-related headers from a larger header dictionary. Useful for logging or passing context to a non-OTel client. """ return {h: headers[h] for h in TRACE_HEADERS if h in headers} diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index d34e391a26..54f3086431 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -13,6 +13,7 @@ from typing import Any, Literal, TypeAlias import huggingface_hub import torch +import transformers.configuration_utils as hf_configuration_utils from huggingface_hub import constants from packaging.version import Version from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE @@ -126,7 +127,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( RefinedWebModel="RWConfig", # For tiiuae/falcon-7b(-instruct) mlp_speculator="MLPSpeculatorConfig", medusa="MedusaConfig", - mellum="MellumConfig", midashenglm="MiDashengLMConfig", minimax_m3_vl="MiniMaxM3Config", minimax_m3_mtp="MiniMaxM3MTPConfig", @@ -151,7 +151,6 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( qwen3_5_text="Qwen3_5TextConfig", qwen3_5_moe="Qwen3_5MoeConfig", qwen3_5_moe_text="Qwen3_5MoeTextConfig", - laguna="LagunaConfig", lfm2_moe="Lfm2MoeConfig", **{"unlimited-ocr": "UnlimitedOCRConfig"}, **{"deepseek_v41": "DeepseekV41Config"}, @@ -163,6 +162,12 @@ _SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"} _PATCH_HF_VALIDATE_ROPE: set[str] = {"sarvam_mla"} +# Model types whose checkpoints store shared RoPE parameters alongside the +# per-layer-type dicts (e.g. Laguna's `original_max_position_embeddings`). +# Since transformers 5.17, `validate_rope` treats every top-level value of such +# a dict as a layer's parameters and raises on the shared ones. +_PATCH_HF_NESTED_ROPE_VALIDATION: set[str] = {"laguna"} + # Model types whose checkpoints declare `layer_types` entries that upstream # transformers has not added to `ALLOWED_LAYER_TYPES` yet, so its strict config # validation rejects them (e.g. GLM-5.2 `glm_moe_dsa` use @@ -226,7 +231,6 @@ def _patch_hf_transformers_validate_rope(): validate_rope() with the ignore_keys parameter work with newer versions of hf transformers (from v5 onwards) """ - if hasattr(PretrainedConfig.validate_rope, "__vllm_patched__"): return @@ -236,6 +240,11 @@ def _patch_hf_transformers_validate_rope(): def patched_validate_rope(self, *args, **kwargs): ignore_keys_param = kwargs.pop("ignore_keys", None) original_ignore_keys = self.ignore_keys_at_rope_validation + if ignore_keys_param is not None: + logger.warning( + "validate_rope() was called with the legacy 'ignore_keys' " + "argument; use 'ignore_keys_at_rope_validation' instead" + ) self.ignore_keys_at_rope_validation = original_ignore_keys or ignore_keys_param result = _original_validate_rope(self, *args, **kwargs) return result @@ -244,6 +253,40 @@ def _patch_hf_transformers_validate_rope(): PretrainedConfig.validate_rope = patched_validate_rope +def _patch_hf_transformers_nested_rope_validation() -> None: + """Drop shared entries sitting alongside a nested ``rope_parameters`` dict. + + Transformers validates a dict with any layer-type key by iterating all of + its values, so a shared entry next to the per-layer dicts raises an + ``AttributeError``. The per-layer dicts already carry their own defaults by + the time validation runs, so the shared entries can be dropped. + """ + if hasattr(PretrainedConfig.validate_rope, "__vllm_nested_rope_patched__"): + return + + _original_validate_rope = PretrainedConfig.validate_rope + + @wraps(_original_validate_rope) + def patched_validate_rope(self, *args, **kwargs): + rope_parameters = getattr(self, "rope_parameters", None) + if isinstance(rope_parameters, dict): + layer_types = set(rope_parameters) & set( + hf_configuration_utils.ALLOWED_LAYER_TYPES + ) + if shared_keys := set(rope_parameters) - layer_types: + for key in shared_keys: + del rope_parameters[key] + logger.warning( + "Dropping %s from rope_parameters; entries shared across " + "layer types are not supported", + sorted(shared_keys), + ) + return _original_validate_rope(self, *args, **kwargs) + + patched_validate_rope.__vllm_nested_rope_patched__ = True # type: ignore[attr-defined] + PretrainedConfig.validate_rope = patched_validate_rope + + def _patch_hf_transformers_allowed_layer_types( extra_layer_types: tuple[str, ...], ) -> None: @@ -304,6 +347,9 @@ class HFConfigParser(ConfigParserBase): if model_type in _PATCH_HF_VALIDATE_ROPE: _patch_hf_transformers_validate_rope() + if model_type in _PATCH_HF_NESTED_ROPE_VALIDATION: + _patch_hf_transformers_nested_rope_validation() + if extra_layer_types := _PATCH_HF_ALLOWED_LAYER_TYPES.get(model_type): _patch_hf_transformers_allowed_layer_types(extra_layer_types) @@ -442,10 +488,11 @@ def register_config_parser(config_format: str): """Register a customized vllm config parser. When a config format is not supported by vllm, you can register a customized config parser to support it. - Args: - config_format (str): The config parser format name. - Examples: + Args: + config_format (str): The config parser format name. + + Examples: >>> from vllm.transformers_utils.config import (get_config_parser, register_config_parser) >>> from vllm.transformers_utils.config_parser_base import ConfigParserBase @@ -464,6 +511,7 @@ def register_config_parser(config_format: str): >>> >>> type(get_config_parser("custom_config_parser")) + """ # noqa: E501 def _wrapper(config_parser_cls): @@ -501,7 +549,6 @@ def set_default_rope_theta(config: PretrainedConfig, default_theta: float) -> No def patch_legacy_rope_type(rope_parameters: dict[str, Any] | None) -> None: """Patch legacy RoPE type fields for backwards compatibility with older custom models which would otherwise fail to load.""" - # No RoPE parameters to patch if rope_parameters is None: return @@ -674,9 +721,7 @@ def is_encoder_decoder(config: PretrainedConfig) -> bool: def _maybe_update_auto_config_kwargs(kwargs: dict[str, Any], model_type: str): - """ - Update kwargs for AutoConfig initialization based on model_type - """ + """Update kwargs for AutoConfig initialization based on model_type.""" if model_type in _AUTO_CONFIG_KWARGS_OVERRIDES: kwargs.update(_AUTO_CONFIG_KWARGS_OVERRIDES[model_type]) return kwargs @@ -701,8 +746,7 @@ def maybe_override_with_speculators( hf_token: bool | str | None = None, **kwargs, ) -> tuple[str, str | None, dict[str, Any] | None]: - """ - Resolve model configuration when speculators are detected. + """Resolve model configuration when speculators are detected. Checks if the provided model is a speculators model and if so, extracts the target model configuration and builds the speculative config. @@ -717,6 +761,7 @@ def maybe_override_with_speculators( Returns: Tuple of (resolved_model, resolved_tokenizer, speculative_config) + """ kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE config_dict, _ = PretrainedConfig.get_config_dict( @@ -891,8 +936,7 @@ def get_pooling_config( model: str, revision: str | None = "main", ) -> dict[str, Any] | None: - """ - This function gets the pooling and normalize + """This function gets the pooling and normalize config from the model - only applies to sentence-transformers models. @@ -904,6 +948,7 @@ def get_pooling_config( Returns: A dictionary containing the pooling type and whether normalization is used, or None if no pooling configuration is found. + """ modules_file_name = "modules.json" @@ -988,19 +1033,17 @@ def parse_pooling_type(pooling_name: str): def get_sentence_transformer_tokenizer_config( model: str | Path, revision: str | None = "main" ) -> dict[str, Any] | None: - """ - Returns the tokenization configuration dictionary for a + """Returns the tokenization configuration dictionary for a given Sentence Transformer BERT model. - Parameters: - - model (str|Path): The name of the Sentence Transformer - BERT model. - - revision (str, optional): The revision of the m - odel to use. Defaults to 'main'. + Args: + model (str|Path): The name of the Sentence Transformer BERT model. + revision (str, optional): The revision of the model to use. Defaults to 'main'. Returns: - dict: A dictionary containing the configuration parameters for the Sentence Transformer BERT model. + """ sentence_transformer_config_files = [ "sentence_bert_config.json", @@ -1261,9 +1304,7 @@ def get_safetensors_params_metadata( *, revision: str | None = None, ) -> dict[str, Any]: - """ - Get the safetensors parameters metadata for remote/local model repository. - """ + """Get the safetensors parameters metadata for remote/local model repository.""" if (model_path := Path(model)).exists(): return _read_safetensors_metadata_in_dir(model_path) @@ -1307,6 +1348,7 @@ def checkpoint_has_lm_head(model: str, *, revision: str | None = None) -> bool | Returns: `None` if the checkpoint contents could not be determined, for example because it is not stored as safetensors. + """ metadata = get_safetensors_params_metadata(model, revision=revision) if not metadata: diff --git a/vllm/transformers_utils/configs/AXK1.py b/vllm/transformers_utils/configs/AXK1.py index 09a3a0f46a..52af5bec48 100644 --- a/vllm/transformers_utils/configs/AXK1.py +++ b/vllm/transformers_utils/configs/AXK1.py @@ -6,14 +6,14 @@ from transformers import PretrainedConfig class AXK1Config(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`AXK1Model`]. + r"""This is the configuration class to store the configuration of a [`AXK1Model`]. It is used to instantiate an A.X model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the A.X K1. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. + Args: vocab_size (`int`, *optional*, defaults to 163840): Vocabulary size of the A.X K1 model. Defines the number of different @@ -112,6 +112,7 @@ class AXK1Config(PretrainedConfig): layers during self-attention. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. + """ model_type = "axk1" diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 971b6ffcb2..a302d5d729 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Model configs may be defined in this directory for the following reasons: +"""Model configs may be defined in this directory for the following reasons: - There is no configuration file defined by HF Hub or Transformers library. - There is a need to override the existing config to support vLLM. @@ -55,10 +54,8 @@ _CLASS_TO_MODULE: dict[str, str] = { # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the # `FalconConfig` class from the official HuggingFace transformers library. "RWConfig": "vllm.transformers_utils.configs.falcon", - "LagunaConfig": "vllm.transformers_utils.configs.laguna", "Lfm2MoeConfig": "vllm.transformers_utils.configs.lfm2_moe", "MedusaConfig": "vllm.transformers_utils.configs.medusa", - "MellumConfig": "vllm.transformers_utils.configs.mellum", "MiDashengLMConfig": "vllm.transformers_utils.configs.midashenglm", "MiniMaxM3Config": "vllm.transformers_utils.configs.minimax_m3", "MiniMaxM3MTPConfig": "vllm.transformers_utils.configs.minimax_m3", @@ -151,10 +148,8 @@ __all__ = [ "HyperCLOVAXConfig", "IsaacConfig", "RWConfig", - "LagunaConfig", "Lfm2MoeConfig", "MedusaConfig", - "MellumConfig", "MiDashengLMConfig", "MiniMaxM3Config", "MiniMaxM3MTPConfig", diff --git a/vllm/transformers_utils/configs/colpali.py b/vllm/transformers_utils/configs/colpali.py index c40c58b25c..a4e0a83e29 100644 --- a/vllm/transformers_utils/configs/colpali.py +++ b/vllm/transformers_utils/configs/colpali.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -ColPali configuration that extends PaliGemmaConfig with embedding projection +"""ColPali configuration that extends PaliGemmaConfig with embedding projection fields. This allows ColPali models to be loaded without trust_remote_code by mapping their custom model_type (colpali) to a standard config class that vLLM understands. diff --git a/vllm/transformers_utils/configs/colqwen3.py b/vllm/transformers_utils/configs/colqwen3.py index 1c09a0a918..c98995491d 100644 --- a/vllm/transformers_utils/configs/colqwen3.py +++ b/vllm/transformers_utils/configs/colqwen3.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -ColQwen3 configuration that extends Qwen3VLConfig with embedding projection +"""ColQwen3 configuration that extends Qwen3VLConfig with embedding projection fields. This allows ColQwen3 models to be loaded without trust_remote_code by mapping their custom model_type (colqwen3, ops_colqwen3, etc.) to a standard config class that vLLM understands. diff --git a/vllm/transformers_utils/configs/deepseek_v41.py b/vllm/transformers_utils/configs/deepseek_v41.py index 8f33245cd6..d8b4335438 100644 --- a/vllm/transformers_utils/configs/deepseek_v41.py +++ b/vllm/transformers_utils/configs/deepseek_v41.py @@ -60,11 +60,3 @@ class DeepseekV41Config(PretrainedConfig): self.vision_max_n_token = vision_config.get("max_image_tokens", 1024) self.vision_min_pixels = vision_config.get("min_pixels", 295936) self.vision_max_wh_ratio = vision_config.get("max_wh_ratio") - # The vision variant needs the mm-prefix plumbing (atomic image-span - # prefill + sparse-SWA window widening); the base arch-config - # convertor reads this config-level attribute. - self.is_mm_prefix_lm = vision_n_layers > 0 - # The sparse-SWA index kernels widen the window within image spans - # in-kernel, so mm-prefix ranges longer than sliding_window must be - # kept (they are only consumed by those kernels). - self.mm_prefix_clamp_sliding_window = vision_n_layers > 0 diff --git a/vllm/transformers_utils/configs/falcon.py b/vllm/transformers_utils/configs/falcon.py index c646d241d4..f8665a1395 100644 --- a/vllm/transformers_utils/configs/falcon.py +++ b/vllm/transformers_utils/configs/falcon.py @@ -18,7 +18,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Falcon configuration""" +"""Falcon configuration.""" from transformers.configuration_utils import PretrainedConfig diff --git a/vllm/transformers_utils/configs/hy_v3.py b/vllm/transformers_utils/configs/hy_v3.py index 9425caf4e0..0a67dd8af7 100644 --- a/vllm/transformers_utils/configs/hy_v3.py +++ b/vllm/transformers_utils/configs/hy_v3.py @@ -6,8 +6,7 @@ from transformers.configuration_utils import PretrainedConfig class HYV3Config(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`HYV3Model`]. + r"""This is the configuration class to store the configuration of a [`HYV3Model`]. It is used to instantiate a HYV3 model (HY V3 MoE language model) according to the specified arguments. @@ -93,6 +92,7 @@ class HYV3Config(PretrainedConfig): >>> config = HYV3Config() >>> model = HYV3Model(config) ``` + """ model_type = "hy_v3" diff --git a/vllm/transformers_utils/configs/hy_v4.py b/vllm/transformers_utils/configs/hy_v4.py index beca23ee74..687a2206c8 100644 --- a/vllm/transformers_utils/configs/hy_v4.py +++ b/vllm/transformers_utils/configs/hy_v4.py @@ -52,6 +52,7 @@ class HYV4Config(PreTrainedConfig): below zero disable the clamp. num_nextn_predict_layers: Number of MTP layers in the checkpoint. mtp_loss_factor: Training-time MTP loss weight (unused at inference). + """ model_type = "hy_v4" diff --git a/vllm/transformers_utils/configs/hyperclovax.py b/vllm/transformers_utils/configs/hyperclovax.py index 9fa823743d..2033d59c11 100644 --- a/vllm/transformers_utils/configs/hyperclovax.py +++ b/vllm/transformers_utils/configs/hyperclovax.py @@ -21,8 +21,7 @@ from transformers.configuration_utils import PretrainedConfig class HyperCLOVAXConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a + r"""This is the configuration class to store the configuration of a [`HyperCLOVAXModel`]. It is used to instantiate a HyperCLOVAX model according to the specified arguments, defining the model architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used @@ -166,6 +165,7 @@ class HyperCLOVAXConfig(PretrainedConfig): `get_rope`. When provided, takes precedence over `rope_theta` and `rope_scaling`. If `None`, it is derived from `rope_theta` and `rope_scaling` automatically. + """ model_type = "hyperclovax" diff --git a/vllm/transformers_utils/configs/kimi_k25.py b/vllm/transformers_utils/configs/kimi_k25.py index 710e9b5636..095db1d8c4 100644 --- a/vllm/transformers_utils/configs/kimi_k25.py +++ b/vllm/transformers_utils/configs/kimi_k25.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Kimi-K2.5 Model Configuration. +"""Kimi-K2.5 Model Configuration. This configuration supports video-chunk as an internal modality type. A video-chunk is the smallest independently processable unit of video. @@ -73,6 +72,7 @@ class KimiK25Config(PretrainedConfig): ignore_index: The ignore index for the loss function. media_placeholder_token_id: The token ID for media placeholders. pad_token_id: The token ID for padding. + """ model_type = "kimi_k25" diff --git a/vllm/transformers_utils/configs/laguna.py b/vllm/transformers_utils/configs/laguna.py deleted file mode 100644 index 2702d3af5a..0000000000 --- a/vllm/transformers_utils/configs/laguna.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers.configuration_utils import PretrainedConfig - - -class LagunaConfig(PretrainedConfig): - model_type = "laguna" - keys_to_ignore_at_inference = ["past_key_values"] - base_model_tp_plan = { - "layers.*.self_attn.q_proj": "colwise", - "layers.*.self_attn.k_proj": "colwise", - "layers.*.self_attn.v_proj": "colwise", - "layers.*.self_attn.g_proj": "colwise", - "layers.*.self_attn.o_proj": "rowwise", - "layers.*.mlp.gate_proj": "colwise", - "layers.*.mlp.up_proj": "colwise", - "layers.*.mlp.down_proj": "rowwise", - } - base_model_pp_plan = { - "embed_tokens": (["input_ids"], ["inputs_embeds"]), - "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), - "norm": (["hidden_states"], ["hidden_states"]), - } - - def __init__( - self, - vocab_size: int = 100352, - hidden_size: int = 2048, - intermediate_size: int = 8192, - num_hidden_layers: int = 40, - num_attention_heads: int = 48, - num_key_value_heads: int = 8, - head_dim: int = 128, - qkv_bias: bool = False, - attention_bias: bool = False, - gating: bool | str = True, - hidden_act: str = "silu", - max_position_embeddings: int = 131072, - initializer_range: float = 0.02, - rms_norm_eps: float = 1e-6, - use_cache: bool = True, - tie_word_embeddings: bool = False, - rope_theta: float = 500000.0, - rope_scaling: dict | None = None, - rope_parameters: dict | None = None, - partial_rotary_factor: float = 1.0, - attention_dropout: float = 0.0, - sliding_window: int | None = None, - layer_types: list[str] | None = None, - swa_attention_sink_enabled: bool = False, - swa_rope_parameters: dict | None = None, - num_attention_heads_per_layer: list[int] | None = None, - num_experts: int = 256, - num_experts_per_tok: int = 8, - moe_intermediate_size: int = 512, - shared_expert_intermediate_size: int = 512, - norm_topk_prob: bool = True, - decoder_sparse_step: int = 1, - mlp_only_layers: list[int] | None = None, - router_aux_loss_coef: float = 0.001, - output_router_logits: bool = False, - moe_routed_scaling_factor: float = 1.0, - moe_apply_router_weight_on_input: bool = False, - **kwargs, - ): - if mlp_only_layers is None: - mlp_only_layers = [0] - - # Accept either v4-style (rope_theta + rope_scaling) or v5-style - # (rope_parameters). Translate v5 → v4 so downstream code has one path. - if rope_parameters is not None: - rp = dict(rope_parameters) - rope_theta = float(rp.pop("rope_theta", rope_theta)) - rt = rp.pop("rope_type", None) - if rt is not None and rt != "default": - rope_scaling = {"rope_type": rt, **rp} - elif rp and rope_scaling is None: - rope_scaling = {"rope_type": "default", **rp} - - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.num_key_value_heads = num_key_value_heads - self.head_dim = head_dim - self.qkv_bias = qkv_bias - self.attention_bias = attention_bias - self.gating = gating - self.hidden_act = hidden_act - self.max_position_embeddings = max_position_embeddings - self.initializer_range = initializer_range - self.rms_norm_eps = rms_norm_eps - self.use_cache = use_cache - self.rope_theta = rope_theta - self.rope_scaling = rope_scaling - self.partial_rotary_factor = partial_rotary_factor - self.attention_dropout = attention_dropout - self.sliding_window = sliding_window - self.layer_types = layer_types - self.swa_attention_sink_enabled = swa_attention_sink_enabled - self.swa_rope_parameters = swa_rope_parameters - self.num_attention_heads_per_layer = num_attention_heads_per_layer - self.num_experts = num_experts - self.num_experts_per_tok = num_experts_per_tok - self.moe_intermediate_size = moe_intermediate_size - self.shared_expert_intermediate_size = shared_expert_intermediate_size - self.norm_topk_prob = norm_topk_prob - self.decoder_sparse_step = decoder_sparse_step - self.mlp_only_layers = mlp_only_layers - self.router_aux_loss_coef = router_aux_loss_coef - self.output_router_logits = output_router_logits - self.moe_routed_scaling_factor = moe_routed_scaling_factor - self.moe_apply_router_weight_on_input = moe_apply_router_weight_on_input - - super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs) - - -__all__ = ["LagunaConfig"] diff --git a/vllm/transformers_utils/configs/lfm2_moe.py b/vllm/transformers_utils/configs/lfm2_moe.py index b399a03c03..34682a62a1 100644 --- a/vllm/transformers_utils/configs/lfm2_moe.py +++ b/vllm/transformers_utils/configs/lfm2_moe.py @@ -6,8 +6,7 @@ from transformers.configuration_utils import PretrainedConfig class Lfm2MoeConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Lfm2MoeModel`]. It is used to instantiate a LFM2 Moe + r"""This is the configuration class to store the configuration of a [`Lfm2MoeModel`]. It is used to instantiate a LFM2 Moe model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the LFM2-8B-A1B model. e.g. [LiquidAI/LFM2-8B-A1B](https://huggingface.co/LiquidAI/LFM2-8B-A1B) @@ -85,7 +84,9 @@ class Lfm2MoeConfig(PretrainedConfig): >>> # Accessing the model configuration >>> configuration = model.config - ```""" # noqa: E501 + ``` + + """ # noqa: E501 model_type = "lfm2_moe" keys_to_ignore_at_inference = ["past_key_values"] diff --git a/vllm/transformers_utils/configs/mellum.py b/vllm/transformers_utils/configs/mellum.py deleted file mode 100644 index 2bed53394b..0000000000 --- a/vllm/transformers_utils/configs/mellum.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from transformers import Qwen3MoeConfig - - -class MellumConfig(Qwen3MoeConfig): - model_type = "mellum" diff --git a/vllm/transformers_utils/configs/mlp_speculator.py b/vllm/transformers_utils/configs/mlp_speculator.py index 75745f227f..6b5a70e3db 100644 --- a/vllm/transformers_utils/configs/mlp_speculator.py +++ b/vllm/transformers_utils/configs/mlp_speculator.py @@ -24,8 +24,7 @@ class MLPSpeculatorConfig(PretrainedConfig): scale_input: bool = False, **kwargs, ): - """ - Initialize an MLPSpeculatorConfig + """Initialize an MLPSpeculatorConfig. Args: vocab_size: int @@ -52,6 +51,7 @@ class MLPSpeculatorConfig(PretrainedConfig): scale_input: bool if True, will scale the initial hidden states from the base model. + """ if top_k_tokens_per_head is None: top_k_tokens_per_head = [5, 4, 3] diff --git a/vllm/transformers_utils/configs/moondream3.py b/vllm/transformers_utils/configs/moondream3.py index 307bb29772..8bff5cd10e 100644 --- a/vllm/transformers_utils/configs/moondream3.py +++ b/vllm/transformers_utils/configs/moondream3.py @@ -148,5 +148,6 @@ class Moondream3Config(PretrainedConfig): Args: decoder: Ignored. Only used for encoder-decoder models. + """ return self.text_config diff --git a/vllm/transformers_utils/configs/nemotron.py b/vllm/transformers_utils/configs/nemotron.py index 62f5270302..bb15a8b433 100644 --- a/vllm/transformers_utils/configs/nemotron.py +++ b/vllm/transformers_utils/configs/nemotron.py @@ -15,7 +15,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Nemotron model configuration""" +"""Nemotron model configuration.""" from transformers import PretrainedConfig from transformers.utils import logging @@ -24,8 +24,7 @@ logger = logging.get_logger(__name__) class NemotronConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a + r"""This is the configuration class to store the configuration of a [`NemotronModel`]. It is used to instantiate a Nemotron model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar @@ -114,7 +113,9 @@ class NemotronConfig(PretrainedConfig): >>> model = NemotronModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config - ```""" + ``` + + """ model_type = "nemotron" keys_to_ignore_at_inference = ["past_key_values"] @@ -193,9 +194,7 @@ class NemotronConfig(PretrainedConfig): ) def _rope_parameters_validation(self): - """ - Validate the `rope_parameters` configuration. - """ + """Validate the `rope_parameters` configuration.""" if self.rope_parameters is None: return diff --git a/vllm/transformers_utils/configs/nemotron_h.py b/vllm/transformers_utils/configs/nemotron_h.py index ed62b5d294..c83313edfb 100644 --- a/vllm/transformers_utils/configs/nemotron_h.py +++ b/vllm/transformers_utils/configs/nemotron_h.py @@ -15,7 +15,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""NemotronH model configuration""" +"""NemotronH model configuration.""" import regex as re from transformers.configuration_utils import PretrainedConfig @@ -25,12 +25,12 @@ logger = logging.get_logger(__name__) class NemotronHConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a + r"""This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model. + Args: vocab_size (`int`, *optional*, defaults to 131072): Vocabulary size of the NemotronH model. Defines the number of @@ -139,6 +139,7 @@ class NemotronHConfig(PretrainedConfig): Size of chunks for Mamba processing. rescale_prenorm_residual (`bool`, *optional*, defaults to `True`): Whether to rescale the pre-normalization residual connections. + """ model_type = "nemotron_h" diff --git a/vllm/transformers_utils/configs/olmo_hybrid.py b/vllm/transformers_utils/configs/olmo_hybrid.py index cdca81757e..6e12cdd274 100644 --- a/vllm/transformers_utils/configs/olmo_hybrid.py +++ b/vllm/transformers_utils/configs/olmo_hybrid.py @@ -6,8 +6,7 @@ from transformers.configuration_utils import PretrainedConfig class OlmoHybridConfig(PretrainedConfig): - r""" - Configuration class for [`OlmoHybridModel`]. It is used to + r"""Configuration class for [`OlmoHybridModel`]. It is used to instantiate an OLMo Hybrid model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar @@ -19,7 +18,7 @@ class OlmoHybridConfig(PretrainedConfig): can be used to control the model outputs. Read the documentation from [`PreTrainedConfig`] for more information. - Args: + Args: vocab_size (`int`, *optional*, defaults to 100352): Vocabulary size of the OlmoHybrid model. Defines the number of different tokens that can be @@ -159,6 +158,7 @@ class OlmoHybridConfig(PretrainedConfig): >>> model = OlmoHybridModel(configuration) >>> configuration = model.config ``` + """ model_type = "olmo_hybrid" diff --git a/vllm/transformers_utils/configs/ovis.py b/vllm/transformers_utils/configs/ovis.py index 294b4c9037..64629566a3 100644 --- a/vllm/transformers_utils/configs/ovis.py +++ b/vllm/transformers_utils/configs/ovis.py @@ -14,6 +14,7 @@ class AIMv2Config(PretrainedConfig): """This is the configuration class to store the configuration of an [`AIMv2Model`]. Instantiating a configuration with the defaults will yield a similar configuration to that of the [apple/aimv2-large-patch14-224](https://huggingface.co/apple/aimv2-large-patch14-224). + Args: hidden_size: Dimension of the hidden representations. intermediate_size: Dimension of the SwiGLU representations. @@ -29,6 +30,7 @@ class AIMv2Config(PretrainedConfig): qkv_bias: Whether to add a bias to the queries, keys and values. use_bias: Whether to add a bias in the feed-forward and projection layers. kwargs: Keyword arguments for the [`PretrainedConfig`]. + """ model_type: str = "aimv2" diff --git a/vllm/transformers_utils/configs/qwen3_5.py b/vllm/transformers_utils/configs/qwen3_5.py index d5820a5783..e30cbe6502 100644 --- a/vllm/transformers_utils/configs/qwen3_5.py +++ b/vllm/transformers_utils/configs/qwen3_5.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Qwen3.5 model configuration""" +"""Qwen3.5 model configuration.""" from transformers.configuration_utils import PretrainedConfig diff --git a/vllm/transformers_utils/configs/qwen3_5_moe.py b/vllm/transformers_utils/configs/qwen3_5_moe.py index ec229ce814..938272d028 100644 --- a/vllm/transformers_utils/configs/qwen3_5_moe.py +++ b/vllm/transformers_utils/configs/qwen3_5_moe.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Qwen3.5-MoE model configuration""" +"""Qwen3.5-MoE model configuration.""" from transformers.configuration_utils import PretrainedConfig diff --git a/vllm/transformers_utils/configs/qwen3_next.py b/vllm/transformers_utils/configs/qwen3_next.py index de579ed2cf..8460725bfd 100644 --- a/vllm/transformers_utils/configs/qwen3_next.py +++ b/vllm/transformers_utils/configs/qwen3_next.py @@ -14,7 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Qwen3-Next model configuration""" +"""Qwen3-Next model configuration.""" from transformers.configuration_utils import PretrainedConfig from transformers.utils import logging @@ -23,8 +23,7 @@ logger = logging.get_logger(__name__) class Qwen3NextConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a + r"""This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a Qwen3-Next model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct). @@ -157,6 +156,7 @@ class Qwen3NextConfig(PretrainedConfig): >>> # Accessing the model configuration >>> configuration = model.config ``` + """ # noqa: E501 model_type = "qwen3_next" diff --git a/vllm/transformers_utils/configs/radio.py b/vllm/transformers_utils/configs/radio.py index e668c5c5e7..4a3e468f84 100644 --- a/vllm/transformers_utils/configs/radio.py +++ b/vllm/transformers_utils/configs/radio.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Radio vision model configuration""" +"""Radio vision model configuration.""" from typing import Any @@ -21,8 +21,7 @@ OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711) class RadioConfig(PretrainedConfig): - r""" - This is the configuration class to store the configuration of a Radio + r"""This is the configuration class to store the configuration of a Radio vision model. It is used to instantiate a Radio model according to the specified arguments, defining the model architecture. @@ -55,6 +54,7 @@ class RadioConfig(PretrainedConfig): dedicated video patch embedder (3*T*P*P -> hidden) separate from the image embedder (3*P*P -> hidden). When False, a single embedder with input size 3*T*P*P is used for both (images are duplicated T times). + """ model_type = "radio" diff --git a/vllm/transformers_utils/configs/speculators/algos.py b/vllm/transformers_utils/configs/speculators/algos.py index 9f45183328..6407fc4868 100644 --- a/vllm/transformers_utils/configs/speculators/algos.py +++ b/vllm/transformers_utils/configs/speculators/algos.py @@ -14,8 +14,7 @@ def register_speculator(name): @register_speculator("eagle3") def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None: - """ - Apply Eagle-3 specific configuration transformations to the `dict` used to + """Apply Eagle-3 specific configuration transformations to the `dict` used to construct the Transformers PreTrainedConfig. Eagle-3 specific fields: @@ -28,7 +27,6 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None: provide intermediate hidden states that help the drafter make better predictions. This is the standard field used in Eagle3 checkpoints. """ - pre_trained_config["draft_vocab_size"] = config_dict.get("draft_vocab_size") if config_dict.get("target_hidden_size") is not None: pre_trained_config["target_hidden_size"] = config_dict["target_hidden_size"] @@ -54,8 +52,7 @@ def update_eagle3(config_dict: dict, pre_trained_config: dict) -> None: @register_speculator("peagle") def update_peagle(config_dict: dict, pre_trained_config: dict) -> None: - """ - Apply PEagle (Parallel Eagle) specific configuration transformations to + """Apply PEagle (Parallel Eagle) specific configuration transformations to the `dict` used to construct the Transformers PreTrainedConfig. PEagle specific fields: @@ -92,8 +89,7 @@ def update_peagle(config_dict: dict, pre_trained_config: dict) -> None: @register_speculator("dflash") def update_dflash(config_dict: dict, pre_trained_config: dict) -> None: - """ - Apply DFlash specific configuration transformations to the `dict` used to + """Apply DFlash specific configuration transformations to the `dict` used to construct the Transformers PreTrainedConfig. DFlash specific fields: @@ -132,8 +128,7 @@ def update_dflash(config_dict: dict, pre_trained_config: dict) -> None: @register_speculator("dflash2") def update_dflash2(config_dict: dict, pre_trained_config: dict) -> None: - """ - Apply DFlash2 specific configuration transformations to the `dict` used + """Apply DFlash2 specific configuration transformations to the `dict` used to construct the Transformers PreTrainedConfig. DFlash2 extends DFlash with a grouped convolution layer and a @@ -156,8 +151,7 @@ def update_dflash2(config_dict: dict, pre_trained_config: dict) -> None: @register_speculator("dspark") def update_dspark(config_dict: dict, pre_trained_config: dict) -> None: - """ - Apply DSpark specific configuration transformations to the `dict` used to + """Apply DSpark specific configuration transformations to the `dict` used to construct the Transformers PreTrainedConfig. DSpark extends DFlash with a Markov logit-bias head, reusing the same diff --git a/vllm/transformers_utils/configs/speculators/base.py b/vllm/transformers_utils/configs/speculators/base.py index 1cbaf6e801..a867749dba 100644 --- a/vllm/transformers_utils/configs/speculators/base.py +++ b/vllm/transformers_utils/configs/speculators/base.py @@ -47,9 +47,7 @@ class SpeculatorsConfig(PretrainedConfig): def extract_transformers_pre_trained_config( cls, config_dict: dict[str, Any] ) -> dict[str, Any]: - """ - Extract standard Transformers PreTrainedConfig config from speculators config. - """ + """Extract the Transformers `PreTrainedConfig` from a speculators config.""" speculators_model_type = config_dict.get("speculators_model_type") if speculators_model_type not in SUPPORTED_SPECULATORS_TYPES: raise ValueError( @@ -99,8 +97,7 @@ class SpeculatorsConfig(PretrainedConfig): def build_vllm_speculative_config( cls, config_dict: dict[str, Any] ) -> dict[str, Any]: - """ - Build vLLM-compatible speculative configuration from speculators format. + """Build vLLM-compatible speculative configuration from speculators format. This method extracts and transforms speculative configuration from the speculators format into the structure expected by vLLM. @@ -110,6 +107,7 @@ class SpeculatorsConfig(PretrainedConfig): Returns: Dictionary with vLLM-compatible speculative configuration + """ # Extract speculators configuration spec_config = config_dict["speculators_config"] diff --git a/vllm/transformers_utils/configs/ultravox.py b/vllm/transformers_utils/configs/ultravox.py index 31b49b9d99..b8bd003199 100644 --- a/vllm/transformers_utils/configs/ultravox.py +++ b/vllm/transformers_utils/configs/ultravox.py @@ -8,8 +8,7 @@ import transformers class UltravoxConfig(transformers.PretrainedConfig): - r""" - This is the configuration class to store the configuration of a + r"""This is the configuration class to store the configuration of a [`UltravoxForConditionalGeneration`]. It is used to instantiate an Ultravox model according to the specified arguments, defining the model architecture. @@ -41,6 +40,7 @@ class UltravoxConfig(transformers.PretrainedConfig): Whether to apply layer normalization at the middle of the projector or at the end. Versions v0.4.1 and below use `False`, but v0.5 and above use `True`. + """ model_type = "ultravox" diff --git a/vllm/transformers_utils/dynamic_module.py b/vllm/transformers_utils/dynamic_module.py index f0702b15bb..f42d5752bf 100644 --- a/vllm/transformers_utils/dynamic_module.py +++ b/vllm/transformers_utils/dynamic_module.py @@ -29,8 +29,7 @@ def try_get_class_from_dynamic_module( warn_on_fail: bool = True, **kwargs, ) -> type | None: - """ - As `transformers.dynamic_module_utils.get_class_from_dynamic_module`, + """As `transformers.dynamic_module_utils.get_class_from_dynamic_module`, but ignoring any errors. """ try: diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 8fd924fcf3..64af034d9a 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -80,9 +80,7 @@ _V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor) class HashableDict(dict): - """ - A dictionary that can be hashed by lru_cache. - """ + """A dictionary that can be hashed by lru_cache.""" # NOTE: pythonic dict is not hashable, # we override on it directly for simplicity @@ -91,9 +89,7 @@ class HashableDict(dict): class HashableList(list): - """ - A list that can be hashed by lru_cache. - """ + """A list that can be hashed by lru_cache.""" def __hash__(self) -> int: # type: ignore[override] return hash(tuple(self)) diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index 249a4e58e6..b46183f4e1 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Multi-modal processors may be defined in this directory for the following +"""Multi-modal processors may be defined in this directory for the following reasons: - There is no processing file defined by HF Hub or Transformers library. diff --git a/vllm/transformers_utils/processors/bagel.py b/vllm/transformers_utils/processors/bagel.py index 3226d7b0c8..b6fefe0327 100644 --- a/vllm/transformers_utils/processors/bagel.py +++ b/vllm/transformers_utils/processors/bagel.py @@ -18,8 +18,7 @@ class BagelProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call- class BagelProcessor(ProcessorMixin): - """ - Constructs a BAGEL processor which wraps a + """Constructs a BAGEL processor which wraps a SigLIP image processor and a Qwen2 tokenizer. """ @@ -36,9 +35,7 @@ class BagelProcessor(ProcessorMixin): images: ImageInput = None, **kwargs: Unpack[BagelProcessorKwargs], ): - """ - Main method to prepare for the model one or several sequences(s) and image(s). - """ + """Prepare one or more sequences and images for the model.""" output_kwargs = self._merge_kwargs( BagelProcessorKwargs, tokenizer_init_kwargs=self.tokenizer.init_kwargs, @@ -62,15 +59,11 @@ class BagelProcessor(ProcessorMixin): return BatchFeature(data={**pixel_values, **text_inputs}) def batch_decode(self, *args, **kwargs): - """ - This method forwards all its arguments to Qwen2TokenizerFast's batch_decode. - """ + """Forward all arguments to `Qwen2TokenizerFast.batch_decode`.""" return self.tokenizer.batch_decode(*args, **kwargs) def decode(self, *args, **kwargs): - """ - This method forwards all its arguments to Qwen2TokenizerFast's decode. - """ + """This method forwards all its arguments to Qwen2TokenizerFast's decode.""" return self.tokenizer.decode(*args, **kwargs) @property diff --git a/vllm/transformers_utils/processors/cohere_asr.py b/vllm/transformers_utils/processors/cohere_asr.py index e1257de4e7..5665d3cbae 100644 --- a/vllm/transformers_utils/processors/cohere_asr.py +++ b/vllm/transformers_utils/processors/cohere_asr.py @@ -228,7 +228,7 @@ class FilterbankFeatures(nn.Module): return self.fb def splice_frames(self, x, frame_splicing): - """Stacks frames together across feature dim + """Stacks frames together across feature dim. input is batch_size, feature_dim, num_frames output is batch_size, feature_dim*frame_splicing, num_frames diff --git a/vllm/transformers_utils/processors/cosmos3_edge.py b/vllm/transformers_utils/processors/cosmos3_edge.py index 95ca2cccd5..a8751cdf6d 100644 --- a/vllm/transformers_utils/processors/cosmos3_edge.py +++ b/vllm/transformers_utils/processors/cosmos3_edge.py @@ -70,8 +70,7 @@ class Cosmos3EdgeImageProcessor(Siglip2ImageProcessor): resample: PILImageResampling | InterpolationMode | int | None = None, **kwargs, ) -> torch.Tensor: - """ - Rescales the image so that the following conditions are met: + """Rescales the image so that the following conditions are met: 1. Both dimensions (height and width) are divisible by 'factor'. 2. The total number of pixels is within the range ['min_pixels', 'max_pixels']. @@ -322,6 +321,7 @@ class Cosmos3EdgeProcessor(Qwen3VLProcessor): normalize, and patch video inputs. chat_template: An optional Jinja template used to render chat messages into a tokenizable string. + """ attributes = ["image_processor", "tokenizer", "video_processor"] diff --git a/vllm/transformers_utils/processors/deepseek_ocr.py b/vllm/transformers_utils/processors/deepseek_ocr.py index 8a9e36448d..55af43a709 100644 --- a/vllm/transformers_utils/processors/deepseek_ocr.py +++ b/vllm/transformers_utils/processors/deepseek_ocr.py @@ -234,9 +234,7 @@ class DeepseekOCRProcessor(ProcessorMixin): images: list[Image.Image], crop_mode: bool = CROP_MODE, ): - """ - - Args: + """Args: prompt (str): the formatted prompt; images (List[ImageType]): the list of images; crop_mode (bool): if True, then crop the image; @@ -248,8 +246,8 @@ class DeepseekOCRProcessor(ProcessorMixin): - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (List[int]): the number of image tokens - """ + """ if prompt is None or images is None: raise ValueError("prompt and images must be used at the same time.") @@ -309,7 +307,6 @@ class DeepseekOCRProcessor(ProcessorMixin): cropping: bool = True, ): """Tokenize text with tags.""" - num_image_tags = conversation.count(self.image_token) if num_image_tags != len(images): raise ValueError( diff --git a/vllm/transformers_utils/processors/deepseek_vl2.py b/vllm/transformers_utils/processors/deepseek_vl2.py index be6d9800a9..6c9966abf8 100644 --- a/vllm/transformers_utils/processors/deepseek_vl2.py +++ b/vllm/transformers_utils/processors/deepseek_vl2.py @@ -186,9 +186,7 @@ class DeepseekVLV2Processor(ProcessorMixin): inference_mode: bool = True, **kwargs: Any, ): - """ - - Args: + """Args: prompt (str): the formatted prompt; images (list[ImageType]): the list of images; inference_mode (bool): if True, then remove the last eos token; @@ -201,8 +199,8 @@ class DeepseekVLV2Processor(ProcessorMixin): - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (list[int]): the number of image tokens - """ + """ if prompt is None or images is None: raise ValueError("prompt and images must be used at the same time.") @@ -276,9 +274,7 @@ class DeepseekVLV2Processor(ProcessorMixin): inference_mode: bool = True, **kwargs: Any, ): - """ - - Args: + """Args: text (str): the formatted prompt; images (list[ImageType]): the list of images; inference_mode (bool): if True, then remove the last eos token; @@ -290,8 +286,8 @@ class DeepseekVLV2Processor(ProcessorMixin): - images (torch.FloatTensor): [n_images, 3, H, W] - image_id (int): the id of the image token - num_image_tokens (list[int]): the number of image tokens - """ + """ prepare = self.process_one( prompt=text, images=images, diff --git a/vllm/transformers_utils/processors/fireredasr2.py b/vllm/transformers_utils/processors/fireredasr2.py index bba7e7ee04..6225e88e6a 100644 --- a/vllm/transformers_utils/processors/fireredasr2.py +++ b/vllm/transformers_utils/processors/fireredasr2.py @@ -68,8 +68,7 @@ class KaldifeatFbank: class FireRedASR2FeatureExtractor(SequenceFeatureExtractor): - r""" - Constructs a FireRedASR2 feature extractor. + r"""Constructs a FireRedASR2 feature extractor. This feature extractor inherits from [`~feature_extraction_sequence_ utils.SequenceFeatureExtractor`] which contains most of the main @@ -99,6 +98,7 @@ class FireRedASR2FeatureExtractor(SequenceFeatureExtractor): Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces the high log_mel_fbank values for signals with hard-zero sections, when VAD cutoff is present in the signal. + """ model_input_names = ["input_features"] @@ -230,9 +230,8 @@ class FireRedASR2FeatureExtractor(SequenceFeatureExtractor): class FireRedASR2Processor(ProcessorMixin): - r""" - Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor and - a FireRedASR2 tokenizer into a single processor. + r"""Constructs a FireRedASR2 processor which wraps a FireRedASR2 feature extractor + and a FireRedASR2 tokenizer into a single processor. [`FireRedASR2Processor`] offers all the functionalities of [`FireRedASR2FeatureExtractor`] and [`Qwen2Tokenizer`]. See the @@ -246,6 +245,7 @@ class FireRedASR2Processor(ProcessorMixin): tokenizer (`Qwen2Tokenizer`): An instance of [`Qwen2Tokenizer`]. The tokenizer is a required input. + """ feature_extractor_class = "FireRedASR2FeatureExtractor" @@ -271,8 +271,7 @@ class FireRedASR2Processor(ProcessorMixin): ) def __call__(self, *args, **kwargs): - """ - Forwards the `audio` argument to FireRedASR2FeatureExtractor's + """Forwards the `audio` argument to FireRedASR2FeatureExtractor's [`~FireRedASR2FeatureExtractor.__call__`] and the `text` argument to [`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the above two methods for more information. diff --git a/vllm/transformers_utils/processors/funasr.py b/vllm/transformers_utils/processors/funasr.py index d7a3c4060c..aaa0bec48c 100644 --- a/vllm/transformers_utils/processors/funasr.py +++ b/vllm/transformers_utils/processors/funasr.py @@ -21,10 +21,7 @@ logger = init_logger(__name__) def apply_cmvn(inputs, cmvn): # noqa - """ - Apply CMVN with mvn data - """ - + """Apply CMVN with mvn data.""" device = inputs.device # dtype = inputs.dtype frame, dim = inputs.shape @@ -219,8 +216,7 @@ class WavFrontend(nn.Module): class FunASRFeatureExtractor(SequenceFeatureExtractor): - r""" - Constructs a FunASR feature extractor. + r"""Constructs a FunASR feature extractor. This feature extractor inherits from [`~feature_extraction_sequence_ utils.SequenceFeatureExtractor`] which contains most of the main @@ -255,6 +251,7 @@ class FunASRFeatureExtractor(SequenceFeatureExtractor): Dithering has similar effect as `spectrogram(mel_floor=...)`. It reduces the high log_mel_fbank values for signals with hard-zero sections, when VAD cutoff is present in the signal. + """ model_input_names = ["input_features"] @@ -369,8 +366,7 @@ class FunASRFeatureExtractor(SequenceFeatureExtractor): class FunASRProcessor(ProcessorMixin): - r""" - Constructs a FunASR processor which wraps a FunASR feature extractor and + r"""Constructs a FunASR processor which wraps a FunASR feature extractor and a FunASR tokenizer into a single processor. [`FunASRProcessor`] offers all the functionalities of @@ -385,6 +381,7 @@ class FunASRProcessor(ProcessorMixin): tokenizer (`Qwen2Tokenizer`): An instance of [`Qwen2Tokenizer`]. The tokenizer is a required input. + """ feature_extractor_class = "FunASRFeatureExtractor" @@ -410,8 +407,7 @@ class FunASRProcessor(ProcessorMixin): ) def __call__(self, *args, **kwargs): - """ - Forwards the `audio` argument to FunASRFeatureExtractor's + """Forwards the `audio` argument to FunASRFeatureExtractor's [`~FunASRFeatureExtractor.__call__`] and the `text` argument to [`~Qwen2Tokenizer.__call__`]. Please refer to the docstring of the above two methods for more information. diff --git a/vllm/transformers_utils/processors/glm4v.py b/vllm/transformers_utils/processors/glm4v.py index a8da239552..e9ee497cea 100644 --- a/vllm/transformers_utils/processors/glm4v.py +++ b/vllm/transformers_utils/processors/glm4v.py @@ -10,8 +10,7 @@ from transformers.processing_utils import ProcessorMixin class GLM4VImageProcessorFast(BaseImageProcessorFast): - """ - Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177 + """Port of https://huggingface.co/zai-org/glm-4v-9b/blob/main/tokenization_chatglm.py#L177 to HF Transformers. """ diff --git a/vllm/transformers_utils/processors/internvl.py b/vllm/transformers_utils/processors/internvl.py index 92c1834c74..84976cb178 100644 --- a/vllm/transformers_utils/processors/internvl.py +++ b/vllm/transformers_utils/processors/internvl.py @@ -358,8 +358,7 @@ class InternVLVideoProcessor(BaseVideoProcessor): class InternVLProcessor(ProcessorMixin): - """ - This model doesn't define its own HF processor, + """This model doesn't define its own HF processor, so we implement our own one here. The code to insert image tokens is based on: diff --git a/vllm/transformers_utils/processors/isaac.py b/vllm/transformers_utils/processors/isaac.py index 2d791df016..7d10e9be86 100644 --- a/vllm/transformers_utils/processors/isaac.py +++ b/vllm/transformers_utils/processors/isaac.py @@ -91,6 +91,7 @@ def get_image_size_for_max_num_patches( `tuple[int, int]`: Height and width (in pixels) that are multiples of `patch_size * pixel_shuffle_scale` and respect both the maximum and optional minimum patch-count constraints. + """ def get_scaled_image_size(scale, original_size, patch_size, pixel_shuffle_scale): @@ -177,9 +178,11 @@ def prepare_image_tensor( The tensor is converted to floating point if needed. scale (`float`, *optional*, defaults to `VISION_SCALE`): Scalar multiplier applied before normalization. + Returns: `torch.Tensor`: Normalized tensor with the same shape as the input and dtype `torch.float32`. + """ if not torch.is_floating_point(image): image = image.float() @@ -209,6 +212,7 @@ def patchify_vision(image: torch.Tensor, patch_size: int) -> torch.Tensor: Raises: ValueError: If `height` or `width` is not divisible by `patch_size`. + """ num_images, height, width, channels = image.shape if height % patch_size or width % patch_size: @@ -265,6 +269,7 @@ def process_vision_for_patches( / patch_size, channels * patch_size**2)` and `dims_virtual` encodes effective `(images, height, width)` dimensions after optional pixel shuffling. + """ # Add batch dim if single image if images.dim() == 3: diff --git a/vllm/transformers_utils/processors/kimi_k25.py b/vllm/transformers_utils/processors/kimi_k25.py index edee9734ce..13aed85de4 100644 --- a/vllm/transformers_utils/processors/kimi_k25.py +++ b/vllm/transformers_utils/processors/kimi_k25.py @@ -28,8 +28,7 @@ class KimiK25Processor(ProcessorMixin): return_tensors: str | TensorType | None = None, **kwargs, ) -> BatchFeature: - """ - Args: + """Args: text: The text to be field to the model. vision_chunks: List of `VisionChunk` items to be processed. For image: `VisionChunkImage` with @@ -44,6 +43,7 @@ class KimiK25Processor(ProcessorMixin): Returned when `vision_chunks` is not `None`. - **grid_thws** -- list of image 3D grid in LLM. Returned when `vision_chunks` is not `None`. + """ if vision_chunks is not None: mm_inputs = self.image_processor.preprocess( diff --git a/vllm/transformers_utils/processors/mimo_v2_omni.py b/vllm/transformers_utils/processors/mimo_v2_omni.py index 22349f7513..0555c941b2 100644 --- a/vllm/transformers_utils/processors/mimo_v2_omni.py +++ b/vllm/transformers_utils/processors/mimo_v2_omni.py @@ -1071,6 +1071,8 @@ class MiMoOmniProcessor(ProcessorMixin): tuples (pre-decoded). audio: list of ``str`` (path/url/base64), ``bytes``, ``(waveform_1D, sample_rate)`` tuples, or ``np.ndarray``. + video_audio: audio track extracted from ``videos``, in the same + formats accepted by ``audio``. return_tensors: Passed to :class:`BatchFeature`. Returns: @@ -1079,6 +1081,7 @@ class MiMoOmniProcessor(ProcessorMixin): - ``pixel_values`` + ``image_grid_thw`` - ``pixel_values_videos`` + ``video_grid_thw`` + ``second_per_grid_ts`` - ``audio_features`` + """ if isinstance(text, list): text = text[0] if len(text) == 1 else "\n".join(text) diff --git a/vllm/transformers_utils/processors/minicpmo.py b/vllm/transformers_utils/processors/minicpmo.py index d9ac841cd8..e59a50ccbe 100644 --- a/vllm/transformers_utils/processors/minicpmo.py +++ b/vllm/transformers_utils/processors/minicpmo.py @@ -14,9 +14,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Processor class for MiniCPMO. -""" +"""Processor class for MiniCPMO.""" import math from typing import Literal, TypeAlias @@ -35,8 +33,7 @@ MiniCPMOBatchFeature: TypeAlias = BatchFeature class MiniCPMOProcessor(ProcessorMixin): - r""" - Constructs a MiniCPMV processor which wraps a MiniCPMV image + r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image processor and a MiniCPMV tokenizer into a single processor. [`MiniCPMVProcessor`] offers all the functionalities of @@ -49,6 +46,7 @@ class MiniCPMOProcessor(ProcessorMixin): The image processor is a required input. tokenizer ([`LlamaTokenizerWrapper`], *optional*): The tokenizer is a required input. + """ attributes = ["image_processor", "feature_extractor", "tokenizer"] @@ -253,8 +251,7 @@ class MiniCPMOProcessor(ProcessorMixin): # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode # with CLIP->Llama def batch_decode(self, *args, **kwargs): - """ - This method forwards all its arguments to LlamaTokenizerFast's + """This method forwards all its arguments to LlamaTokenizerFast's [`~PythonBackend.batch_decode`]. Please refer to the docstring of this method for more information. """ @@ -274,8 +271,7 @@ class MiniCPMOProcessor(ProcessorMixin): # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode # with CLIP->Llama def decode(self, *args, **kwargs): - """ - This method forwards all its arguments to LlamaTokenizerFast's + """This method forwards all its arguments to LlamaTokenizerFast's [`~PythonBackend.decode`]. Please refer to the docstring of this method for more information. """ @@ -551,15 +547,11 @@ class MelSpectrogramFeatures(torch.nn.Module): ) def __call__(self, audio: torch.Tensor) -> torch.Tensor: - """ - audio: Tensor([num_channels, num_samples]) - """ + """audio: Tensor([num_channels, num_samples]).""" return super().__call__(audio) def forward(self, audio: torch.Tensor) -> torch.Tensor: - """ - audio: Tensor([num_channels, num_samples]) - """ + """audio: Tensor([num_channels, num_samples]).""" mel: torch.Tensor = self.mel_spec(audio) features = torch.log(torch.clip(mel, min=1e-5)) return features diff --git a/vllm/transformers_utils/processors/minicpmv.py b/vllm/transformers_utils/processors/minicpmv.py index 712cbbb47a..aaedaa3767 100644 --- a/vllm/transformers_utils/processors/minicpmv.py +++ b/vllm/transformers_utils/processors/minicpmv.py @@ -13,9 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Processor class for MiniCPMV. -""" +"""Processor class for MiniCPMV.""" from typing import TypeAlias @@ -36,8 +34,7 @@ MiniCPMVBatchFeature: TypeAlias = BatchFeature class MiniCPMVProcessor(ProcessorMixin): - r""" - Constructs a MiniCPMV processor which wraps a MiniCPMV image + r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image processor and a MiniCPMV tokenizer into a single processor. [`MiniCPMVProcessor`] offers all the functionalities of @@ -50,6 +47,7 @@ class MiniCPMVProcessor(ProcessorMixin): The image processor is a required input. tokenizer ([`LlamaTokenizerWrapper`], *optional*): The tokenizer is a required input. + """ attributes = ["image_processor", "tokenizer"] @@ -93,8 +91,7 @@ class MiniCPMVProcessor(ProcessorMixin): # Copied from transformers.models.clip.processing_clip.CLIPProcessor # .batch_decode with CLIP->Llama def batch_decode(self, *args, **kwargs): - """ - This method forwards all its arguments to LlamaTokenizerFast's + """This method forwards all its arguments to LlamaTokenizerFast's [`~PythonBackend.batch_decode`]. Please refer to the docstring of this method for more information. """ @@ -126,8 +123,7 @@ class MiniCPMVProcessor(ProcessorMixin): # Copied from transformers.models.clip.processing_clip.CLIPProcessor # .decode with CLIP->Llama def decode(self, *args, **kwargs): - """ - This method forwards all its arguments to LlamaTokenizerFast's + """This method forwards all its arguments to LlamaTokenizerFast's [`~PythonBackend.decode`]. Please refer to the docstring of this method for more information. """ diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py index ae6833b200..22bdeef21a 100644 --- a/vllm/transformers_utils/processors/moondream3.py +++ b/vllm/transformers_utils/processors/moondream3.py @@ -67,8 +67,7 @@ def select_tiling( class Moondream3Processor(ProcessorMixin): - """ - Constructs a Moondream3 processor which handles image preprocessing + """Constructs a Moondream3 processor which handles image preprocessing and tokenization for the Moondream3 multimodal model. Args: @@ -78,6 +77,7 @@ class Moondream3Processor(ProcessorMixin): max_crops: Maximum number of crops per image. overlap_margin: Margin for overlapping crops in patches. patch_size: Size of each patch. + """ attributes = ["tokenizer"] @@ -189,8 +189,7 @@ class Moondream3Processor(ProcessorMixin): pretrained_model_name_or_path, **kwargs, ): - """ - Load the processor, using a separate tokenizer repo. + """Load the processor, using a separate tokenizer repo. The moondream3 model uses a custom tokenizer from 'moondream/starmie-v1' instead of having tokenizer files in the model repo. @@ -294,8 +293,7 @@ class Moondream3Processor(ProcessorMixin): | list[PreTokenizedInput] = None, **kwargs: Unpack[Moondream3ProcessorKwargs], ) -> BatchFeature: - """ - Process images and text for Moondream3 model. + """Process images and text for Moondream3 model. Args: images: Input images (PIL Image, numpy array, or list thereof). @@ -304,6 +302,7 @@ class Moondream3Processor(ProcessorMixin): Returns: BatchFeature with processed inputs. + """ output_kwargs = self._merge_kwargs( Moondream3ProcessorKwargs, @@ -428,8 +427,7 @@ class Moondream3Processor(ProcessorMixin): convert_to_rgb: bool = True, return_tensors: str = "pt", ) -> tuple[torch.Tensor, tuple[int, int]]: - """ - Preprocess an image using overlap-and-resize cropping strategy. + """Preprocess an image using overlap-and-resize cropping strategy. Args: image: Input PIL image, numpy array, or torch tensor. @@ -442,6 +440,7 @@ class Moondream3Processor(ProcessorMixin): Returns: Tuple of (pixel_values tensor, tiling tuple). + """ image = self._to_pil_image(image) if convert_to_rgb: diff --git a/vllm/transformers_utils/processors/nano_nemotron_vl.py b/vllm/transformers_utils/processors/nano_nemotron_vl.py index 3cddbec6c8..3d4d432de2 100644 --- a/vllm/transformers_utils/processors/nano_nemotron_vl.py +++ b/vllm/transformers_utils/processors/nano_nemotron_vl.py @@ -292,8 +292,7 @@ class DynamicResolutionImageTiler: self, target_num_tokens_post_shuffle: int, ) -> tuple[int, int]: - """ - TODO: optimize this so it squeezes closer to target number of tokens. + """TODO: optimize this so it squeezes closer to target number of tokens. Calculate image dimensions that produce approximately `target` tokens after pixel_shuffle. @@ -318,6 +317,7 @@ class DynamicResolutionImageTiler: ... height // PATCH_SIZE ... ) // 2**2 == 8100 # tokens post-shuffle >>> assert tiler._get_num_embeddings(width=width, height=height) == 8100 + """ side_pixels = ( math.isqrt(target_num_tokens_post_shuffle) @@ -386,6 +386,7 @@ class DynamicResolutionImageTiler: num_tokens_available: Number of tokens available for this media Returns: DynamicResolutionParams for the media + """ current_num_tokens_available = num_tokens_available assert isinstance(media, Image.Image), ( @@ -473,6 +474,7 @@ class DynamicResolutionImageTiler: num_tokens_available: Total number of tokens available across all media Returns: List of ImageTilingParams for each media item + """ num_tokens_available = ( num_tokens_available @@ -571,8 +573,7 @@ class DynamicResolutionImageTiler: class BaseNanoNemotronVLProcessor(ABC): - """ - This model doesn't define its own HF processor, + """This model doesn't define its own HF processor, so we implement our own one here. The code to insert image tokens is based on: @@ -761,8 +762,7 @@ class BaseNanoNemotronVLProcessor(ABC): class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor): - """ - HF Processor with extended video processing logic. + """HF Processor with extended video processing logic. Code for video processing is adapted from video example: https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers """ @@ -1125,8 +1125,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor): img_context_token_ids: list[int], video_temporal_patch_size: int = 1, ) -> PromptUpdateDetails: - """ - Build prompt replacement for a video. + """Build prompt replacement for a video. The replacement returned is not actually used to replace the placeholder tokens - it's just used to make sure we allocate the correct number of tokens. diff --git a/vllm/transformers_utils/processors/nemotron_vl.py b/vllm/transformers_utils/processors/nemotron_vl.py index 9c5436aa27..9504f5ee2b 100644 --- a/vllm/transformers_utils/processors/nemotron_vl.py +++ b/vllm/transformers_utils/processors/nemotron_vl.py @@ -191,8 +191,7 @@ class LlamaNemotronNanoVLImageProcessor(InternVLImageProcessor): class LlamaNemotronNanoVLProcessor(InternVLProcessor): - """ - This model doesn't define its own HF processor, + """This model doesn't define its own HF processor, so we implement our own one here. The image processor is given by: @@ -287,8 +286,7 @@ class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor): class LlamaNemotronVLEmbedProcessor(InternVLProcessor): - """ - Processor for LlamaNemotronVL embedding model. + """Processor for LlamaNemotronVL embedding model. Inherits from NemotronVLProcessor and specializes it for embedding tasks: - Uses SigLIP transform with normalization instead of base transform diff --git a/vllm/transformers_utils/processors/ovis.py b/vllm/transformers_utils/processors/ovis.py index 907c68035f..a77072745f 100644 --- a/vllm/transformers_utils/processors/ovis.py +++ b/vllm/transformers_utils/processors/ovis.py @@ -50,10 +50,10 @@ class OvisProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-a class OvisProcessor(ProcessorMixin): - r""" - Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor. + r"""Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor. [`OvisProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] for more information. + Args: image_processor ([`Qwen2VLImageProcessor`], *optional*): The image processor is a required input. @@ -61,6 +61,7 @@ class OvisProcessor(ProcessorMixin): The tokenizer is a required input. chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages in a chat into a tokenizable string. + """ attributes = ["image_processor", "tokenizer"] @@ -107,12 +108,12 @@ class OvisProcessor(ProcessorMixin): | list[PreTokenizedInput] = None, **kwargs: Unpack[OvisProcessorKwargs], ) -> BatchFeature: - """ - Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` + """Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text` and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. - Args: + + Args: images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`): The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch tensor. Both channels-first and channels-last formats are supported. @@ -129,7 +130,8 @@ class OvisProcessor(ProcessorMixin): - `'pt'`: Return PyTorch `torch.Tensor` objects. - `'np'`: Return NumPy `np.ndarray` objects. - `'jax'`: Return JAX `jnp.ndarray` objects. - Returns: + + Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when @@ -140,8 +142,8 @@ class OvisProcessor(ProcessorMixin): - **image_grid_thw** -- List of image 3D grid in LLM. Returned when `images` is not `None`. - **video_grid_thw** -- List of video 3D grid in LLM. Returned when `videos` is not `None`. - **second_per_grid_ts** -- List of video seconds per time grid. Returned when `videos` is not `None`. - """ + """ max_partition = kwargs.pop("max_partition", 9) covering_threshold = kwargs.pop("covering_threshold", 0.9) @@ -416,28 +418,28 @@ class OvisProcessor(ProcessorMixin): return torch.tensor(pixel_values), image_placeholders, torch.tensor(grid) def batch_decode(self, *args, **kwargs): - """ - This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please + """This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.batch_decode`]. Please refer to the docstring of this method for more information. """ return self.tokenizer.batch_decode(*args, **kwargs) def decode(self, *args, **kwargs): - """ - This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to + """This method forwards all its arguments to Qwen2TokenizerFast's [`~PythonBackend.decode`]. Please refer to the docstring of this method for more information. """ return self.tokenizer.decode(*args, **kwargs) def post_process_image_text_to_text(self, generated_outputs): - """ - Post-process the output of the model to decode the text. + """Post-process the output of the model to decode the text. + Args: generated_outputs (`torch.Tensor` or `np.ndarray`): The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)` or `(sequence_length,)`. + Returns: `list[str]`: The decoded text. + """ return self.tokenizer.batch_decode( generated_outputs, diff --git a/vllm/transformers_utils/processors/ovis2_5.py b/vllm/transformers_utils/processors/ovis2_5.py index b9c2d11240..c31ea91520 100644 --- a/vllm/transformers_utils/processors/ovis2_5.py +++ b/vllm/transformers_utils/processors/ovis2_5.py @@ -33,13 +33,13 @@ class Ovis2_5ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[cal class Ovis2_5Processor(ProcessorMixin): - r""" - Constructs an Ovis processor which wraps an Ovis image processor + r"""Constructs an Ovis processor which wraps an Ovis image processor and a Qwen2 tokenizer into a single processor. [`OvisProcessor`] offers all the functionalities of [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] for more information. + Args: image_processor ([`Qwen2VLImageProcessor`], *optional*): The image processor is a required input. @@ -48,6 +48,7 @@ class Ovis2_5Processor(ProcessorMixin): chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages in a chat into a tokenizable string. + """ attributes = ["image_processor", "tokenizer"] @@ -110,43 +111,39 @@ class Ovis2_5Processor(ProcessorMixin): | list[PreTokenizedInput] = None, **kwargs: Unpack[Ovis2_5ProcessorKwargs], ) -> BatchFeature: - """ - Main method to prepare for the model one or several sequences(s) + """Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`and `kwargs` arguments to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` is not `None` to encode the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`. - Args: - images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, - `list[PIL.Image.Image]`, `list[np.ndarray]`, - `list[torch.Tensor]`): - The image or batch of images to be prepared. - Each image can be a PIL image, NumPy array or PyTorch - tensor. Both channels-first and channels-last formats - are supported. - text (`str`, `list[str]`, `list[list[str]]`): - The sequence or batch of sequences to be encoded. - Each sequence can be a string or a list of strings - (pretokenized string). If the sequences are provided as - list of strings (pretokenized), you must set - `is_split_into_words=True` (to lift the ambiguity with - a batch of sequences). - videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, - `list[torch.Tensor]`): - The image or batch of videos to be prepared. Each video - can be a 4D NumPy array or PyTorch tensor, or a nested - list of 3D frames. Both channels-first and channels-last - formats are supported. - return_tensors (`str` or [`~utils.TensorType`], *optional*): - If set, will return tensors of a particular framework. - Acceptable values are: - - `'tf'`: Return TensorFlow `tf.constant` objects. - - `'pt'`: Return PyTorch `torch.Tensor` objects. - - `'np'`: Return NumPy `np.ndarray` objects. - - `'jax'`: Return JAX `jnp.ndarray` objects. - Returns: + + Args: + images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, or a list): + The image or batch of images to be prepared. Each image can + be a PIL image, NumPy array or PyTorch tensor. Both + channels-first and channels-last formats are supported. + videos (`np.ndarray`, `torch.Tensor`, or a list of either): + The video or batch of videos to be prepared. Each video can + be a 4D NumPy array or PyTorch tensor, or a nested list of + 3D frames. Both channels-first and channels-last formats are + supported. + text (`str`, `list[str]`, `list[list[str]]`): + The sequence or batch of sequences to be encoded. Each + sequence can be a string or a list of strings (pretokenized + string). If the sequences are provided as list of strings + (pretokenized), you must set `is_split_into_words=True` (to + lift the ambiguity with a batch of sequences). + return_tensors (`str` or [`~utils.TensorType`], *optional*): + If set, will return tensors of a particular framework. + Acceptable values are: + - `'tf'`: Return TensorFlow `tf.constant` objects. + - `'pt'`: Return PyTorch `torch.Tensor` objects. + - `'np'`: Return NumPy `np.ndarray` objects. + - `'jax'`: Return JAX `jnp.ndarray` objects. + + Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- list of token ids to be fed to a model. Returned when `text` is not `None`. @@ -164,6 +161,7 @@ class Ovis2_5Processor(ProcessorMixin): when `videos` is not `None`. - **second_per_grid_ts** -- list of video seconds per time grid. Returned when `videos` is not `None`. + """ output_kwargs = self._merge_kwargs( Ovis2_5ProcessorKwargs, diff --git a/vllm/transformers_utils/processors/pixtral.py b/vllm/transformers_utils/processors/pixtral.py index 588ad6b7fa..85604eb5b7 100644 --- a/vllm/transformers_utils/processors/pixtral.py +++ b/vllm/transformers_utils/processors/pixtral.py @@ -11,8 +11,7 @@ from vllm.tokenizers.mistral import MistralTokenizer class MistralCommonImageProcessor(ImageProcessingMixin): - """ - Provide a HF-compatible interface for + """Provide a HF-compatible interface for `mistral_common.tokens.tokenizers.multimodal.ImageEncoder`. """ diff --git a/vllm/transformers_utils/processors/voxtral.py b/vllm/transformers_utils/processors/voxtral.py index 5403a7e47f..4cc23c4775 100644 --- a/vllm/transformers_utils/processors/voxtral.py +++ b/vllm/transformers_utils/processors/voxtral.py @@ -18,8 +18,7 @@ from vllm.tokenizers.mistral import MistralTokenizer class MistralCommonFeatureExtractor(SequenceFeatureExtractor): - """ - Provide a HF-compatible interface for + """Provide a HF-compatible interface for `mistral_common.tokens.tokenizers.multimodal.AudioEncoder`. """ diff --git a/vllm/transformers_utils/repo_utils.py b/vllm/transformers_utils/repo_utils.py index ca20390aad..6b9ff9ec5e 100644 --- a/vllm/transformers_utils/repo_utils.py +++ b/vllm/transformers_utils/repo_utils.py @@ -66,6 +66,7 @@ def resolve_revision( Returns: The resolved revision, or `revision` unchanged if it cannot be resolved (local path, ModelScope, or any Hub error). + """ if Path(repo_id).exists() or envs.VLLM_USE_MODELSCOPE: return revision @@ -106,6 +107,7 @@ def with_retry( retry_delay: Seconds to wait after the first failure, doubled each time. fatal_errors: Exceptions that are a definitive answer rather than a transient failure. These are raised immediately, without logging. + """ for attempt in range(max_retries): try: @@ -353,8 +355,7 @@ def get_hf_file_bytes( def try_get_local_file( model: str | Path, file_name: str, revision: str | None = "main" ) -> Path | Any | None: - """ - Try to get a local file from the HuggingFace repository. + """Try to get a local file from the HuggingFace repository. The possible return values are: @@ -385,20 +386,19 @@ def try_get_local_file( def get_hf_file_to_dict( file_name: str, model: str | Path, revision: str | None = "main" ): - """ - Downloads a file from the Hugging Face Hub and returns + """Downloads a file from the Hugging Face Hub and returns its contents as a dictionary. - Parameters: - - file_name (str): The name of the file to download. - - model (str): The name of the model on the Hugging Face Hub. - - revision (str): The specific version of the model. + Args: + file_name (str): The name of the file to download. + model (str): The name of the model on the Hugging Face Hub. + revision (str): The specific version of the model. Returns: - config_dict (dict): A dictionary containing the contents of the downloaded file. - """ + """ file_path = try_get_local_file(model=model, file_name=file_name, revision=revision) if file_path is None: diff --git a/vllm/transformers_utils/runai_utils.py b/vllm/transformers_utils/runai_utils.py index 248ede6a6f..d02992b10b 100644 --- a/vllm/transformers_utils/runai_utils.py +++ b/vllm/transformers_utils/runai_utils.py @@ -25,14 +25,14 @@ except ImportError: def list_safetensors(path: str = "") -> list[str]: - """ - List full file names from object path and filter by allow pattern. + """List full file names from object path and filter by allow pattern. Args: path: The object storage path to list from. Returns: list[str]: List of full object storage paths allowed by the pattern + """ return runai_list_safetensors(path) @@ -42,8 +42,7 @@ def is_runai_obj_uri(model_or_path: str) -> bool: class ObjectStorageModel: - """ - A class representing an ObjectStorage model mirrored into a + """A class representing an ObjectStorage model mirrored into a temporary directory. Attributes: @@ -51,6 +50,7 @@ class ObjectStorageModel: Methods: pull_files(): Pull model from object storage to the temporary directory. + """ def __init__(self, url: str) -> None: @@ -86,8 +86,7 @@ class ObjectStorageModel: allow_pattern: list[str] | None = None, ignore_pattern: list[str] | None = None, ) -> None: - """ - Pull files from object storage into the temporary directory. + """Pull files from object storage into the temporary directory. Args: model_path: The object storage path of the model. diff --git a/vllm/transformers_utils/s3_utils.py b/vllm/transformers_utils/s3_utils.py index 7dfee9fabe..d0f403c690 100644 --- a/vllm/transformers_utils/s3_utils.py +++ b/vllm/transformers_utils/s3_utils.py @@ -36,8 +36,7 @@ def glob( path: str = "", allow_pattern: list[str] | None = None, ) -> list[str]: - """ - List full file names from S3 path and filter by allow pattern. + """List full file names from S3 path and filter by allow pattern. Args: s3: S3 client to use. @@ -46,6 +45,7 @@ def glob( Returns: list[str]: List of full S3 paths allowed by the pattern + """ if s3 is None: s3 = boto3.client("s3") @@ -61,8 +61,7 @@ def list_files( allow_pattern: list[str] | None = None, ignore_pattern: list[str] | None = None, ) -> tuple[str, str, list[str]]: - """ - List files from S3 path and filter by pattern. + """List files from S3 path and filter by pattern. Args: s3: S3 client to use. @@ -77,6 +76,7 @@ def list_files( and the prefix as a dir like string - The third element is a list of files allowed or disallowed by pattern + """ parts = path.removeprefix("s3://").split("/") prefix = "/".join(parts[1:]) diff --git a/vllm/transformers_utils/utils.py b/vllm/transformers_utils/utils.py index 674f6a05bf..ad07d901b4 100644 --- a/vllm/transformers_utils/utils.py +++ b/vllm/transformers_utils/utils.py @@ -80,16 +80,15 @@ def _maybe_space_split_dict(path: str | PathLike) -> dict[str, str]: @cache def maybe_model_redirect(model: str) -> str: - """ - Use model_redirect to redirect the model name to a local folder. + """Use model_redirect to redirect the model name to a local folder. Args: model: hf model name Returns: maybe redirect to a local folder - """ + """ model_redirect_path = envs.VLLM_MODEL_REDIRECT_PATH if not model_redirect_path: diff --git a/vllm/utils/argparse_utils.py b/vllm/utils/argparse_utils.py index fc906cf227..e573cbc293 100644 --- a/vllm/utils/argparse_utils.py +++ b/vllm/utils/argparse_utils.py @@ -34,6 +34,7 @@ def human_readable_int(value: str) -> int: - '1k' -> 1,000 - '1K' -> 1,024 - '25.6k' -> 25,600 + """ value = value.strip() @@ -82,6 +83,7 @@ def human_readable_int_or_auto(value: str) -> int: - '1K' -> 1,024 - '25.6k' -> 25,600 - '-1' or 'auto' -> -1 (special value for auto-detection) + """ value = value.strip() @@ -95,8 +97,7 @@ class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpForma """SortedHelpFormatter that sorts arguments by their option strings.""" def _split_lines(self, text, width): - """ - 1. Sentences split across lines have their single newlines removed. + """1. Sentences split across lines have their single newlines removed. 2. Paragraphs and lists are split into separate lines. 3. Each line is wrapped to the specified width (width of terminal). """ @@ -466,7 +467,7 @@ class FlexibleArgumentParser(ArgumentParser): The arguments in config file will be inserted between the argument list. - example: + Example: ```yaml port: 12323 tensor-parallel-size: 4 @@ -492,6 +493,7 @@ class FlexibleArgumentParser(ArgumentParser): Please note how the config args are inserted after the sub command. this way the order of priorities is maintained when these are args parsed by super(). + """ assert args.count("--config") <= 1, "More than one config file specified!" diff --git a/vllm/utils/async_utils.py b/vllm/utils/async_utils.py index 3faf728384..b2c0d57dc9 100644 --- a/vllm/utils/async_utils.py +++ b/vllm/utils/async_utils.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Contains helpers related to asynchronous code. +"""Contains helpers related to asynchronous code. This is similar in concept to the `asyncio` module. """ @@ -29,8 +28,7 @@ def make_async( func: Callable[P, T], executor: Executor | None = None, ) -> Callable[P, Awaitable[T]]: - """ - Take a blocking function, and run it on in an executor thread. + """Take a blocking function, and run it on in an executor thread. This function prevents the blocking function from blocking the asyncio event loop. @@ -49,8 +47,7 @@ def make_async_with_semaphore( func: Callable[P, T], executor: ThreadPoolExecutor, ) -> Callable[P, Awaitable[T]]: - """ - Take a blocking function, and run it on in an executor thread. + """Take a blocking function, and run it on in an executor thread. This function prevents the blocking function from blocking the asyncio event loop. @@ -59,7 +56,6 @@ def make_async_with_semaphore( The function is wrapped in a semaphore to limit the number of concurrent executions making it easier to cancel tasks before they start. """ - semaphore = asyncio.Semaphore(executor._max_workers) async def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T: diff --git a/vllm/utils/cache.py b/vllm/utils/cache.py index 791b8d848d..9fca5ec465 100644 --- a/vllm/utils/cache.py +++ b/vllm/utils/cache.py @@ -102,8 +102,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]): return self.currsize / self.maxsize def stat(self, *, delta: bool = False) -> CacheInfo: - """ - Gets the cumulative number of hits and queries against this cache. + """Gets the cumulative number of hits and queries against this cache. If `delta=True`, instead gets these statistics since the last call that also passed `delta=True`. @@ -165,6 +164,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]): Returns: `True` if the item was cached, otherwise `False`. + """ try: self[key] = value @@ -175,8 +175,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]): return True def pin(self, key: _K) -> None: - """ - Pins a key in the cache preventing it from being + """Pins a key in the cache preventing it from being evicted in the LRU order. """ if key not in self: @@ -184,8 +183,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]): self.pinned_items.add(key) def _unpin(self, key: _K) -> None: - """ - Unpins a key in the cache allowing it to be + """Unpins a key in the cache allowing it to be evicted in the LRU order. """ self.pinned_items.remove(key) diff --git a/vllm/utils/collection_utils.py b/vllm/utils/collection_utils.py index 5852aff549..de32ed0c15 100644 --- a/vllm/utils/collection_utils.py +++ b/vllm/utils/collection_utils.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Contains helpers that are applied to collections. +"""Contains helpers that are applied to collections. This is similar in concept to the `collections` module. """ @@ -20,8 +19,7 @@ _V = TypeVar("_V") class LazyDict(Mapping[str, _V], Generic[_V]): - """ - Evaluates dictionary items only when they are accessed. + """Evaluates dictionary items only when they are accessed. Adapted from: https://stackoverflow.com/a/47212782/5082708 """ @@ -116,8 +114,7 @@ def flatten_2d_lists(lists: Iterable[Iterable[T]]) -> list[T]: def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]): - """ - Unlike [`itertools.groupby`][], groups are not broken by + """Unlike [`itertools.groupby`][], groups are not broken by non-contiguous data. """ groups = defaultdict[_K, list[_V]](list) diff --git a/vllm/utils/counter.py b/vllm/utils/counter.py index c2dce32e97..6251cdcfb1 100644 --- a/vllm/utils/counter.py +++ b/vllm/utils/counter.py @@ -19,10 +19,10 @@ class Counter: class AtomicCounter: - """An atomic, thread-safe counter""" + """An atomic, thread-safe counter.""" def __init__(self, initial: int = 0) -> None: - """Initialize a new atomic counter to given initial value""" + """Initialize a new atomic counter to given initial value.""" super().__init__() self._value = initial @@ -33,13 +33,13 @@ class AtomicCounter: return self._value def inc(self, num: int = 1) -> int: - """Atomically increment the counter by num and return the new value""" + """Atomically increment the counter by num and return the new value.""" with self._lock: self._value += num return self._value def dec(self, num: int = 1) -> int: - """Atomically decrement the counter by num and return the new value""" + """Atomically decrement the counter by num and return the new value.""" with self._lock: self._value -= num return self._value diff --git a/vllm/utils/cpu_resource_utils.py b/vllm/utils/cpu_resource_utils.py index a1e7ab0ee3..296fe46646 100644 --- a/vllm/utils/cpu_resource_utils.py +++ b/vllm/utils/cpu_resource_utils.py @@ -128,6 +128,7 @@ def check_cgroup_memory_available( Low headroom logs a warning, but does not reject the allocation because cgroup usage can include reclaimable memory. If the cgroup limit or usage cannot be read, the check is skipped. + """ cgroup_limit = get_cgroup_memory_limit() cgroup_usage = get_cgroup_memory_usage() @@ -171,7 +172,7 @@ def get_memory_affinity(pid: int = 0) -> list[int]: def parse_id_list(raw_str: str) -> list[int]: - """Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8]""" + """Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8].""" result: list[int] = [] if not raw_str: return result diff --git a/vllm/utils/cpu_triton_utils.py b/vllm/utils/cpu_triton_utils.py index 9ce33b8357..d1696317aa 100644 --- a/vllm/utils/cpu_triton_utils.py +++ b/vllm/utils/cpu_triton_utils.py @@ -1,9 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Contains replacement functions to fallback Triton usages in CPU backend -""" +"""Contains replacement functions to fallback Triton usages in CPU backend.""" import ctypes from collections.abc import Callable diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 6fef007224..3d96b496f1 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -64,7 +64,7 @@ class DeepGemmQuantScaleFMT(Enum): @classmethod def init_oracle_cache(cls) -> None: - """Initialize the oracle decision and store it in the class cache""" + """Initialize the oracle decision and store it in the class cache.""" cached = getattr(cls, "_oracle_cache", None) if cached is not None: return @@ -426,7 +426,7 @@ def mk_alignment_scope(value: int): def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor: - """Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor""" + """Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor.""" _lazy_init() if _get_mn_major_tma_aligned_tensor_impl is None: return _missing() @@ -568,6 +568,7 @@ def fp8_fp4_mqa_logits( Returns: Logits tensor of shape [M, N], dtype `torch.float32`. + """ _lazy_init() if _fp8_fp4_mqa_logits_impl is None: @@ -623,6 +624,7 @@ def get_paged_mqa_logits_metadata( Returns: Tensor of shape [slots + 1, 2] consumed by `fp8_fp4_paged_mqa_logits` to schedule work across SMs. + """ _lazy_init() if _get_paged_mqa_logits_metadata_impl is None: @@ -674,6 +676,7 @@ def fp8_fp4_paged_mqa_logits( Returns: Logits tensor of shape [B * next_n, max_model_len], dtype `torch.float32`. + """ _lazy_init() if _fp8_fp4_paged_mqa_logits_impl is None: @@ -740,6 +743,7 @@ def get_sparse_mqa_logits_metadata( qk_dtype: dtype of the packed Q values (``torch.int8`` for MXFP4, ``torch.float8_e4m3fn`` for FP8). sparse_block_kv: Tokens per sparse block, 8 or 16. + """ _lazy_init() if _get_sparse_mqa_logits_metadata_impl is None: @@ -777,6 +781,7 @@ def get_paged_sparse_mqa_logits_metadata( block ids, [num_q_tokens, num_max_sparse_blocks], int32. qk_dtype: dtype of the packed Q values. sparse_block_kv: Tokens per sparse block, 8 or 16. + """ _lazy_init() if _get_paged_sparse_mqa_logits_metadata_impl is None: @@ -817,6 +822,7 @@ def fp8_fp4_sparse_mqa_logits( bf16 logits of shape [M, num_max_sparse_blocks * sparse_block_kv]; column ``j * sparse_block_kv + o`` scores the token at ``sparse_kv_block_indices[row, j] * sparse_block_kv + ks % sbk + o``. + """ _lazy_init() if _fp8_fp4_sparse_mqa_logits_impl is None: @@ -848,10 +854,13 @@ def fp8_fp4_paged_sparse_mqa_logits( page stride 512B-aligned. weights: [num_q_tokens, H] ``torch.bfloat16``. metadata: From `get_paged_sparse_mqa_logits_metadata`. + num_max_sparse_blocks: Candidate blocks per row. + sparse_block_kv: Tokens per sparse block, 8 or 16. Returns: bf16 logits of shape [num_q_tokens, num_max_sparse_blocks * sparse_block_kv]. + """ _lazy_init() if _fp8_fp4_paged_sparse_mqa_logits_impl is None: @@ -873,8 +882,7 @@ def tf32_hc_prenorm_gemm( sqrsum: torch.Tensor, num_split: int, ) -> torch.Tensor: - """ - Perform the following computation: + """Perform the following computation: out = x.float() @ fn.T sqrsum = x.float().square().sum(-1) @@ -950,7 +958,6 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor): and report `1 - sim`. Once kernel accuracy improves this helper can be removed. """ - x, y = x.double(), y.double() denominator = (x * x + y * y).sum() sim = 2 * (x * y).sum() / denominator diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 7c76dd7ba2..7647179134 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -589,8 +589,7 @@ def supports_trtllm_attention(is_prefill: bool = False) -> bool: def force_use_trtllm_attention() -> bool | None: - """ - This function should only be called during initialization stage when vllm config + """This function should only be called during initialization stage when vllm config is set. Return `None` if --attention-config.use_trtllm_attention is not set, return `True` if TRTLLM attention is forced to be used, @@ -628,7 +627,6 @@ def use_trtllm_attention( has_spec: bool = False, ) -> bool: """Return `True` if TRTLLM attention is used.""" - # CLI argument is set to 0 - respect it if force_use_trtllm is not None and not force_use_trtllm: return False @@ -736,6 +734,7 @@ if has_flashinfer(): k_nope: The nope part of k, shape [num_tokens, num_heads, nope_dim]. k_pe: The rope part of k (shared), shape [num_tokens, 1, rope_dim]. This is broadcast to all heads. + """ from flashinfer.concat_ops import concat_mla_k diff --git a/vllm/utils/func_utils.py b/vllm/utils/func_utils.py index 5ce23e6a00..69600decda 100644 --- a/vllm/utils/func_utils.py +++ b/vllm/utils/func_utils.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Contains helpers that are applied to functions. +"""Contains helpers that are applied to functions. This is similar in concept to the `functools` module. """ @@ -127,8 +126,7 @@ def get_allowed_kwarg_only_overrides( requires_kw_only: bool = True, allow_var_kwargs: bool = False, ) -> dict[str, Any]: - """ - Given a callable which has one or more keyword only params and a dict + """Given a callable which has one or more keyword only params and a dict mapping param names to values, drop values that can be not be kwarg expanded to overwrite one or more keyword-only args. This is used in a few places to handle custom processor overrides for multimodal models, @@ -139,12 +137,14 @@ def get_allowed_kwarg_only_overrides( callable: Callable which takes 0 or more keyword only arguments. If None is provided, all overrides names are allowed. overrides: Potential overrides to be used when invoking the callable. + requires_kw_only: If True, only keyword-only params may be overridden. allow_var_kwargs: Allows overrides that are expandable for var kwargs. Returns: Dictionary containing the kwargs to be leveraged which may be used to overwrite one or more keyword only arguments when invoking the callable. + """ if not overrides: return {} diff --git a/vllm/utils/gc_utils.py b/vllm/utils/gc_utils.py index 32bdfb7648..be6ab4a23f 100644 --- a/vllm/utils/gc_utils.py +++ b/vllm/utils/gc_utils.py @@ -14,8 +14,7 @@ logger = init_logger(__name__) class GCDebugConfig: - """ - Config for GC Debugger. + """Config for GC Debugger. - 0: disable GC debugger - 1: enable GC debugger with gc.collect elapsed times - '{"top_objects":5}': enable GC debugger with top 5 collected objects @@ -44,8 +43,7 @@ class GCDebugConfig: class GCDebugger: - """ - Debugger for GC which logs helpful information for GC understanding. + """Debugger for GC which logs helpful information for GC understanding. To enable, you should call maybe_attach_gc_debug_callback in the process. """ @@ -59,9 +57,7 @@ class GCDebugger: self.gc_top_collected_objects: str = "" def handle(self, phase: str, info: dict[str, int]) -> None: - """ - Handles a GC event (e.g. GC start or GC finish) - """ + """Handles a GC event (e.g. GC start or GC finish).""" generation = info.get("generation") if generation is None: return @@ -122,8 +118,7 @@ def freeze_gc_for_cudagraph_capture(): def freeze_gc_heap() -> None: - """ - Freeze all objects tracked by the garbage collector. It should be invoked + """Freeze all objects tracked by the garbage collector. It should be invoked after server init / warmup, to reduce GC overhead from static objects during serving time. """ @@ -137,9 +132,7 @@ def freeze_gc_heap() -> None: def maybe_attach_gc_debug_callback() -> None: - """ - Attached a callback for GC debug when VLLM_GC_DEBUG is enabled. - """ + """Attached a callback for GC debug when VLLM_GC_DEBUG is enabled.""" config = GCDebugConfig(envs.VLLM_GC_DEBUG) if config.enabled: debugger: GCDebugger = GCDebugger(config) @@ -151,8 +144,7 @@ def maybe_attach_gc_debug_callback() -> None: def _compute_detailed_type(o: Any) -> str: - """ - Detailed object type. + """Detailed object type. TODO(Jialin): Further enhance the detailed type with element types for easier debugging. We tried but occasionally it would run into signals @@ -167,9 +159,7 @@ def _compute_detailed_type(o: Any) -> str: def _compute_top_gc_collected_objects(objects: list[Any], top: int) -> str: - """ - Group collected objects by types. - """ + """Group collected objects by types.""" if top <= 0: return "" object_types = [_compute_detailed_type(o) for o in objects] diff --git a/vllm/utils/hashing.py b/vllm/utils/hashing.py index f01c6b074f..9caee37493 100644 --- a/vllm/utils/hashing.py +++ b/vllm/utils/hashing.py @@ -35,6 +35,7 @@ def sha256(input: Any) -> bytes: Returns: Bytes representing the SHA-256 hash of the serialized input. + """ input_bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL) return hashlib.sha256(input_bytes).digest() @@ -53,6 +54,7 @@ def sha256_cbor(input: Any) -> bytes: Returns: Bytes representing the SHA-256 hash of the CBOR serialized input. + """ input_bytes = cbor2.dumps(input, canonical=True) return hashlib.sha256(input_bytes).digest() @@ -87,6 +89,7 @@ def get_hash_fn_by_name(hash_fn_name: str) -> Callable[[Any], bytes]: Returns: A hash function. + """ if hash_fn_name == "sha256": return sha256 @@ -110,6 +113,7 @@ def safe_hash(data: bytes, usedforsecurity: bool = True) -> HASH: Returns: Hash object + """ try: return hashlib.md5(data, usedforsecurity=usedforsecurity) diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 20c510edc0..d6a2aabbb4 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Contains helpers related to importing modules. +"""Contains helpers related to importing modules. This is similar in concept to the `importlib` module. """ @@ -24,8 +23,7 @@ logger = init_logger(__name__) def import_pynvml(): - """ - Historical comments: + """Historical comments: libnvml.so is the library behind nvidia-smi, and pynvml is a Python wrapper around it. We use it to get GPU @@ -57,8 +55,7 @@ def import_pynvml(): @cache def import_triton_kernels(): - """ - For convenience, prioritize triton_kernels that is available in + """For convenience, prioritize triton_kernels that is available in `site-packages`. Use `vllm.third_party.triton_kernels` as a fall-back. """ if _has_module("triton_kernels"): @@ -83,8 +80,7 @@ def import_triton_kernels(): def import_plugin(plugin_path: str) -> ModuleType | None: - """ - Import a user-defined plugin. + """Import a user-defined plugin. Plugin can be either: * a module in site-packages @@ -104,8 +100,7 @@ def import_plugin(plugin_path: str) -> ModuleType | None: def import_from_path(module_name: str, file_path: str | os.PathLike) -> ModuleType: - """ - Import a Python file according to its file path. + """Import a Python file according to its file path. Based on the official recipe: https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly @@ -123,9 +118,7 @@ def import_from_path(module_name: str, file_path: str | os.PathLike) -> ModuleTy def resolve_obj_by_qualname(qualname: str) -> Any: - """ - Resolve an object by its fully-qualified class name. - """ + """Resolve an object by its fully-qualified class name.""" module_name, obj_name = qualname.rsplit(".", 1) module = importlib.import_module(module_name) return getattr(module, obj_name) @@ -148,8 +141,7 @@ def get_vllm_optional_dependencies(): class _PlaceholderBase: - """ - Disallows downstream usage of placeholder modules. + """Disallows downstream usage of placeholder modules. We need to explicitly override each dunder method because [`__getattr__`][vllm.utils.import_utils._PlaceholderBase.__getattr__] @@ -160,8 +152,7 @@ class _PlaceholderBase: """ def __getattr__(self, key: str) -> Never: - """ - The main class should implement this to throw an error + """The main class should implement this to throw an error for attribute accesses representing downstream usage. """ raise NotImplementedError @@ -304,8 +295,7 @@ class _PlaceholderBase: class PlaceholderModule(_PlaceholderBase): - """ - A placeholder object to use when a module does not exist. + """A placeholder object to use when a module does not exist. This enables more informative errors when trying to access attributes of a module that does not exist. @@ -360,8 +350,7 @@ class _PlaceholderModuleAttr(_PlaceholderBase): class LazyLoader(ModuleType): - """ - `LazyLoader` module borrowed from [Tensorflow] + """`LazyLoader` module borrowed from [Tensorflow] (https://github.com/tensorflow/tensorflow/blob/main/tensorflow/python/util/lazy_loader.py) with an addition of "module caching". @@ -562,7 +551,6 @@ def has_tilelang() -> bool: def has_arctic_inference() -> bool: """Whether the optional `arctic_inference` package is available.""" - return _has_module("arctic_inference") diff --git a/vllm/utils/jit_monitor.py b/vllm/utils/jit_monitor.py index bb0bb8df9a..ffb63bc493 100644 --- a/vllm/utils/jit_monitor.py +++ b/vllm/utils/jit_monitor.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Monitor unexpected kernel JIT compilation during inference. +"""Monitor unexpected kernel JIT compilation during inference. After server warmup completes, any kernel JIT compilation or autotuning event indicates a cache miss or unexpected input shape that causes a latency spike. diff --git a/vllm/utils/jsontree.py b/vllm/utils/jsontree.py index fe757c2f33..0f430935ac 100644 --- a/vllm/utils/jsontree.py +++ b/vllm/utils/jsontree.py @@ -142,8 +142,7 @@ def json_reduce_leaves( initial: _U = ..., # type: ignore[assignment] /, ) -> _T | _U: - """ - Apply a function of two arguments cumulatively to each leaf in a + """Apply a function of two arguments cumulatively to each leaf in a nested JSON structure, from left to right, so as to reduce the sequence to a single value. """ diff --git a/vllm/utils/math_utils.py b/vllm/utils/math_utils.py index c681e708b5..8fb0009d60 100644 --- a/vllm/utils/math_utils.py +++ b/vllm/utils/math_utils.py @@ -13,7 +13,7 @@ def cdiv(a: int, b: int) -> int: def next_power_of_2(n: int) -> int: - """The next power of 2 (inclusive)""" + """The next power of 2 (inclusive).""" return 1 if n < 1 else 1 << (n - 1).bit_length() diff --git a/vllm/utils/mem_utils.py b/vllm/utils/mem_utils.py index c092dff410..1b449058b2 100644 --- a/vllm/utils/mem_utils.py +++ b/vllm/utils/mem_utils.py @@ -54,6 +54,7 @@ def release_device_memory_under_pressure(device: torch.device) -> bool: Returns: True if memory was released. + """ if device.type != "cuda" or not current_platform.is_integrated_gpu(device.index): return False @@ -230,8 +231,7 @@ def memory_profiling( baseline_snapshot: MemorySnapshot, weights_memory: int = 0, ) -> Generator[MemoryProfilingResult, None, None]: - """ - Memory profiling context manager. + """Memory profiling context manager. baseline_snapshot: the memory snapshot before the current vLLM instance. weights_memory: memory used by PyTorch when loading the model weights. diff --git a/vllm/utils/multi_stream_utils.py b/vllm/utils/multi_stream_utils.py index a8286c8480..6190b76991 100644 --- a/vllm/utils/multi_stream_utils.py +++ b/vllm/utils/multi_stream_utils.py @@ -41,6 +41,7 @@ def maybe_execute_in_parallel( Returns: Tuple of (fn0_result, fn1_result). + """ if aux_stream is not None: from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture @@ -100,6 +101,7 @@ def execute_in_parallel( Returns: Tuple of (default_result, aux_results) where aux_results[i] is the result of aux_fns[i] (or None when skipped). + """ aux_results: list[Any] if aux_streams is None or not enable: diff --git a/vllm/utils/network_utils.py b/vllm/utils/network_utils.py index 356d913879..fa24d4d5f9 100644 --- a/vllm/utils/network_utils.py +++ b/vllm/utils/network_utils.py @@ -172,8 +172,7 @@ def _get_reserved_port_range() -> range: def get_open_port() -> int: - """ - Get an open port for the vLLM process to listen on. + """Get an open port for the vLLM process to listen on. An edge case to handle, is when we run data parallel, we need to avoid ports that are potentially used by the data parallel master process. @@ -298,6 +297,7 @@ def make_zmq_path(scheme: str, host: str, port: int | None = None) -> str: Returns: A properly formatted ZMQ path string. + """ if port is None: return f"{scheme}://{host}" @@ -317,7 +317,6 @@ def make_zmq_socket( router_handover: bool = False, ) -> zmq.Socket | zmq.asyncio.Socket: # type: ignore[name-defined] """Make a ZMQ socket with the proper bind/connect semantics.""" - mem = psutil.virtual_memory() socket = ctx.socket(socket_type) @@ -377,8 +376,7 @@ def zmq_socket_ctx( identity: bytes | None = None, router_handover: bool = False, ) -> Iterator[zmq.Socket]: - """Context manager for a ZMQ socket""" - + """Context manager for a ZMQ socket.""" ctx = zmq.Context() # type: ignore[attr-defined] try: yield make_zmq_socket( diff --git a/vllm/utils/nvtx_pytorch_hooks.py b/vllm/utils/nvtx_pytorch_hooks.py index 25f76edbfa..5d979b04cf 100644 --- a/vllm/utils/nvtx_pytorch_hooks.py +++ b/vllm/utils/nvtx_pytorch_hooks.py @@ -25,7 +25,7 @@ def print_tensor(tensor_obj, prefix, tensor_list=None): def process_layer_params(module_obj): - """Extract the static parameters from LLM and VLM relevant layer types""" + """Extract the static parameters from LLM and VLM relevant layer types.""" param_info = {} # Extract parameters for layers commonly used in LLMs and VLMs if isinstance(module_obj, (torch.nn.Conv1d, torch.nn.Conv2d, torch.nn.Conv3d)): @@ -186,6 +186,7 @@ def layerwise_nvtx_marker_context(module_name, module_obj, in_tensor=None, kwarg kwargs=kwargs) as ctx: ctx.result = module(*args, **kwargs) return ctx.result + """ holder = ResultHolder() @@ -221,9 +222,9 @@ class PytHooks: a PytHook object. Then call the register_hooks method. Example: - my_hook = PytHook() my_hook.register_hooks(my_network_model) + """ def __init__(self): diff --git a/vllm/utils/ompmultiprocessing.py b/vllm/utils/ompmultiprocessing.py index 0a704e6d26..57f0016b22 100644 --- a/vllm/utils/ompmultiprocessing.py +++ b/vllm/utils/ompmultiprocessing.py @@ -187,18 +187,19 @@ class OMPProcessManager: def _get_autobind_cpu_ids( self, cpu_selector: Callable[[list[LogicalCPUInfo]], list[LogicalCPUInfo]] ) -> tuple[list[list[LogicalCPUInfo]], list[LogicalCPUInfo]]: - """ - Return CPU ids to bind based on NUMA nodes, and CPU ids reserved for + """Return CPU ids to bind based on NUMA nodes, and CPU ids reserved for other processes. Currently for rank N, only CPU ids on the N-th node in available NUMA node list will be selected. + Args: cpu_selector: a callable object to select CPUs from a CPU list - of a physical core. The input is a LogicalCPUInfo list contains - logical CPUs of a physical CPU, sorted by the LogicalCPUInfo.id. - A selected LogicalCPUInfo list should be returned. - """ + of a physical core. The input is a LogicalCPUInfo list contains + logical CPUs of a physical CPU, sorted by the + LogicalCPUInfo.id. A selected LogicalCPUInfo list should be + returned. + """ # this memory node list has been sliced for DP offset allowed_numa_nodes = cr_utils.get_visible_memory_node() logical_cpu_list = cr_utils.get_allowed_cpu_list() diff --git a/vllm/utils/registry.py b/vllm/utils/registry.py index a136d450e7..6df0bead0d 100644 --- a/vllm/utils/registry.py +++ b/vllm/utils/registry.py @@ -6,8 +6,7 @@ _T = TypeVar("_T", bound=type) class ExtensionManager: - """ - A registry for managing pluggable extension classes. + """A registry for managing pluggable extension classes. This class provides a simple mechanism to register and instantiate extension classes by name. It is commonly used to implement plugin @@ -26,15 +25,11 @@ class ExtensionManager: """ def __init__(self) -> None: - """ - Initialize an empty extension registry. - """ + """Initialize an empty extension registry.""" self.name2class: dict[str, type] = {} def register(self, name: str): - """ - Decorator to register a class with the given name. - """ + """Decorator to register a class with the given name.""" def wrap(cls_to_register: _T) -> _T: self.name2class[name] = cls_to_register @@ -43,9 +38,7 @@ class ExtensionManager: return wrap def load(self, cls_name: str, *args, **kwargs) -> Any: - """ - Instantiate and return a registered extension class by name. - """ + """Instantiate and return a registered extension class by name.""" cls = self.name2class.get(cls_name) assert cls is not None, f"Extension class {cls_name} not found" return cls(*args, **kwargs) diff --git a/vllm/utils/sparse_utils.py b/vllm/utils/sparse_utils.py index 79330271dd..582d91093e 100644 --- a/vllm/utils/sparse_utils.py +++ b/vllm/utils/sparse_utils.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Safe deserialization helpers for client-supplied embedding tensors. +"""Safe deserialization helpers for client-supplied embedding tensors. `check_sparse_tensor_invariants_threadsafe()` is a thread-safe wrapper for sparse tensor invariant validation. `safe_to_dense()` bounds the memory a diff --git a/vllm/utils/system_utils.py b/vllm/utils/system_utils.py index 6d56df378f..cc83a62a16 100644 --- a/vllm/utils/system_utils.py +++ b/vllm/utils/system_utils.py @@ -60,8 +60,7 @@ def set_env_var(key: str, value: str) -> Iterator[None]: @contextlib.contextmanager def suppress_stdout(): - """ - Suppress stdout from C libraries at the file descriptor level. + """Suppress stdout from C libraries at the file descriptor level. Only suppresses stdout, not stderr, to preserve error messages. Suppression is disabled when VLLM_LOGGING_LEVEL is set to DEBUG. @@ -70,6 +69,7 @@ def suppress_stdout(): with suppress_stdout(): # C library calls that would normally print to stdout torch.distributed.new_group(ranks, backend="gloo") + """ # Don't suppress if logging level is DEBUG if envs.VLLM_LOGGING_LEVEL == "DEBUG": @@ -114,7 +114,6 @@ def unique_filepath(fn: Callable[[int], Path]) -> Path: def _sync_visible_devices_env_vars(): """Sync HIP/CUDA visibility env vars before spawning (ROCm only).""" - if not current_platform.is_rocm(): return @@ -254,11 +253,11 @@ def decorate_logs( def kill_process_tree(pid: int): - """ - Kills all descendant processes of the given pid by sending SIGKILL. + """Kills all descendant processes of the given pid by sending SIGKILL. Args: pid (int): Process ID of the parent process + """ try: parent = psutil.Process(pid) diff --git a/vllm/utils/tensor_schema.py b/vllm/utils/tensor_schema.py index 526dfd38ba..02ddda03f1 100644 --- a/vllm/utils/tensor_schema.py +++ b/vllm/utils/tensor_schema.py @@ -161,7 +161,6 @@ class TensorSchema: dynamic_dims: set[str], ) -> None: """Validate that the actual tensor shape matches the expected shape.""" - if len(actual_shape) != len(expected_shape): raise ValueError( f"{field_name} has rank {len(actual_shape)} " diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index 0b330edf00..849d0a38f5 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -96,8 +96,7 @@ def is_meta_module(module: torch.nn.Module) -> bool: def is_strictly_contiguous(t: torch.Tensor) -> bool: - """ - Check if tensor is contiguous AND has no degenerate strides. + """Check if tensor is contiguous AND has no degenerate strides. A degenerate stride occurs when a dimension has size 1 but the stride doesn't match the canonical contiguous layout. This can cause issues @@ -294,8 +293,7 @@ def set_torch_threads_for_runtime() -> None: @contextlib.contextmanager def set_default_torch_num_threads(num_threads: int | None = None): - """ - Sets the default number of threads for PyTorch to the given value. + """Sets the default number of threads for PyTorch to the given value. `None` means using the value of the environment variable `OMP_NUM_THREADS` (or `1` if that is not available). @@ -363,8 +361,7 @@ def _get_precision_level(dtype: torch.dtype) -> int: def is_lossless_cast(src_dtype: torch.dtype, tgt_dtype: torch.dtype): - """ - Test whether it is lossless to cast a tensor from + """Test whether it is lossless to cast a tensor from `src_dtype` to `tgt_dtype`. """ if src_dtype == tgt_dtype: @@ -395,8 +392,7 @@ def is_lossless_cast(src_dtype: torch.dtype, tgt_dtype: torch.dtype): def common_broadcastable_dtype(dtypes: Collection[torch.dtype]): - """ - Get the common `dtype` where all of the other `dtypes` can be + """Get the common `dtype` where all of the other `dtypes` can be cast to it without losing any information. """ return max( @@ -567,6 +563,7 @@ def nvfp4_split_data_scale( Returns: ``(data, scale)`` where *data* is uint8 and *scale* is float8_e4m3fn, both views of the same storage. + """ num_pages = kv_side.shape[0] dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2] @@ -744,8 +741,7 @@ def make_ndarray_with_pad( *, max_len: int | None = None, ) -> npt.NDArray: - """ - Make a padded array from 2D inputs. + """Make a padded array from 2D inputs. The padding is applied to the end of each inner list until it reaches `max_len`. @@ -771,8 +767,7 @@ def make_tensor_with_pad( device: str | torch.device | None = None, pin_memory: bool = False, ) -> torch.Tensor: - """ - Make a padded tensor from 2D inputs. + """Make a padded tensor from 2D inputs. The padding is applied to the end of each inner list until it reaches `max_len`. @@ -806,8 +801,7 @@ class _StreamPlaceholder: def current_stream() -> torch.cuda.Stream: - """ - replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`. + """Replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`. it turns out that `torch.cuda.current_stream()` is quite expensive, as it will construct a new stream object at each call. here we patch `torch.cuda.set_stream` to keep track of the current stream @@ -858,9 +852,7 @@ _aux_stream: torch.cuda.Stream | None = None def aux_stream() -> torch.cuda.Stream | None: - """ - Ensures aux_stream is initialized only once - """ + """Ensures aux_stream is initialized only once.""" global _aux_stream from vllm.platforms import current_platform @@ -872,8 +864,7 @@ def aux_stream() -> torch.cuda.Stream | None: def weak_ref_tensor(tensor: Any) -> Any: - """ - Create a weak reference to a tensor. + """Create a weak reference to a tensor. The new tensor will share the same data as the original tensor, but will not keep the original tensor alive. This ignores 0-size tensors as those don't allocate any memory. @@ -890,8 +881,7 @@ def weak_ref_tensors( | tuple[torch.Tensor] | IntermediateTensors, ) -> torch.Tensor | list[Any] | tuple[Any] | Any: - """ - Convenience function to create weak references to tensors, + """Convenience function to create weak references to tensors, for single tensor, list of tensors or tuple of tensors. """ if isinstance(tensors, torch.Tensor): @@ -913,9 +903,7 @@ def weak_ref_tensors( def get_accelerator_view_from_cpu_tensor(cpu_tensor: torch.Tensor) -> torch.Tensor: - """ - Get an accelerator view of a CPU tensor using Unified Virtual Addressing (UVA). - """ + """Get an accelerator view of a CPU tensor via Unified Virtual Addressing.""" from vllm.platforms import current_platform if current_platform.is_xpu(): @@ -951,6 +939,7 @@ def is_torch_equal_or_newer(target: str) -> bool: Returns: Whether the condition meets. + """ try: return _is_torch_equal_or_newer(str(torch.__version__), target) @@ -979,6 +968,7 @@ def is_torch_equal(target: str) -> bool: Returns: Whether the condition meets. + """ try: return _is_torch_equal(target) @@ -1064,8 +1054,7 @@ def direct_register_custom_op( dispatch_key: str | None = None, tags: tuple[torch.Tag, ...] = (), ): - """ - `torch.library.custom_op` can have significant overhead because it + """`torch.library.custom_op` can have significant overhead because it needs to consider complicated dispatching logic. This function directly registers a custom op and dispatches it to the CUDA backend. See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5 diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 01599e9e9c..afafa3c925 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -34,8 +34,7 @@ from vllm.v1.kv_cache_interface import KVCacheLayout, get_kv_quant_mode class AttentionType(str, Enum): - """ - Attention type. + """Attention type. Use string to be compatible with `torch.compile`. """ @@ -383,8 +382,7 @@ T = TypeVar("T", bound=AttentionMetadata) @dataclass class CommonAttentionMetadata: - """ - Per-batch attention metadata, shared across layers and backends. + """Per-batch attention metadata, shared across layers and backends. AttentionMetadataBuilder instances use it to construct per-layer metadata. For many of the tensors we keep both GPU and CPU versions. @@ -655,8 +653,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]): common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, ) -> M: - """ - Central method that builds attention metadata. + """Central method that builds attention metadata. Some builders (MLA) require reorder_batch to be called prior to build. Args: @@ -665,6 +662,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]): fast_build: The meta-data will prioritize speed of building over then speed at execution. Can be used for spec-decode where the result of a build call may only be used for few layers/iters. + """ raise NotImplementedError @@ -674,8 +672,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]): blk_table: torch.Tensor, slot_mapping: torch.Tensor, ) -> M: - """ - Update the block table for the attention metadata. + """Update the block table for the attention metadata. Faster when theres multiple kv-cache groups that create virtually the same metadata but just with different block tables. @@ -686,8 +683,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]): def build_for_cudagraph_capture( self, common_attn_metadata: CommonAttentionMetadata ) -> M: - """ - Build attention metadata for CUDA graph capture. Uses build by default. + """Build attention metadata for CUDA graph capture. Uses build by default. Subclasses that override this method should call self.build or super().build_for_cudagraph_capture. """ @@ -700,8 +696,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]): common_attn_metadata: CommonAttentionMetadata, draft_index: int, ) -> M: - """ - Build attention metadata for draft model. Uses build by default. + """Build attention metadata for draft model. Uses build by default. Args: common_attn_metadata: The common attention metadata. @@ -710,6 +705,7 @@ class AttentionMetadataBuilder(ABC, Generic[M]): draft attempt for the i-th token. For tree-based attention, this index instead refers to the draft attempt for the i-th level in the tree of tokens. + """ return self.build( common_prefix_len=0, @@ -910,8 +906,7 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): raise NotImplementedError def fused_output_quant_supported(self, quant_key: "QuantKey") -> bool: - """ - Does this attention implementation support fused output quantization. + """Does this attention implementation support fused output quantization. This is used by the AttnFusionPass to only fuse output quantization onto implementations that support it. @@ -920,20 +915,19 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): Returns: is fusion supported for this type of quantization + """ return False def fused_qk_norm_rope_kvcache_supported(self): - """ - Does this attention implementation support fused QKNorm+RoPE+KVCache fusion. + """Does this attention implementation support fused QKNorm+RoPE+KVCache fusion. This is used by the QkNormRopeKvCachePattern to only fuse the QKNorm ops with the RoPE ops and the KV cache update for implementations that support it. """ return False def fused_rope_kvcache_supported(self): - """ - Does this attention implementation support RoPE+KVCache fusion. + """Does this attention implementation support RoPE+KVCache fusion. This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops with the KV cache update for implementations that support it. """ @@ -954,8 +948,7 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): kv_cache: torch.Tensor, layer_slot_mapping: torch.Tensor, ): - """ - If `fused_qk_norm_rope_kvcache_supported` returns True, this method + """If `fused_qk_norm_rope_kvcache_supported` returns True, this method will be called by the fused custom op. Applies QK-norm + RoPE and writes K/V to the KV cache. Results are written to the pre-allocated q_out and k_out tensors; V is split from QKV at the graph level. @@ -974,8 +967,7 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): kv_cache: torch.Tensor, layer_slot_mapping: torch.Tensor, ): - """ - If `fused_rope_kvcache_supported` returns True, this method will be called + """If `fused_rope_kvcache_supported` returns True, this method will be called by torch.ops.vllm.fused_rope_and_unified_kv_cache_update to perform the inplace RoPE and KV cache update. """ @@ -1040,8 +1032,7 @@ class MLAAttentionImpl(AttentionImplBase[T], Generic[T]): raise NotImplementedError def fused_output_quant_supported(self, quant_key: "QuantKey"): - """ - Does this attention implementation support fused output quantization. + """Does this attention implementation support fused output quantization. Since MLA quantization is done manually in forward_impl (common code), all MLA backends support it by default. """ @@ -1080,9 +1071,7 @@ def subclass_attention_backend( attention_backend_cls: type[AttentionBackend], builder_cls: type[AttentionMetadataBuilder[M]], ) -> type[AttentionBackend]: - """ - Return a new subclass where `get_builder_cls` returns `builder_cls`. - """ + """Return a new subclass where `get_builder_cls` returns `builder_cls`.""" name: str = name_prefix + attention_backend_cls.__name__ # type: ignore return type( diff --git a/vllm/v1/attention/backends/cpu_attn.py b/vllm/v1/attention/backends/cpu_attn.py index dd3f34c916..b6864897bf 100644 --- a/vllm/v1/attention/backends/cpu_attn.py +++ b/vllm/v1/attention/backends/cpu_attn.py @@ -349,14 +349,22 @@ class CPUAttentionBackendImpl(AttentionImpl): """Forward pass for CPU attention backend. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization; not supported + by this backend. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] + """ if output_scale is not None or output_block_scale is not None: raise NotImplementedError( diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index 0de6b85217..650fc1dea0 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -357,6 +357,7 @@ def is_flash_attn_varlen_func_available() -> bool: Returns: bool: True if a working flash_attn_varlen_func implementation is available. + """ if current_platform.is_cuda() or current_platform.is_xpu(): # CUDA and XPU always have flash_attn_varlen_func available diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index eb2268cc62..b57af3fd29 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -766,8 +766,7 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, ) -> FlashAttentionMetadata: - """ - fast_build disables AOT scheduling, used when there will be few + """fast_build disables AOT scheduling, used when there will be few iterations i.e. spec-decode """ num_reqs = common_attn_metadata.num_reqs @@ -1178,17 +1177,25 @@ class FlashAttentionImpl(AttentionImpl): """Forward pass with FlashAttention. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization; not supported + by this backend. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] NOTE: FP8 quantization, flash-attn expect the size of {q,k,v}_descale to be (num_sequences, num_kv_heads). We use torch's .expand() to avoid duplicating values + """ assert self.vllm_flash_attn_version is not None, ( "FlashAttention version not detected." @@ -1665,6 +1672,7 @@ class FlashAttentionImpl(AttentionImpl): output: shape = [num_encoder_tokens, num_heads, head_size] attn_metadata: Encoder attention metadata layer: The attention layer + """ assert self.vllm_flash_attn_version is not None, ( "FlashAttention version not detected." diff --git a/vllm/v1/attention/backends/flash_attn_diffkv.py b/vllm/v1/attention/backends/flash_attn_diffkv.py index 40846758d8..4bb58ef4ea 100644 --- a/vllm/v1/attention/backends/flash_attn_diffkv.py +++ b/vllm/v1/attention/backends/flash_attn_diffkv.py @@ -131,17 +131,25 @@ class FlashAttentionDiffKVImpl(FlashAttentionImpl): """Forward pass with FlashAttention. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size_v] kv_cache: shape = [num_blocks, block_size, num_kv_heads, head_size + head_size_v] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization; not supported + by this backend. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size_v] NOTE: FP8 quantization, flash-attn expect the size of {q,k,v}_descale to be (num_sequences, num_kv_heads). We use torch's .expand() to avoid duplicating values + """ assert self.vllm_flash_attn_version is not None, ( "FlashAttention version not detected." diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 7ab2b2018e..3113c261dd 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -1253,9 +1253,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): num_reqs: int, page_size: int, ) -> torch.Tensor: - """ - Compute paged_kv_indptr, paged_kv_indices, paged_kv_last_page_len for FlashInfer - attention. + """Compute paged_kv_indptr, paged_kv_indices and paged_kv_last_page_len. Results are stored in self.paged_kv_indptr, self.paged_kv_indices, self.paged_kv_last_page_len buffers. @@ -1964,6 +1962,7 @@ class FlashInferImpl(AttentionImpl): """Forward pass with FlashInfer. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size], or None for a KV-sharing decoder layer. @@ -1971,8 +1970,15 @@ class FlashInferImpl(AttentionImpl): KV-sharing decoder layer. kv_cache: [num_blocks, num_kv_heads, block_size, 2*head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization. Enables the + attention+quantization fusion path when provided. + output_block_scale: Block scale for fused output quantization, + required for nvfp4 output and rejected for fp8 output. + Returns: shape = [num_tokens, num_heads * head_size] + """ if attn_metadata is None: # Profiling run. @@ -2638,8 +2644,7 @@ def fast_plan_decode( fixed_split_size: int = -1, disable_split_kv: bool = False, ) -> None: - """ - A faster version of BatchDecodeWithPagedKVCacheWrapper::plan used for + """A faster version of BatchDecodeWithPagedKVCacheWrapper::plan used for cudagraph capture/replay, while the no cudagraph version turns back to the original plan. using original plan after passing host-side buffers: diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index 70eded1d09..9d0ac08172 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -165,8 +165,7 @@ def physical_to_logical_mapping( block_size: int, total_blocks: int, ) -> torch.Tensor: - """ - Creates an inverse mapping from physical block locations to logical indices. + """Creates an inverse mapping from physical block locations to logical indices. The original block_table maps from logical blocks to physical locations: @@ -239,6 +238,7 @@ def physical_to_logical_mapping( A tensor of shape [max_reqs, total_blocks] where each entry physical_to_logical[req_id, physical_block] contains the logical block index for that physical block, or -1 if unused. + """ max_reqs, max_num_blocks = block_table.shape device = block_table.device @@ -275,8 +275,7 @@ def unique_static_unsorted( ignored_val: int = 0, # value to ignore pad_val: int = -1, # sentinel for unused slots ) -> torch.Tensor: - """ - - Keeps the first occurrence of each non-zero value while preserving order, + """- Keeps the first occurrence of each non-zero value while preserving order, then left-packs those uniques and fills the rest with `pad_val`. - Returns (packed, keep_mask) with the *same shape* as `x`. - Requires that all values be in the range [0, M] @@ -286,6 +285,7 @@ def unique_static_unsorted( Example: x =[3, 1, 0, 1, 2], M=3, ignored_val=0 => [3, 1, 2, -1, -1] + """ if not (-1 <= pad_val <= M): raise ValueError("`pad_val` must lie in [-1, M]") @@ -349,6 +349,7 @@ class BlockSparsityHint(NamedTuple): hint_fn: (q_block_idx [num_tokens, 1], kv_block_idx [1, num_kv_blocks], block_size int) -> bool Tensor [num_tokens, num_kv_blocks]. Returns True for block pairs that may contain non-masked elements. + """ hint_fn: _block_sparsity_hint_signature @@ -435,6 +436,7 @@ class FlexAttentionMetadata: Returns: tuple of (is_valid, logical_q_idx, logical_kv_idx) + """ # Map query indices to corresponding request indices q_req = request_lookup[q_idx] @@ -512,7 +514,6 @@ class FlexAttentionMetadata: Note that the sliding window mask here is bidirectional, we need to mask it with the bidirectional/causal mask for encoder/decoder. """ - if self.sliding_window is None: raise ValueError("sliding_window must be set for sliding window attention") @@ -540,7 +541,6 @@ class FlexAttentionMetadata: def get_prefix_lm_mask_mod(self) -> _mask_mod_signature: """Creates the prefix LM mask_mod function for FlexAttention.""" - assert self.doc_ids is not None request_lookup = self.doc_ids @@ -587,7 +587,6 @@ class FlexAttentionMetadata: far-away generated tokens that fall outside the window and outside the prefix. """ - assert self.doc_ids is not None assert self.rswa_prefix_lens is not None assert self.rswa_window is not None @@ -1291,14 +1290,22 @@ class FlexAttentionImpl(AttentionImpl): """Forward pass with FLexAttention. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization; not supported + by this backend. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] + """ if output_scale is not None or output_block_scale is not None: raise NotImplementedError( diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 23daff5430..7a839ff73b 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -524,8 +524,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] def build_for_cudagraph_capture( self, common_attn_metadata: CommonAttentionMetadata ): - """ - This method builds the metadata for full cudagraph capture. + """This method builds the metadata for full cudagraph capture. Currently, only decode is supported for full cudagraphs with Mamba. """ m = common_attn_metadata diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index 189ef54b23..0a4584632d 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -23,8 +23,7 @@ def compute_varlen_chunk_metadata( query_start_loc: torch.Tensor, chunk_size: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Build chunk-aligned, variable-length metadata used by Mamba2 SSD kernels. + """Build chunk-aligned, variable-length metadata used by Mamba2 SSD kernels. Given per-sequence cumulative token starts `query_start_loc` of shape [B+1] and a physical `chunk_size`, returns three tensors on the same device: diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index e0b5fab465..b5f0b9b182 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -227,8 +227,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): def build_for_cudagraph_capture( self, common_attn_metadata: CommonAttentionMetadata ) -> M: - """ - This method builds the metadata for full cudagraph capture. + """This method builds the metadata for full cudagraph capture. Currently, only decode is supported for full cudagraphs with Mamba. """ m = common_attn_metadata @@ -276,8 +275,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_decode_draft_tokens_cpu: torch.Tensor | None = None, **kwargs: Any, ) -> M: - """ - Default build implementation for Mamba-like attention backends. + """Default build implementation for Mamba-like attention backends. Subclasses (e.g., Mamba2) can override to add additional metadata. """ return self._compute_common_metadata( @@ -294,8 +292,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_computed_tokens_p_cpu: torch.Tensor, query_start_loc_p_cpu: torch.Tensor, ) -> tuple[list[int], list[int], list[int]]: - """ - Compute chunk-specific metadata for Mamba models. + """Compute chunk-specific metadata for Mamba models. The code below carefully constructs the chunks such that: 1. Chunks contain tokens from a *single* sequence only. @@ -382,8 +379,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): common: M, common_attn_metadata: CommonAttentionMetadata, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Compute chunk metadata and return as device tensors. + """Compute chunk metadata and return as device tensors. Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p). """ num_prefills = common.num_prefills @@ -463,9 +459,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): prev_last_scheduled_idx: torch.Tensor | None = None, num_decode_draft_tokens_cpu: torch.Tensor | None = None, ) -> M: - """ - Compute metadata common to both Mamba1 and Mamba2. - """ + """Compute metadata common to both Mamba1 and Mamba2.""" num_reqs = common_attn_metadata.num_reqs # Treat multi-token queries as decode requests when @@ -752,8 +746,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): self, metadata: M, ) -> M: - """ - Update the metadata for cudagraph capture. + """Update the metadata for cudagraph capture. Currently, only decode is supported for full cudagraphs with Mamba. """ state_indices_tensor_d = metadata.state_indices_tensor_d diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 3a9ca2ffdb..265fb779c3 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -437,7 +437,6 @@ class FlashMLASparseMetadataBuilder( be the full batch or only decodes when prefills use dense MHA. This avoids the BF16 prefill kernel's head-padding overhead at high TP. """ - scheduler_metadata, _ = get_mla_metadata() return FlashMLASparseMetadata.FP8KernelMetadata( scheduler_metadata=scheduler_metadata, diff --git a/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py b/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py index 3ba5a5a5fa..1d03b9e773 100644 --- a/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py +++ b/vllm/v1/attention/backends/mla/prefill/aiter_flash_attn.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: class AiterFlashAttnPrefillBackend(MLAPrefillBackend): - """AITER FlashAttention backend for MLA prefill""" + """AITER FlashAttention backend for MLA prefill.""" @staticmethod def get_name() -> str: diff --git a/vllm/v1/attention/backends/mla/prefill/registry.py b/vllm/v1/attention/backends/mla/prefill/registry.py index e7e00a0d01..1cf1761a77 100644 --- a/vllm/v1/attention/backends/mla/prefill/registry.py +++ b/vllm/v1/attention/backends/mla/prefill/registry.py @@ -65,6 +65,7 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta): Raises: ValueError: If Backend.CUSTOM is used without being registered + """ path = _MLA_PREFILL_OVERRIDES.get(self, self.value) if not path: @@ -85,6 +86,7 @@ class MLAPrefillBackendEnum(Enum, metaclass=_MLAPrefillBackendEnumMeta): Raises: ImportError: If the backend class cannot be imported ValueError: If CUSTOM is used without being registered + """ return resolve_obj_by_qualname(self.get_path()) @@ -130,6 +132,7 @@ def register_mla_prefill_backend( MLAPrefillBackendEnum.CUSTOM, "my.module.MyCustomPrefillBackend" ) + """ def decorator(cls: type) -> type: diff --git a/vllm/v1/attention/backends/mla/prefill/selector.py b/vllm/v1/attention/backends/mla/prefill/selector.py index c7e935fa75..a2b74e4327 100644 --- a/vllm/v1/attention/backends/mla/prefill/selector.py +++ b/vllm/v1/attention/backends/mla/prefill/selector.py @@ -57,6 +57,7 @@ def _get_mla_prefill_backend_priorities( Returns: List of backends in priority order (highest priority first). + """ from vllm.platforms import current_platform @@ -104,6 +105,7 @@ def get_mla_prefill_backend( Returns: The selected prefill backend class. + """ from vllm.platforms import current_platform @@ -173,6 +175,7 @@ def _auto_select_mla_prefill_backend( Returns: The selected prefill backend class. + """ priorities = _get_mla_prefill_backend_priorities( device_capability, diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index c9ba2b0a56..75173e59a3 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -710,6 +710,7 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): attn_out_dtype: Dtype of the attention output buffer, used to size the padded-head output scratch (small head counts only). device: Target device for the buffers. + """ from aiter import get_ps_metadata_info_v1 @@ -1328,8 +1329,7 @@ def _expand_page_indices_kernel( class AiterMLAHelper: - """ - AITER MLA persistent (asm) decode requires a multiple of 16 heads. Unaligned + """AITER MLA persistent (asm) decode requires a multiple of 16 heads. Unaligned head counts through 128 are padded to the next multiple of 16 by tiling the query heads and slicing to the padded size. Native H24 AITER builds bypass that padding. Small divisors of 16 retain the existing repeat_interleave and diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 86b77db79a..e1365f4d30 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -167,8 +167,7 @@ def triton_convert_req_index_to_global_index( NUM_TOPK_TOKENS: int = 2048, BLOCK_N: int = 128, # tile width along columns ): - """ - out[token_id, indice_id] = + """out[token_id, indice_id] = block_table[req_id[token_id], token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_SIZE + token_indices[token_id, indice_id] % BLOCK_SIZE diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 700f328a37..e88631c32b 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -452,13 +452,15 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): assert hasattr(hf_config, "sliding_window") self.window_size = hf_config.sliding_window - # Vision variant: image spans (up to vision_max_n_token tokens) are + # V4 vision variant: image spans (up to vision_max_n_token tokens) are # visible bidirectionally, so prefill index rows widen from - # window_size to window_size + max_image_tokens. Text-only models keep - # max_image_tokens == 0 and take the original code paths everywhere. + # window_size to window_size + max_image_tokens. The V4 config sets + # mm_prefix_clamp_sliding_window exactly for these in-kernel-widened + # ranges; V4.1 image tokens use the plain causal window, and text-only + # models keep max_image_tokens == 0 everywhere. self.max_image_tokens = ( getattr(hf_config, "vision_max_n_token", 0) - if getattr(hf_config, "vision_n_layers", 0) > 0 + if getattr(hf_config, "mm_prefix_clamp_sliding_window", False) else 0 ) self.prefill_index_width = self.window_size + self.max_image_tokens diff --git a/vllm/v1/attention/backends/mla/sparse_utils.py b/vllm/v1/attention/backends/mla/sparse_utils.py index 242c8f3b78..0ac131cee5 100644 --- a/vllm/v1/attention/backends/mla/sparse_utils.py +++ b/vllm/v1/attention/backends/mla/sparse_utils.py @@ -426,6 +426,7 @@ def _remap_tiling( Returns: (single_tile, block_n, tiles_per_row, num_warps) + """ single_tile = ( count_valid and triton.next_power_of_2(NUM_TOPK_TOKENS) == NUM_TOPK_TOKENS @@ -454,8 +455,7 @@ def triton_convert_req_index_to_global_index( out: torch.Tensor | None = None, valid_counts_out: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """ - out[token_id, indice_id] = + """out[token_id, indice_id] = block_table[req_id[token_id], token_indices[token_id, indice_id] // BLOCK_SIZE] * BLOCK_SIZE + token_indices[token_id, indice_id] % BLOCK_SIZE diff --git a/vllm/v1/attention/backends/registry.py b/vllm/v1/attention/backends/registry.py index dcaaffaaa8..32e2fabeb5 100644 --- a/vllm/v1/attention/backends/registry.py +++ b/vllm/v1/attention/backends/registry.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Attention backend registry""" +"""Attention backend registry.""" from collections.abc import Callable from enum import Enum, EnumMeta @@ -161,6 +161,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): Raises: ValueError: If Backend.CUSTOM is used without being registered + """ path = _ATTN_OVERRIDES.get(self, self.value) if not path: @@ -181,6 +182,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): Raises: ImportError: If the backend class cannot be imported ValueError: If Backend.CUSTOM is used without being registered + """ return resolve_obj_by_qualname(self.get_path()) @@ -189,6 +191,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): Returns: True if the backend has a registered override + """ return self in _ATTN_OVERRIDES @@ -224,6 +227,7 @@ class MambaAttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): Raises: ValueError: If Backend.CUSTOM is used without being registered + """ path = _MAMBA_ATTN_OVERRIDES.get(self, self.value) if not path: @@ -244,6 +248,7 @@ class MambaAttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): Raises: ImportError: If the backend class cannot be imported ValueError: If Backend.CUSTOM is used without being registered + """ return resolve_obj_by_qualname(self.get_path()) @@ -252,6 +257,7 @@ class MambaAttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): Returns: True if the backend has a registered override + """ return self in _MAMBA_ATTN_OVERRIDES @@ -275,6 +281,7 @@ def register_backend( backend: The AttentionBackendEnum member to register class_path: Optional class path. If not provided and used as decorator, will be auto-generated from the class. + is_mamba: Whether the backend is a Mamba attention backend. Returns: Decorator function if class_path is None, otherwise a no-op @@ -300,6 +307,7 @@ def register_backend( AttentionBackendEnum.CUSTOM, "my.module.MyCustomBackend" ) + """ def decorator(cls: type) -> type: diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 062aa82285..1615f883bc 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -770,8 +770,7 @@ class AiterFlashAttentionMetadataBuilder( common_attn_metadata: CommonAttentionMetadata, draft_index: int, ) -> AiterFlashAttentionMetadata: - """ - Build attention metadata for draft model without CPU-GPU sync. + """Build attention metadata for draft model without CPU-GPU sync. During EAGLE drafting all requests are uniform decodes, so we can skip split_decodes_prefills_and_extends() and avoid all .cpu() / @@ -1189,17 +1188,25 @@ class AiterFlashAttentionImpl(AttentionImpl): """Forward pass with AiterFlashAttention. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [num_blocks, 2, block_size, num_kv_heads, head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization; not supported + by this backend. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] NOTE: FP8 quantization, flash-attn expect the size of {q,k,v}_descale to be (num_sequences, num_kv_heads). We use torch's .expand() to avoid duplicating values + """ if output_scale is not None or output_block_scale is not None: raise NotImplementedError( diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index e0e4b38550..9e3267c526 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -207,14 +207,21 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): """Forward pass with FlashAttention. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [num_blocks, 2, block_size, num_kv_heads, head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] + """ if output_block_scale is not None: raise NotImplementedError( diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index f3bb3b3514..2671249fb2 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -344,6 +344,7 @@ class RocmAttentionImpl(AttentionImpl): output: shape = [num_encoder_tokens, num_heads, head_size] attn_metadata: Encoder attention metadata layer: The attention layer + """ # For encoder attention, process FP8 quantization if needed if is_quantized_kv_cache(self.kv_cache_dtype): @@ -390,14 +391,21 @@ class RocmAttentionImpl(AttentionImpl): """Forward pass with FlashAttention. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: logical [num_blocks, 2, block_size, num_kv_heads * head_size] under LHBNC (physically K/V-group-first) attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] + """ if output_block_scale is not None: raise NotImplementedError( diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index b7e2a41d00..1b290c0ae9 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -557,14 +557,21 @@ class TritonAttentionImpl(AttentionImpl): """Forward pass with Paged Attention impl. in Triton. Args: + layer: The attention layer, providing the q/k/v quantization scales. query: shape = [num_tokens, num_heads, head_size] key: shape = [num_tokens, num_kv_heads, head_size] value: shape = [num_tokens, num_kv_heads, head_size] kv_cache: shape = [num_blocks, num_kv_heads, block_size, 2 * head_size] attn_metadata: Metadata for attention. + output: Tensor that the attention result is written into. + output_scale: Scale for fused output quantization. + output_block_scale: Block scale for fused output quantization; + not supported by this backend. + Returns: shape = [num_tokens, num_heads * head_size] + """ if output_block_scale is not None: raise NotImplementedError( @@ -724,6 +731,7 @@ class TritonAttentionImpl(AttentionImpl): output: shape = [num_encoder_tokens, num_heads, head_size] attn_metadata: Encoder attention metadata layer: The attention layer + """ # Quantized KV cache is not supported for encoder attention. if is_quantized_kv_cache(self.kv_cache_dtype): diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index e656343a91..3413c5ad00 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -324,8 +324,7 @@ def resolve_kv_cache_layout( @dataclass class PerLayerParameters: - """ - Currently, FlashInfer backend only support models in which all layers share + """Currently, FlashInfer backend only support models in which all layers share the same values for the following hyperparameters. Should not be used for trtllm-gen backend since it supports different values for the following hyperparameters. @@ -343,11 +342,9 @@ class PerLayerParameters: def get_per_layer_parameters( vllm_config: VllmConfig, layer_names: list[str], cls_: type["AttentionImpl"] ) -> dict[str, PerLayerParameters]: - """ - Scan layers in `layer_names` and determine some hyperparameters + """Scan layers in `layer_names` and determine some hyperparameters to use during `plan`. """ - layers = get_layers_from_vllm_config( vllm_config, AttentionLayerBase, # type: ignore[type-abstract] @@ -405,8 +402,7 @@ def get_num_attention_heads_from_layers( def infer_global_hyperparameters( per_layer_params: dict[str, PerLayerParameters], ) -> PerLayerParameters: - """ - Currently, FlashInfer backend other than trtllm-gen + """Currently, FlashInfer backend other than trtllm-gen only support models in which all layers share the same values for the following hyperparameters: - `window_left` @@ -416,7 +412,6 @@ def infer_global_hyperparameters( So this function asserts that all layers share the same values for these hyperparameters and returns the global values. """ - assert len(per_layer_params) > 0, "No attention layers found in the model." param_sets = list(per_layer_params.values()) @@ -710,8 +705,7 @@ def split_decodes_prefills_and_extends( common_attn_metadata: CommonAttentionMetadata, decode_threshold: int = 1, ) -> tuple[int, int, int, int, int, int]: - """ - Assuming a reordered batch, finds the boundary between prefill and decode + """Assuming a reordered batch, finds the boundary between prefill and decode requests. Args: @@ -726,6 +720,7 @@ def split_decodes_prefills_and_extends( num_decode_tokens: The number of tokens in the decode requests. num_extend_tokens: The number of tokens in the extend requests. num_prefill_tokens: The number of tokens in the prefill requests. + """ max_query_len = common_attn_metadata.max_query_len num_reqs = common_attn_metadata.num_reqs @@ -784,8 +779,7 @@ def split_decodes_and_prefills( require_uniform: bool = False, treat_short_extends_as_decodes: bool = True, ) -> tuple[int, int, int, int]: - """ - Assuming a reordered batch, finds the boundary between prefill and decode + """Assuming a reordered batch, finds the boundary between prefill and decode requests. The batch is expected to be ordered as: @@ -807,6 +801,7 @@ def split_decodes_and_prefills( num_prefills: The number of prefill requests. num_decode_tokens: The number of tokens in the decode requests. num_prefill_tokens: The number of tokens in the prefill requests. + """ max_query_len = common_attn_metadata.max_query_len num_reqs = common_attn_metadata.num_reqs @@ -855,16 +850,17 @@ def split_decodes_and_prefills( def split_prefill_chunks( seq_lens_cpu: torch.Tensor, workspace_size: int, request_offset: int = 0 ) -> list[tuple[int, int]]: - """ - Split the prefill requests into chunks such that the total sequence length + """Split the prefill requests into chunks such that the total sequence length of each chunk is less than or equal to the workspace size. Args: seq_lens_cpu: The sequence lengths of the prefill requests on CPU. workspace_size: The maximum workspace size (in tokens) per chunk. request_offset: The offset to add to the request indices. + Returns: A list of tuples of (reqs_start, reqs_end) representing chunk boundaries. + """ chunk_bounds = [] i, n = 0, len(seq_lens_cpu) @@ -884,8 +880,7 @@ def reorder_batch_to_split_decodes_and_prefills( scheduler_output: "SchedulerOutput", decode_threshold: int = 1, ) -> bool: - """ - Reorders the batch to split into prefill and decode requests; places all + """Reorders the batch to split into prefill and decode requests; places all requests with <= decode_threshold tokens at the front of the batch. The batch is reordered into 4 regions: @@ -896,6 +891,7 @@ def reorder_batch_to_split_decodes_and_prefills( Returns: True if the batch was modified, False otherwise. + """ num_reqs = len(input_batch.req_ids) num_scheduled_tokens = [ @@ -960,8 +956,7 @@ def reorder_batch_to_split_decodes_and_prefills( def reshape_query_for_spec_decode(query: torch.Tensor, batch_size: int) -> torch.Tensor: - """ - Reshapes the query tensor for the specified batch size, so that + """Reshapes the query tensor for the specified batch size, so that it has shape (batch_size, seq_len, num_heads, head_dim). """ assert query.dim() == 3, f"query must be 3D, got {query.dim()}D" @@ -976,8 +971,7 @@ def reshape_query_for_spec_decode(query: torch.Tensor, batch_size: int) -> torch def reshape_attn_output_for_spec_decode(attn_output: torch.Tensor) -> torch.Tensor: - """ - Reshapes the attention output tensor, so that + """Reshapes the attention output tensor, so that the batch_size and seq_len dimensions are combined. """ if attn_output.dim() == 3: @@ -1143,8 +1137,7 @@ def mamba_get_block_table_tensor( kv_cache_spec: KVCacheSpec, mamba_cache_mode: str, ) -> torch.Tensor: - """ - Get the block table tensor for mamba kernels from the input + """Get the block table tensor for mamba kernels from the input common_attn_metadata.block_table_tensor given different mamba cache modes. - "all": input (#requests, cdiv(max_model_len, block_size) diff --git a/vllm/v1/attention/ops/common.py b/vllm/v1/attention/ops/common.py index b561817055..5c1677cac4 100644 --- a/vllm/v1/attention/ops/common.py +++ b/vllm/v1/attention/ops/common.py @@ -178,6 +178,7 @@ def pack_seq_triton( Returns: packed: [B, Lmax, ...] — packed tensor. + """ is_uint8 = x.dtype == torch.uint8 if is_uint8: @@ -334,8 +335,7 @@ def unpack_seq_triton( block_t: int = 64, block_d: int = 64, ) -> torch.Tensor: - """ - Unpack a packed decode query tensor back to the original format. + """Unpack a packed decode query tensor back to the original format. Efficient Triton implementation. Args: @@ -346,8 +346,8 @@ def unpack_seq_triton( Returns: unpacked_tensor: [N, ...] where N = sum(lengths) - """ + """ # Handle multi-dimensional input by reshaping to (B, Lmax, -1) original_shape = packed_tensor.shape if len(original_shape) > 3: diff --git a/vllm/v1/attention/ops/dcp.py b/vllm/v1/attention/ops/dcp.py index 58ac1e14a0..8f844b6476 100644 --- a/vllm/v1/attention/ops/dcp.py +++ b/vllm/v1/attention/ops/dcp.py @@ -133,8 +133,7 @@ class CorrectAttnCPOutKernel(VllmTritonJitKernel["CorrectAttnCPOutKernel.Compile N_ROUNDED: tl.constexpr, IS_BASE_E: tl.constexpr, ): - """ - Apply the all-gathered lses to correct each local rank's attention + """Apply the all-gathered lses to correct each local rank's attention output. we still need perform a cross-rank reduction to obtain the final attention output. @@ -147,6 +146,17 @@ class CorrectAttnCPOutKernel(VllmTritonJitKernel["CorrectAttnCPOutKernel.Compile Pointer to output tensor of shape [ B, H, D ] vlse_ptr (triton.PointerType): Pointer to output tensor of shape [ B, H ] + outputs_stride_B (int): Batch stride of ``outputs_ptr`` + outputs_stride_H (int): Head stride of ``outputs_ptr`` + outputs_stride_D (int): Head-dim stride of ``outputs_ptr`` + lses_stride_N (int): Rank stride of ``lses_ptr`` + lses_stride_B (int): Batch stride of ``lses_ptr`` + lses_stride_H (int): Head stride of ``lses_ptr`` + lse_idx (int): Index of this rank's lse within the all-gathered tensor + HEAD_DIM: Head dimension, as a constexpr + N_ROUNDED: Rank count rounded to a power of two, as a constexpr + IS_BASE_E: Whether the lses are natural-log based, as a constexpr + """ batch_idx = tl.program_id(axis=0).to(tl.int64) head_idx = tl.program_id(axis=1).to(tl.int64) @@ -376,9 +386,11 @@ def correct_attn_out( lses: Tensor of shape [ N, B, H ] cp_rank: Current rank in the context-parallel group ctx: Triton context to avoid recompilation + is_lse_base_on_e: Whether the lses use base e rather than base 2 Returns: Tuple of (out, lse) with corrected attention and final log-sum-exp. + """ if ctx is None: ctx = CPTritonContext() @@ -431,8 +443,7 @@ def _cp_lse_common( seq_lens: torch.Tensor | None = None, query_start_loc: torch.Tensor | None = None, ): - """ - cp_attn_out: [ B, H, D ] + """cp_attn_out: [ B, H, D ] cp_attn_lse: [ B, H ] """ if cp_group.world_size == 1: @@ -466,8 +477,7 @@ def cp_lse_ag_out_rs( seq_lens: torch.Tensor | None = None, query_start_loc: torch.Tensor | None = None, ): - """ - cp_attn_out: [ B, H, D ] + """cp_attn_out: [ B, H, D ] cp_attn_lse: [ B, H ] """ out, lse = _cp_lse_common( @@ -499,8 +509,7 @@ def cp_lse_ag_out_ar( seq_lens: torch.Tensor | None = None, query_start_loc: torch.Tensor | None = None, ): - """ - cp_attn_out: [ B, H, D ] + """cp_attn_out: [ B, H, D ] cp_attn_lse: [ B, H ] """ out, lse = _cp_lse_common( @@ -528,8 +537,7 @@ def _lse_weighted_combine( return_lse: bool = False, is_lse_base_on_e: bool = True, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """ - CPU reference implementation for LSE-weighted combination. + """CPU reference implementation for LSE-weighted combination. This is a pure PyTorch implementation used for testing and validation. @@ -545,6 +553,7 @@ def _lse_weighted_combine( Returns: Combined output [B, H, D], and optionally global LSE [B, H] + """ N, B, H, D = outputs.shape @@ -936,8 +945,7 @@ def dcp_a2a_lse_reduce( seq_lens: torch.Tensor | None = None, query_start_loc: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """ - Combine partial attention outputs across DCP ranks using All-to-All. + """Combine partial attention outputs across DCP ranks using All-to-All. The output and LSE are packed into a single output-dtype buffer, sent with one All-to-All, then unpacked and combined with exact LSE weighting. @@ -955,6 +963,7 @@ def dcp_a2a_lse_reduce( Returns: Combined output [B, H/N, D] (head-scattered) If return_lse=True, also returns global_lse [B, H/N] + """ world_size = cp_group.world_size diff --git a/vllm/v1/attention/ops/flashmla.py b/vllm/v1/attention/ops/flashmla.py index c84a495859..eb5383e5d1 100644 --- a/vllm/v1/attention/ops/flashmla.py +++ b/vllm/v1/attention/ops/flashmla.py @@ -49,9 +49,7 @@ def _is_flashmla_available() -> tuple[bool, str | None]: def is_flashmla_dense_supported() -> tuple[bool, str | None]: - """ - Return: is_supported_flag, unsupported_reason (optional). - """ + """Return: is_supported_flag, unsupported_reason (optional).""" is_available, maybe_reason = _is_flashmla_available() if not is_available: return False, maybe_reason @@ -61,9 +59,7 @@ def is_flashmla_dense_supported() -> tuple[bool, str | None]: def is_flashmla_sparse_supported() -> tuple[bool, str | None]: - """ - Return: is_supported_flag, unsupported_reason (optional). - """ + """Return: is_supported_flag, unsupported_reason (optional).""" is_available, maybe_reason = _is_flashmla_available() if not is_available: return False, maybe_reason diff --git a/vllm/v1/attention/ops/merge_attn_states.py b/vllm/v1/attention/ops/merge_attn_states.py index 6791e1f5fd..d28c78bdb9 100644 --- a/vllm/v1/attention/ops/merge_attn_states.py +++ b/vllm/v1/attention/ops/merge_attn_states.py @@ -106,8 +106,8 @@ def merge_attn_states( are treated as having context. output_scale: Optional scalar tensor for FP8 static quantization. When provided, output must be FP8 dtype. - """ + """ # Both the CUDA and Triton kernels derive the suffix head stride from # prefix_output, so suffix_output must share the same head stride. assert prefix_output.stride(1) == suffix_output.stride(1), ( diff --git a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py index 0e342cb0e0..07cdf18106 100644 --- a/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/ops/rocm_aiter_mla_sparse.py @@ -654,6 +654,7 @@ def rocm_fp8_paged_mqa_logits( Returns: Logits tensor of shape [B * next_n, max_model_len], dtype `torch.float32`. + """ from vllm._aiter_ops import rocm_aiter_ops @@ -737,6 +738,7 @@ def fp8_mqa_logits_torch( Returns: Logits tensor of shape [M, N], dtype `torch.float32`. + """ k_fp8, scale = kv seq_len_kv = k_fp8.shape[0] @@ -805,8 +807,8 @@ def rocm_fp8_mqa_logits( Returns: Logits tensor of shape [M, N], dtype `torch.float32`. - """ + """ from vllm._aiter_ops import rocm_aiter_ops k_fp8, scale = kv diff --git a/vllm/v1/attention/ops/triton_decode_attention.py b/vllm/v1/attention/ops/triton_decode_attention.py index 6aec1db2e5..130760d1de 100644 --- a/vllm/v1/attention/ops/triton_decode_attention.py +++ b/vllm/v1/attention/ops/triton_decode_attention.py @@ -24,8 +24,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -""" -Memory-efficient attention for decoding. +"""Memory-efficient attention for decoding. It supports page size >= 1. """ diff --git a/vllm/v1/attention/ops/triton_prefill_attention.py b/vllm/v1/attention/ops/triton_prefill_attention.py index 0fb31ce7e1..00d630edcd 100644 --- a/vllm/v1/attention/ops/triton_prefill_attention.py +++ b/vllm/v1/attention/ops/triton_prefill_attention.py @@ -19,8 +19,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -""" -Memory-efficient attention for prefill. +"""Memory-efficient attention for prefill. It supports page size = 1. """ @@ -229,8 +228,7 @@ def context_attention_fwd( sinks: torch.Tensor | None = None, sinks_bias_key0: bool = False, ): - """ - q, k, v: [b * s, head, head_dim] + """q, k, v: [b * s, head, head_dim] b_start_loc: [b] b_seq_len: [b] out: [b * s, head, head_dim] diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index b7aeb8c6a7..b90eb58e9b 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -This file contains ops for ViT attention to be compatible with torch.compile +"""This file contains ops for ViT attention to be compatible with torch.compile as there are operations here not supported by torch.compile (for instance, `.item()` in flash attention) @@ -238,8 +237,7 @@ def apply_sdpa( scale: float | None = None, enable_gqa: bool = False, ) -> torch.Tensor: - """ - Input shape: + """Input shape: (batch_size x seq_len x num_heads x head_size) """ q, k, v = (einops.rearrange(x, "b s h d -> b h s d") for x in [q, k, v]) diff --git a/vllm/v1/attention/ops/xpu_mla_sparse.py b/vllm/v1/attention/ops/xpu_mla_sparse.py index 9c1a92e10f..1e7a9ec3fe 100644 --- a/vllm/v1/attention/ops/xpu_mla_sparse.py +++ b/vllm/v1/attention/ops/xpu_mla_sparse.py @@ -185,15 +185,20 @@ def triton_bf16_mla_sparse_interface( d_v: int = 512, block_dpe: int = 64, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - out : [num_tokens, num_heads_q, d_v] + """Out : [num_tokens, num_heads_q, d_v] max_logits : [num_tokens, num_heads_q] lse : logsumexp, [num_tokens, num_heads_q] Args: + q: Query tensor, [num_tokens, num_heads_q, dim_qk]. + kv: Latent KV cache tensor. + indices: Sparse KV indices selected for each query token. + sm_scale: Softmax scale applied to the QK product. + d_v: Size of the value dimension. block_dpe: Size of positional embedding portion of dim_qk. Set to 0 when q/kv contain only the nope latent (e.g. DSv4 prefill where RoPE is not split out). + """ num_tokens, num_heads_q, dim_qk = q.shape _, num_heads_kv, _ = kv.shape diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index 286fdcb8df..695916d819 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -85,6 +85,7 @@ def get_attn_spec_kind( Returns: The ``KVCacheSpecKind`` the layer maps to. + """ from vllm.v1.kv_cache_interface import KVCacheSpecKind @@ -115,7 +116,6 @@ def get_attn_backend( has_sliding_window: bool = False, ) -> type[AttentionBackend]: """Selects which attention backend to use and lazily imports it.""" - if kv_cache_dtype is not None: valid_cache_dtypes = get_args(CacheDType) assert kv_cache_dtype in valid_cache_dtypes, ( diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index e2a06e7e4e..b4c9dcaa29 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -31,8 +31,7 @@ logger = init_logger(__name__) class BlockHashToBlockMap: - """ - Cache of blocks that are used for prefix caching. It caches blocks + """Cache of blocks that are used for prefix caching. It caches blocks from hash directly to a block or multiple blocks (i.e. {block_hash: KVCacheBlocks}) - Mostly block_hash maps to a single KVCacheBlock, and KVCacheBlocks @@ -59,9 +58,7 @@ class BlockHashToBlockMap: ] = {} def get_one_block(self, key: BlockHashWithGroupId) -> KVCacheBlock | None: - """ - Gets any block with the given block hash key. - """ + """Gets any block with the given block hash key.""" blocks = self._cache.get(key) if blocks is not None: if isinstance(blocks, KVCacheBlock): @@ -72,9 +69,7 @@ class BlockHashToBlockMap: return None def contain(self, key: BlockHashWithGroupId, block_id: int) -> bool: - """ - Checks whether the key maps to the given block ID. - """ + """Checks whether the key maps to the given block ID.""" blocks = self._cache.get(key) if blocks is None: return False @@ -86,9 +81,7 @@ class BlockHashToBlockMap: return False def insert(self, key: BlockHashWithGroupId, block: KVCacheBlock) -> None: - """ - Inserts the KVCacheBlock to the cache - """ + """Inserts the KVCacheBlock to the cache""" blocks = self._cache.get(key) if blocks is None: # When key is not found, attach a single block to the key @@ -104,9 +97,7 @@ class BlockHashToBlockMap: self._unexpected_blocks_type(blocks) def pop(self, key: BlockHashWithGroupId, block_id: int) -> KVCacheBlock | None: - """ - Checks if block_hash exists and pop block_id from the cache - """ + """Checks if block_hash exists and pop block_id from the cache""" blocks = self._cache.pop(key, None) if blocks is None: # block_hash not found in the cache @@ -158,6 +149,7 @@ class BlockPool: enable_kv_cache_events: Whether to enable kv cache events. metrics_collector: Optional metrics collector for tracking block residency. medium: Storage medium reported in KV cache events. + """ def __init__( @@ -214,6 +206,7 @@ class BlockPool: Returns: The cached blocks if exists, or None. + """ cached_blocks = [] for group_id in kv_cache_group_ids: @@ -261,6 +254,7 @@ class BlockPool: consults a subset of blocks (e.g. SWA tail-window), so blocks that can never serve a hit stay out of the prefix-cache hash map. + """ if num_cached_blocks >= num_full_blocks: return @@ -395,6 +389,7 @@ class BlockPool: num_cached_blocks: Number of blocks that were cache hits. block_size: Number of tokens per block. kv_cache_group_id: The KV cache group ID. + """ if not self.enable_kv_cache_events or num_cached_blocks == 0: return @@ -491,6 +486,7 @@ class BlockPool: Returns: The hash key with group ID if a partial entry can be registered; otherwise ``None`` for null blocks. + """ if block.is_null: return None @@ -671,6 +667,7 @@ class BlockPool: Returns: A list of new block. + """ if num_blocks > self.get_num_free_blocks(): raise ValueError(f"Cannot get {num_blocks} free blocks from the pool") @@ -724,8 +721,7 @@ class BlockPool: self.free_block_queue.append_n(released) def _maybe_evict_cached_block(self, block: KVCacheBlock) -> bool: - """ - If a block is cached in `cached_block_hash_to_block`, we reset its hash + """If a block is cached in `cached_block_hash_to_block`, we reset its hash metadata and evict it from the cache. Args: @@ -733,6 +729,7 @@ class BlockPool: Returns: True if the block is evicted, False otherwise. + """ # Clean up metrics tracking first to prevent leaks if self.metrics_collector: @@ -753,6 +750,7 @@ class BlockPool: Args: blocks: A list of blocks to touch. + """ for block in blocks: # ref_cnt=0 means this block is in the free list (i.e. eviction @@ -774,6 +772,7 @@ class BlockPool: Args: ordered_blocks: A list of blocks to free ordered by their eviction priority. + """ # Identify blocks with hash (LRU cache) and without it (never match APC) blocks_to_evict_last = [] @@ -800,7 +799,7 @@ class BlockPool: pool.free_blocks(blocks) def evict_blocks(self, block_ids: set[int]) -> None: - """evict blocks from the prefix cache by their block IDs. + """Evict blocks from the prefix cache by their block IDs. only evicts blocks that are currently cached (have a hash). blocks with ref_cnt > 0 are not freed from the block pool, only evicted @@ -808,6 +807,7 @@ class BlockPool: Args: block_ids: Set of block IDs to evict from cache. + """ for block_id in block_ids: assert block_id < len(self.blocks), ( @@ -826,6 +826,7 @@ class BlockPool: Returns: bool: True if the prefix cache is successfully reset, False otherwise. + """ num_used_blocks = self.num_gpu_blocks - self.get_num_free_blocks() if num_used_blocks != 1: # The null block is always marked as used @@ -863,6 +864,7 @@ class BlockPool: Returns: The number of free blocks. + """ return self.free_block_queue.num_free_blocks @@ -871,8 +873,8 @@ class BlockPool: Returns: The KV cache usage (between 0.0 and 1.0). - """ + """ # Subtract 1 to account for null block. total_gpu_blocks = self.num_gpu_blocks - 1 if not total_gpu_blocks: @@ -884,6 +886,7 @@ class BlockPool: Returns: A list of KV cache events. + """ if not self.enable_kv_cache_events: return [] diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index a3ed0c7aa1..f069ce7453 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -64,6 +64,7 @@ class EncoderCacheManager: make space when needed. freed: List of mm_hash strings that were actually evicted since the last call to get_freed_mm_hashes(). This list is cleared on return. + """ @classmethod @@ -113,6 +114,7 @@ class EncoderCacheManager: Returns: True if the encoder output for this input is already cached + """ mm_hash = request.mm_features[input_id].identifier # Not cached at all @@ -162,6 +164,7 @@ class EncoderCacheManager: Note: This method does not allocate physical memory for the encoder output but only the state of EncoderCacheManager. + """ num_embeds = request.get_num_encoder_embeds(input_id) @@ -198,8 +201,8 @@ class EncoderCacheManager: Note: This method assumes can_allocate() returned True for the same input. - """ + """ mm_hash = request.mm_features[input_id].identifier request_id = request.request_id if mm_hash not in self.cached: @@ -222,7 +225,7 @@ class EncoderCacheManager: return self.request_cached_ids.get(request.request_id, set()) def free_encoder_input(self, request: Request, input_id: int) -> None: - """Free the request's reference to the encoder input (`mm_data`) + """Free the request's reference to the encoder input (`mm_data`). When the reference set for the corresponding `mm_hash` becomes empty, the entry is appended to `freeable` and `num_freeable_slots` is @@ -279,6 +282,7 @@ class EncoderCacheManager: call to be used by the scheduler to notify workers about which encoder outputs can be removed from their caches. The internal list is cleared after this call. + """ # An entry evicted early in the scheduling pass can be allocated again # later in the same pass. Keep its worker-side tensor in that case. @@ -307,8 +311,8 @@ def compute_mm_encoder_budget( from the input sequence. - Space budget for encoder cache size, measured in number of tokens from the input sequence. - """ + """ if not mm_max_toks_per_item: logger.warning( "All non-text modalities supported by the model have been " diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index afb03ac3ea..8061999409 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -64,9 +64,7 @@ def _validate_prefix_cache_retention_interval( class KVCacheCoordinator(ABC): - """ - Coordinate the KV cache of different KV cache groups. - """ + """Coordinate the KV cache of different KV cache groups.""" enable_partial_hash_hits = False @@ -179,8 +177,7 @@ class KVCacheCoordinator(ABC): num_tokens_main_model: int, apply_admission_cap: bool = False, ) -> int: - """ - Get the number of device blocks needed to be allocated for the request. + """Get the number of device blocks needed to be allocated for the request. Args: request_id: The request ID. @@ -203,6 +200,7 @@ class KVCacheCoordinator(ABC): Returns: The number of blocks to allocate. + """ num_blocks_to_allocate = 0 for i, manager in enumerate(self.single_type_managers): @@ -237,8 +235,7 @@ class KVCacheCoordinator(ABC): num_local_computed_tokens: int, num_external_computed_tokens: int, ) -> None: - """ - Add the new computed blocks to the request. Optionally allocate new + """Add the new computed blocks to the request. Optionally allocate new blocks for external computed tokens (if any). Args: @@ -247,6 +244,7 @@ class KVCacheCoordinator(ABC): prefix cache. num_local_computed_tokens: The number of local computed tokens. num_external_computed_tokens: The number of external computed tokens. + """ # A running request is already tracked in num_cached_block and won't # have new prefix-cache hits, so this is a no-op for it. @@ -283,8 +281,7 @@ class KVCacheCoordinator(ABC): num_tokens_main_model: int, num_encoder_tokens: int = 0, ) -> tuple[list[KVCacheBlock], ...]: - """ - Allocate new blocks for the request to give it at least `num_tokens` + """Allocate new blocks for the request to give it at least `num_tokens` token slots. Args: @@ -299,6 +296,7 @@ class KVCacheCoordinator(ABC): Returns: The new allocated blocks. + """ return tuple( manager.allocate_new_blocks( @@ -333,14 +331,14 @@ class KVCacheCoordinator(ABC): return tuple(sorted({max(resend - block, 0), max(extension - block, 0)})) def cache_blocks(self, request: Request, num_computed_tokens: int) -> None: - """ - Cache the blocks for the request. + """Cache the blocks for the request. Args: request: The request. num_computed_tokens: The total number of tokens that need to be cached (including tokens that are already cached). + """ boundaries = self.get_replay_boundaries(request) for manager in self.single_type_managers: @@ -380,18 +378,17 @@ class KVCacheCoordinator(ABC): return all([pool.reset_prefix_cache() for pool in pools]) def free(self, request_id: str) -> None: - """ - Free the blocks for the request. + """Free the blocks for the request. Args: request_id: The request ID. + """ for manager in self.single_type_managers: manager.free(request_id) def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: - """ - Pop the request's bookkeeping from all single-type managers and + """Pop the request's bookkeeping from all single-type managers and return its blocks without returning them to the block pool. The caller must eventually pass the returned blocks to `block_pool.free_blocks`, freeing them in reverse order (so that @@ -402,6 +399,7 @@ class KVCacheCoordinator(ABC): Returns: The request's blocks in allocation order. + """ blocks: list[KVCacheBlock] = [] for manager in self.single_type_managers: @@ -409,8 +407,7 @@ class KVCacheCoordinator(ABC): return blocks def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]: - """ - Get the number of common prefix blocks for all requests with allocated + """Get the number of common prefix blocks for all requests with allocated KV cache for each kv cache group. Args: @@ -419,6 +416,7 @@ class KVCacheCoordinator(ABC): Returns: list[int]: The number of common prefix blocks for each kv cache group. + """ return [ manager.get_num_common_prefix_blocks(running_request_id) @@ -431,8 +429,7 @@ class KVCacheCoordinator(ABC): processed_computed_tokens: int, num_prompt_tokens: int | None = None, ) -> None: - """ - Remove the blocks that are no longer needed from `blocks` and replace + """Remove the blocks that are no longer needed from `blocks` and replace the removed blocks with null_block. Args: @@ -442,6 +439,7 @@ class KVCacheCoordinator(ABC): num_prompt_tokens: Optional prompt length. R-SWA managers use this to free gap blocks between the prefill tail and decode window; other manager types ignore it. + """ for manager in self.single_type_managers: manager.remove_skipped_blocks( @@ -449,9 +447,7 @@ class KVCacheCoordinator(ABC): ) def get_blocks(self, request_id: str) -> tuple[list[KVCacheBlock], ...]: - """ - Get the blocks for the request. - """ + """Get the blocks for the request.""" return tuple( manager.req_to_blocks.get(request_id) or [] for manager in self.single_type_managers @@ -475,8 +471,7 @@ class KVCacheCoordinator(ABC): class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator): - """ - KV cache coordinator to use if prefix caching is disabled or unsupported. + """KV cache coordinator to use if prefix caching is disabled or unsupported. In contrast to UnitaryKVCacheCoordinator and HybridKVCacheCoordinator, supports arbitrary numbers of KV cache groups (including 0 groups). Does not implement any features related to prefix caching. @@ -527,8 +522,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator): class UnitaryKVCacheCoordinator(KVCacheCoordinator): - """ - KV cache coordinator for models with only one KV cache group. This is the + """KV cache coordinator for models with only one KV cache group. This is the case for models with only one KV cache type, e.g., all attention layers use full attention or all attention layers use sliding window attention. """ @@ -613,8 +607,7 @@ class SpecGroup(NamedTuple): class HybridKVCacheCoordinator(KVCacheCoordinator): - """ - KV cache coordinator for hybrid models with multiple KV cache types, and + """KV cache coordinator for hybrid models with multiple KV cache types, and thus multiple kv cache groups. """ @@ -733,8 +726,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): ) def verify_and_split_kv_cache_groups(self) -> None: - """ - Groups KV cache groups by their spec type for efficient batch processing + """Groups KV cache groups by their spec type for efficient batch processing during cache hit lookup. """ self.attention_groups: list[SpecGroup] = [] @@ -843,8 +835,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): block_hashes: list[BlockHash], max_cache_hit_length: int, ) -> tuple[tuple[list[KVCacheBlock], ...], int, int]: - """ - Find the longest cache hit using an iterative fixed-point algorithm. + """Find the longest cache hit using an iterative fixed-point algorithm. Each attention type either accepts the current candidate length or reduces it. If any type reduces the length, restart checks over all @@ -861,8 +852,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): - The number of tokens of the reconciled (combined) cache hit. - ``num_uncached_common_prefix_tokens``: a shared prefix that a sparse-retention group has not cached yet (0 unless hybrid). - """ + """ num_groups = len(self.kv_cache_config.kv_cache_groups) hit_length = max_cache_hit_length longest_hit_length = 0 @@ -985,8 +976,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator): Returns: (blocks_per_group, hit_lengths_per_group) - """ + """ num_groups = len(self.kv_cache_config.kv_cache_groups) hit_blocks: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)] hit_lengths: list[int] = [0] * num_groups diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 226da3a1ed..49f62a2670 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -33,8 +33,7 @@ logger = init_logger(__name__) @dataclass class KVCacheBlocks: - """ - The allocation result of KVCacheManager, work as the interface between + """The allocation result of KVCacheManager, work as the interface between Scheduler and KVCacheManager, to hide KVCacheManager's internal data structure from the Scheduler. """ @@ -85,8 +84,7 @@ class KVCacheBlocks: *, group_ids: Sequence[int] | None = None, ) -> tuple[list[int], ...] | None: - """ - Converts the KVCacheBlocks instance to block_ids. + """Converts the KVCacheBlocks instance to block_ids. Args: allow_none: Return None when every selected group is empty. @@ -97,6 +95,7 @@ class KVCacheBlocks: - the outer tuple corresponds to KV cache groups - each inner list contains the block_ids of the blocks in that group + """ groups = ( self.blocks @@ -125,9 +124,7 @@ class KVCacheBlocks: ] def new_empty(self) -> "KVCacheBlocks": - """ - Creates a new KVCacheBlocks instance with no blocks. - """ + """Creates a new KVCacheBlocks instance with no blocks.""" return KVCacheBlocks(tuple(() for _ in range(len(self.blocks)))) @@ -232,6 +229,7 @@ class KVCacheManager: Returns: The KV cache usage (between 0.0 and 1.0). + """ return self.block_pool.get_usage() @@ -240,6 +238,7 @@ class KVCacheManager: Returns: The current prefix caching stats, or None if logging is disabled. + """ if not self.log_stats: return None @@ -278,6 +277,7 @@ class KVCacheManager: window) has not cached yet (Marconi-style APC), or 0 if none. Pinned so sparse prefix-cache retention does not drop the junction and defeat cross-request reuse. + """ # We skip finding the prefix cache hit when prefix caching is # disabled or the request is marked as skipping kv cache read @@ -340,6 +340,7 @@ class KVCacheManager: Returns: The ``get_computed_blocks`` triple (blocks, number of local computed tokens, shared-prefix boundary) plus ``hit_diverged``. + """ coordinator = self.coordinator if not ( @@ -462,6 +463,7 @@ class KVCacheManager: Returns: A list of new allocated blocks. + """ # When loading KV data asynchronously, we may have zero new tokens to # compute while still allocating slots for externally computed tokens. @@ -607,6 +609,7 @@ class KVCacheManager: Args: request: The request to free the blocks. + """ self.coordinator.free(request.request_id) @@ -624,6 +627,7 @@ class KVCacheManager: processed_computed_tokens: Computed-token prefix length covering fully processed and committed tokens only (safe to free). num_prompt_tokens: Optional prompt length for R-SWA gap eviction. + """ self.coordinator.remove_skipped_blocks( request_id, processed_computed_tokens, num_prompt_tokens @@ -639,14 +643,16 @@ class KVCacheManager: Returns: The request's blocks in allocation order. + """ return self.coordinator.pop_blocks_for_free(request.request_id) def evict_blocks(self, block_ids: set[int]) -> None: - """evict blocks from the prefix cache by their block IDs. + """Evict blocks from the prefix cache by their block IDs. Args: block_ids: Set of block IDs to evict from cache. + """ self.block_pool.evict_blocks(block_ids) @@ -658,6 +664,7 @@ class KVCacheManager: Returns: bool: True if the prefix cache is successfully reset, False otherwise. + """ if not self.coordinator.reset_prefix_cache(): return False @@ -697,6 +704,7 @@ class KVCacheManager: Returns: list[int]: The number of common prefix blocks for each kv cache group. + """ return self.coordinator.get_num_common_prefix_blocks(running_request_id) @@ -705,6 +713,7 @@ class KVCacheManager: Returns: A list of KV cache events. + """ events = self.block_pool.take_events() for event in events: @@ -792,6 +801,7 @@ class KVCacheManager: request: The request to cache the blocks. num_computed_tokens: The number of computed tokens, including tokens that are already cached and tokens to be cached. + """ if self.enable_caching: self.coordinator.cache_blocks(request, num_computed_tokens) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index c55c06abf1..656765c896 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -263,6 +263,7 @@ class FreeKVCacheBlockQueue: Args: blocks: A list of KVCacheBlock objects. + """ def __init__(self, blocks: list[KVCacheBlock]) -> None: @@ -300,6 +301,7 @@ class FreeKVCacheBlockQueue: Returns: The first free block. + """ if ( self.fake_free_list_head.next_free_block is self.fake_free_list_tail @@ -340,6 +342,7 @@ class FreeKVCacheBlockQueue: Returns: A list of n free blocks. + """ if n == 0: return [] @@ -370,6 +373,7 @@ class FreeKVCacheBlockQueue: Args: block: The block to remove. + """ if block.prev_free_block is None or block.next_free_block is None: # This should not happen if the block is from the free list. @@ -391,6 +395,7 @@ class FreeKVCacheBlockQueue: Args: block: The block to append. + """ if self.fake_free_list_tail.prev_free_block is None: raise RuntimeError( @@ -434,6 +439,7 @@ class FreeKVCacheBlockQueue: Args: blocks: The blocks to append. + """ if len(blocks) == 0: return @@ -459,6 +465,7 @@ class FreeKVCacheBlockQueue: Returns: A list of free blocks. + """ ret = [] if self.fake_free_list_head.next_free_block is None: @@ -505,6 +512,7 @@ def _gen_mm_extra_hash_keys( Returns: A tuple of extra keys and the next multi-modal index. + """ extra_keys: list[Any] = [] @@ -565,6 +573,7 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]: Returns: Return LoRA name of the request if it is a LoRA request. Return empty list otherwise. + """ if not request.lora_request: return [] @@ -584,6 +593,7 @@ def _gen_prompt_embeds_extra_hash_keys( Returns: Return a stable hash of the block prompt embeddings if prompt embeds are present. Return empty list otherwise. + """ if request.prompt_embeds is None: return [] @@ -612,6 +622,7 @@ def generate_block_hash_extra_keys( Returns: A tuple of extra keys and the next multi-modal index. + """ mm_extra_keys: list[Any] mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys( @@ -645,6 +656,7 @@ def hash_block_tokens( the contents of the preceding block(s). The hash value is used for prefix caching. We use LRU cache for this function to avoid recomputing hash values for the same block contents. + Args: hash_function: The hash function used to compute block hash. parent_block_hash: The hash of the parent block. None @@ -652,9 +664,11 @@ def hash_block_tokens( curr_block_token_ids: A list of token ids in the current block. The current block is assumed to be full. extra_keys: Extra keys for the block. + Returns: The hash value of the block and the token ids in the block. The entire tuple is used as the hash key of the block. + """ if not parent_block_hash: parent_block_hash = NONE_HASH @@ -813,8 +827,7 @@ def get_request_block_hasher( hash_block_size: int, caching_hash_fn: Callable[[Any], bytes], ) -> Callable[[Request], list[BlockHash]]: - """ - Returns a function which computes the list of un-computed block hashes + """Returns a function which computes the list of un-computed block hashes of a request. Hashes are computed at ``hash_block_size`` granularity and chained over the @@ -912,9 +925,7 @@ def _check_enough_kv_cache_memory( def max_memory_usage_bytes( vllm_config: VllmConfig, kv_cache_specs: Iterable[KVCacheSpec] ) -> int: - """ - Get the maximum memory usage in bytes for the given KV cache specs. - """ + """Get the maximum memory usage in bytes for the given KV cache specs.""" return sum(spec.max_memory_usage_bytes(vllm_config) for spec in kv_cache_specs) @@ -923,8 +934,7 @@ def estimate_max_model_len( kv_cache_spec: dict[str, KVCacheSpec], available_memory: int, ) -> int: - """ - Estimates the maximum model length that can fit in the available memory + """Estimates the maximum model length that can fit in the available memory using binary search. This function temporarily modifies max_model_len during estimation but @@ -937,6 +947,7 @@ def estimate_max_model_len( Returns: The estimated maximum model length that can fit in the available memory. + """ # Save the original max_model_len to restore after estimation original_max_model_len = vllm_config.model_config.max_model_len @@ -977,8 +988,7 @@ def check_enough_kv_cache_memory( kv_cache_spec: dict[str, KVCacheSpec], available_memory: int, ): - """ - Checks whether `available_memory` is enough for the KV cache to hold at + """Checks whether `available_memory` is enough for the KV cache to hold at least one request with the model's max_model_len. Args: @@ -988,8 +998,8 @@ def check_enough_kv_cache_memory( Raises: ValueError: If there is not enough memory available for the KV cache. - """ + """ # No need to check for available memory if the kv_cache_spec is empty if kv_cache_spec: # Reserve the null block BlockPool permanently holds back, so the check @@ -1012,8 +1022,7 @@ def check_enough_kv_cache_memory( def create_kv_cache_group_specs( kv_cache_spec: dict[str, KVCacheSpec], grouped_layer_names: list[list[str]] ) -> list[KVCacheGroupSpec]: - """ - Create KVCacheGroupSpec object for each kv cache group layer. + """Create KVCacheGroupSpec object for each kv cache group layer. The layers in the same group should share the same KVCacheSpec. @@ -1024,8 +1033,10 @@ def create_kv_cache_group_specs( A list of kv cache groups, where each element is a list of layer names that belong to the same group and should share the same KVCacheSpec. + Returns: A list of KVCacheGroupSpec objects, one for each group. + """ kv_cache_groups = [] for layer_names_one_group in grouped_layer_names: @@ -1040,8 +1051,7 @@ def create_kv_cache_group_specs( def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool: - """ - Whether all layers in the given KVCacheSpec have the same KV cache spec. + """Whether all layers in the given KVCacheSpec have the same KV cache spec. Note that we regard FullAttentionSpec with and without sliding window as the same type. @@ -1050,8 +1060,8 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool: Returns: True if all layers have the same type, False otherwise. - """ + """ if not kv_cache_spec: # Encoder-only models do not have KV cache, kv_cache_type can be # regarded as uniform. @@ -1067,8 +1077,7 @@ def is_kv_cache_spec_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool: def get_max_concurrency_for_kv_cache_config( vllm_config: VllmConfig, kv_cache_config: KVCacheConfig ) -> float: - """ - Get the maximum concurrency for the given KV cache configuration. + """Get the maximum concurrency for the given KV cache configuration. A request at max_model_len consumes whole blocks from each group's block table — cdiv(per-request bytes, page bytes) of the group's spec — and all @@ -1101,8 +1110,7 @@ def get_max_concurrency_for_kv_cache_config( def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int: - """ - Override the number of kv cache blocks if `num_gpu_blocks_override` is set. + """Override the number of kv cache blocks if `num_gpu_blocks_override` is set. The override is logged once, at the call site in `get_kv_cache_configs`. """ if vllm_config.cache_config.num_gpu_blocks_override is not None: @@ -1111,8 +1119,7 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int: def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int: - """ - Bytes consumed by one block in the worker's shared KV cache pool, mirroring + """Bytes consumed by one block in the worker's shared KV cache pool, mirroring the divisor used by `get_kv_cache_config_from_groups` to convert `available_memory` into `num_blocks`. Used to compute the effective KV cache capacity once `num_gpu_blocks_override` is applied. @@ -1121,9 +1128,7 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int: def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int: - """ - Get the page size of the KV cache. - """ + """Get the page size of the KV cache.""" page_sizes = {layer.page_size_bytes for layer in kv_cache_specs} assert len(page_sizes) == 1 return page_sizes.pop() @@ -1132,8 +1137,7 @@ def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int: def _get_kv_cache_groups_uniform_spec( kv_cache_specs: dict[str, KVCacheSpec], ) -> list[KVCacheGroupSpec]: - """ - Generates the KV cache configuration for a model with the same KV cache + """Generates the KV cache configuration for a model with the same KV cache spec for all layers. Args: @@ -1141,16 +1145,15 @@ def _get_kv_cache_groups_uniform_spec( Returns: The generated KVCacheGroupSpecs - """ + """ return create_kv_cache_group_specs(kv_cache_specs, [list(kv_cache_specs.keys())]) def _get_kv_cache_groups_uniform_type( spec: UniformTypeKVCacheSpecs, ) -> list[KVCacheGroupSpec]: - """ - Generates the KV cache configuration for a model with one type of KV cache + """Generates the KV cache configuration for a model with one type of KV cache but different hidden sizes. All layers are merged into one group. Args: @@ -1158,8 +1161,8 @@ def _get_kv_cache_groups_uniform_type( Returns: The generated KVCacheGroupSpecs - """ + """ return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)] @@ -1365,8 +1368,7 @@ def _glm5_next_tensor_layout( def unify_kv_cache_spec_page_size( kv_cache_spec: dict[str, KVCacheSpec], ) -> dict[str, KVCacheSpec]: - """ - Unify the page size of the given KVCacheSpec. If the page size of all layers + """Unify the page size of the given KVCacheSpec. If the page size of all layers are the same, return the original KVCacheSpec. If not same, unify the page size by increasing the block size of layers with smaller page size. Two cases cannot be unified by block size alone and pad their physical page to @@ -1385,6 +1387,7 @@ def unify_kv_cache_spec_page_size( Returns: The updated KVCacheSpec with the same page_size_bytes. + """ page_sizes = {layer.page_size_bytes for layer in kv_cache_spec.values()} if len(page_sizes) <= 1: @@ -1435,8 +1438,7 @@ def is_kv_cache_type_attention_free(kv_cache_spec: dict[str, KVCacheSpec]) -> bo def _get_kv_cache_groups_uniform_page_size( kv_cache_spec: dict[str, KVCacheSpec], ) -> list[KVCacheGroupSpec]: - """ - Generates the KV cache groups for hybrid models with multiple + """Generates the KV cache groups for hybrid models with multiple attention types but still with a uniform page size (physical memory per block per layer) for all layers. @@ -1496,6 +1498,7 @@ def _get_kv_cache_groups_uniform_page_size( kv_cache_spec: The KVCacheSpec of each attention layer in the model Returns: The generated KVCacheGroupSpecs + """ # Group all layers by kv_cache_spec. # E.g., 2 full attention layers and 3 sliding window attention layers, @@ -1659,8 +1662,7 @@ def get_kv_cache_config_from_groups( kv_cache_groups: list[KVCacheGroupSpec], available_memory: int, ) -> KVCacheConfig: - """ - Generate the KV cache configuration from the KV cache groups and spec + """Generate the KV cache configuration from the KV cache groups and spec of each layer. Args: @@ -1669,6 +1671,7 @@ def get_kv_cache_config_from_groups( available_memory: Memory available for KV cache in bytes Returns: The generated KVCacheConfig + """ if len(kv_cache_groups) == 0: # Attention free models do not have KV cache. @@ -1926,15 +1929,14 @@ def _try_get_full_allocation_fallback_groups( def unify_hybrid_kv_cache_specs(kv_cache_spec: dict[str, KVCacheSpec]): - """ - This function tries to convert the KV cache specs to one type if the model + """This function tries to convert the KV cache specs to one type if the model is a hybrid model with multiple type of KV cache. It will convert all SlidingWindowSpec to FullAttentionSpec if both types are present. Args: kv_cache_spec: The kv cache spec of each attention layer in the model - """ + """ if is_kv_cache_spec_uniform( kv_cache_spec ) or UniformTypeKVCacheSpecs.is_uniform_type(kv_cache_spec): @@ -2166,6 +2168,7 @@ def _annotate_eagle_groups( kv_cache_groups: Groups to annotate in place. use_deepseek_v4_fallback: Enable rule 2 for a DeepseekV4/V4.1 packed group. + """ spec_config = vllm_config.speculative_config if spec_config is None or not spec_config.use_eagle_block_drop(): @@ -2196,6 +2199,7 @@ def _warn_if_unannotated_eagle_mamba( Args: vllm_config: Config supplying the speculative method, if any. kv_cache_groups: Groups as they will be handed to consumers. + """ spec_config = vllm_config.speculative_config if spec_config is None or not spec_config.use_eagle(): @@ -2262,8 +2266,7 @@ def get_kv_cache_groups( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec], ) -> list[KVCacheGroupSpec]: - """ - Split the layers in the model into groups with the same KV cache spec. + """Split the layers in the model into groups with the same KV cache spec. Args: vllm_config: The global VllmConfig @@ -2271,6 +2274,7 @@ def get_kv_cache_groups( Returns: The generated KVCacheGroups + """ if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager: unify_hybrid_kv_cache_specs(kv_cache_spec) @@ -2356,9 +2360,7 @@ def get_kv_cache_groups( def generate_scheduler_kv_cache_config( kv_cache_configs: list[KVCacheConfig], ) -> KVCacheConfig: - """ - Generate the KV cache configuration for the scheduler. - """ + """Generate the KV cache configuration for the scheduler.""" assert all( [cfg.num_blocks == kv_cache_configs[0].num_blocks for cfg in kv_cache_configs] ) @@ -2382,9 +2384,7 @@ def generate_scheduler_kv_cache_config( def get_kv_cache_capacity( vllm_config: VllmConfig, kv_cache_config: KVCacheConfig ) -> tuple[int, float]: - """ - Get the group-aware KV cache token capacity and max concurrency. - """ + """Get the group-aware KV cache token capacity and max concurrency.""" max_model_len = vllm_config.model_config.max_model_len max_concurrency = get_max_concurrency_for_kv_cache_config( vllm_config, kv_cache_config @@ -2415,8 +2415,7 @@ def _max_memory_usage_bytes_from_groups( vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec], ) -> int: - """ - Calculate maximum memory usage in bytes from KV cache groups. + """Calculate maximum memory usage in bytes from KV cache groups. This correctly accounts for padding in hybrid models. For example, if a model has 8 full attention layers and 9 sliding window layers, they will @@ -2475,8 +2474,7 @@ def _estimate_max_model_len_from_groups( kv_cache_groups: list[KVCacheGroupSpec], available_memory: int, ) -> int: - """ - Binary search for the maximum model length that fits in available memory. + """Binary search for the maximum model length that fits in available memory. Returns 0 if even 1 token doesn't fit. """ original_max = vllm_config.model_config.max_model_len @@ -2522,8 +2520,7 @@ def _auto_fit_max_model_len( projected_groups_per_worker: list[list[KVCacheGroupSpec]], available_memory: list[int], ) -> None: - """ - When max_model_len is set to -1, this function estimates the largest + """When max_model_len is set to -1, this function estimates the largest context length that can be supported with the available GPU memory. It uses binary search to find the maximum length that fits across all workers. @@ -2533,6 +2530,7 @@ def _auto_fit_max_model_len( projected_groups_per_worker: KV cache groups projected to each worker. available_memory: Memory available for KV cache in bytes for each worker. + """ original_max = vllm_config.model_config.max_model_len @@ -2585,8 +2583,7 @@ def _project_kv_cache_groups_to_worker( global_kv_cache_groups: list[KVCacheGroupSpec], worker_spec: dict[str, KVCacheSpec], ) -> list[KVCacheGroupSpec]: - """ - Projects global KV cache groups onto a single worker's assigned layers. + """Projects global KV cache groups onto a single worker's assigned layers. In pipeline parallelism, each worker only owns a subset of layers. This function filters the global groups to include only layers present on the @@ -2598,6 +2595,7 @@ def _project_kv_cache_groups_to_worker( Returns: The projected KV cache groups containing only this worker's layers. + """ projected_groups: list[KVCacheGroupSpec] = [] for group in global_kv_cache_groups: @@ -2628,8 +2626,7 @@ def get_kv_cache_configs( kv_cache_specs: list[dict[str, KVCacheSpec]], available_memory: list[int], ) -> list[KVCacheConfig]: - """ - Generates the KV cache configurations for a model. + """Generates the KV cache configurations for a model. Since we use a shared centralized controller for all workers, we need the `kv_cache_config` to be consistent across all workers to make sure the KV cache allocation can be applied to all workers. However, different @@ -2656,8 +2653,8 @@ def get_kv_cache_configs( Returns: The generated KVCacheConfigs for each worker. - """ + """ # Merge the KV cache specs of all workers. Different PP stages may have # different layer names, and different TP ranks of the same PP stage should # have the same KV cache spec. @@ -2778,8 +2775,7 @@ def get_kv_cache_configs( class BlockHashListWithBlockSize: - """ - Convert block-hash granularity from `hash_block_size` to `target_block_size`. + """Convert block-hash granularity from `hash_block_size` to `target_block_size`. Used when KV cache groups have different block sizes: `hash_block_size` is the size used to compute the original `block_hashes`; `target_block_size` is the group's actual block size. @@ -2809,6 +2805,7 @@ class BlockHashListWithBlockSize: block_hashes: Block hashes to convert, computed at `hash_block_size`. hash_block_size: Block size at which `block_hashes` were computed. target_block_size: Desired block size; must be a multiple of `hash_block_size`. + """ def __init__( diff --git a/vllm/v1/core/sched/interface.py b/vllm/v1/core/sched/interface.py index c15296dd05..c868d249c8 100644 --- a/vllm/v1/core/sched/interface.py +++ b/vllm/v1/core/sched/interface.py @@ -79,6 +79,7 @@ class SchedulerInterface(ABC): Returns: A SchedulerOutput object containing information about the scheduled requests. + """ raise NotImplementedError @@ -105,6 +106,7 @@ class SchedulerInterface(ABC): Returns: A dict of client index to EngineCoreOutputs object containing the outputs for each request originating from that client. + """ raise NotImplementedError @@ -115,6 +117,7 @@ class SchedulerInterface(ABC): Args: draft_token_ids: The input draft token ids for each request. + """ raise NotImplementedError @@ -129,6 +132,7 @@ class SchedulerInterface(ABC): draft_token_ids: The input draft token ids for each request. scheduler_output: Update the given scheduler_output with the corresponding draft token ids. + """ raise NotImplementedError @@ -138,6 +142,7 @@ class SchedulerInterface(ABC): Args: request: The new request being added. + """ raise NotImplementedError @@ -162,6 +167,7 @@ class SchedulerInterface(ABC): Returns: List of requests that were aborted. Will not include any that were already finished. + """ raise NotImplementedError @@ -218,6 +224,8 @@ class SchedulerInterface(ABC): preempted and moved to the waiting queue. Otherwise, this method will only reset the KV prefix cache when there is no running request taking KV cache. + reset_connector: If True, also reset any KV connector state. + """ raise NotImplementedError diff --git a/vllm/v1/core/sched/request_queue.py b/vllm/v1/core/sched/request_queue.py index 38c7db94a4..ce58e6e1c7 100644 --- a/vllm/v1/core/sched/request_queue.py +++ b/vllm/v1/core/sched/request_queue.py @@ -129,8 +129,7 @@ class FCFSRequestQueue(deque[Request], RequestQueue): class PriorityRequestQueue(RequestQueue): - """ - A priority queue that supports heap operations. + """A priority queue that supports heap operations. Respects the ordering defined in the Request class, where requests with a smaller value of `priority` are processed first. diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index f1e11e9b1f..19ea411aa1 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry from vllm.multimodal.encoder_budget import MultiModalBudget from vllm.multimodal.utils import get_mm_features_in_window +from vllm.utils.math_utils import cdiv from vllm.v1.core.encoder_cache_manager import ( EncoderCacheManager, EncoderDecoderCacheManager, @@ -1140,9 +1141,13 @@ class Scheduler(SchedulerInterface): if load_kv_async: # An async load holds its blocks for the whole transfer with # no forward progress and isn't preemptible here. Admit it - # only if it fits in (free - other in-flight reservations), to - # avoid deadlock and predictable preemptions. - reserved_blocks = self._inflight_prefill_reserved_blocks() + # only if it fits in (free - other in-flight reservations) + # plus its own spec decode step blocks, to avoid deadlock and + # predictable preemptions. + reserved_blocks = ( + self._inflight_prefill_reserved_blocks() + + self._spec_decode_step_blocks() + ) new_blocks = self.kv_cache_manager.allocate_slots( request, @@ -1558,12 +1563,10 @@ class Scheduler(SchedulerInterface): def _update_request_as_session( self, session: Request, update: StreamingUpdate ) -> None: - """ - Updates the waiting session with the next streaming update. + """Updates the waiting session with the next streaming update. Discards the last sampled output token from the prior input chunk. """ - # Current streaming input behaviour: Keep only computed output tokens # (discard final sampled output token). num_computed_tokens = session.num_computed_tokens @@ -1665,8 +1668,7 @@ class Scheduler(SchedulerInterface): encoder_compute_budget: int, shift_computed_tokens: int = 0, ) -> tuple[list[int], int, int, list[int]]: - """ - Determine which encoder inputs need to be scheduled in the current step, + """Determine which encoder inputs need to be scheduled in the current step, and update `num_new_tokens` and encoder token budget accordingly. An encoder input will be scheduled if: @@ -2110,6 +2112,17 @@ class Scheduler(SchedulerInterface): finished = self._handle_stopped_request(request) if finished: kv_transfer_params, ec_transfer_params = self._free_request(request) + if ( + prefill_stats is not None + and kv_transfer_params is not None + and kv_transfer_params.get("do_remote_prefill") + ): + # P-side cache hits, so D can report them in + # prompt_tokens_details instead of its own (~100%) + # hit rate from the KV transfer. + kv_transfer_params["remote_prefill_cached_tokens"] = ( + prefill_stats.num_cached_tokens + ) if status_before_stop == RequestStatus.RUNNING: stopped_running_reqs.add(request) @@ -2496,6 +2509,7 @@ class Scheduler(SchedulerInterface): Returns: List of requests that were aborted. Will not include any that were already finished. + """ assert RequestStatus.is_finished(finished_status) if isinstance(request_ids, str): @@ -2830,8 +2844,7 @@ class Scheduler(SchedulerInterface): def _connector_finished( self, request: Request ) -> tuple[bool, dict[str, Any] | None]: - """ - Invoke the KV connector request_finished() method if applicable. + """Invoke the KV connector request_finished() method if applicable. Returns optional kv transfer parameters to be included with the request outputs. @@ -2884,9 +2897,9 @@ class Scheduler(SchedulerInterface): return delay_free or partial_tail_delay, kv_xfer_params def _request_remaining_blocks(self, request: Request) -> int: - """Blocks `request` still needs to allocate to hold its full sequence.""" + """Blocks `request` still needs to hold its full sequence and be promoted.""" full_num_tokens = min(request.num_tokens, self.max_model_len) - return self.kv_cache_manager.coordinator.get_num_blocks_to_allocate( + num_blocks = self.kv_cache_manager.coordinator.get_num_blocks_to_allocate( request_id=request.request_id, num_tokens=full_num_tokens, new_computed_blocks=self.kv_cache_manager.empty_kv_cache_blocks.blocks, @@ -2896,17 +2909,29 @@ class Scheduler(SchedulerInterface): num_tokens_main_model=full_num_tokens, apply_admission_cap=True, ) + return num_blocks + self._spec_decode_step_blocks() + + def _spec_decode_step_blocks(self) -> int: + """Number of blocks for the extra KV slots a spec decode step needs. + + When using async kv load, scheduler must reserve enough blocks for + full sequence + the spec decode step, otherwise request cannot be + able to run after the async_load if we are out of kv blocks. + """ + if not self.num_spec_tokens: + return 0 + return cdiv( + 1 + self.num_spec_tokens + self.num_lookahead_tokens, self.block_size + ) def _inflight_prefill_reserved_blocks(self) -> int: """Num blocks in-flight prefills still need to finish (their reservation).""" - return sum( self._request_remaining_blocks(req) for req in self._inflight_prefills ) def _update_waiting_for_remote_kv(self, request: Request) -> None: - """ - KV Connector: update request state after async recv is finished. + """KV Connector: update request state after async recv is finished. When the kv transfer is ready, we cache the blocks and the request state will be moved back to WAITING from @@ -2948,9 +2973,7 @@ class Scheduler(SchedulerInterface): self.finished_recving_kv_req_ids.remove(request.request_id) def _try_promote_blocked_waiting_request(self, request: Request) -> bool: - """ - Try to promote a blocked waiting request back to schedulable states. - """ + """Try to promote a blocked waiting request back to schedulable states.""" if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS: # finished_recving_kv_req_ids is populated during # update_from_output(), based on worker-side connector signals @@ -2984,8 +3007,7 @@ class Scheduler(SchedulerInterface): ) def _update_from_kv_xfer_finished(self, kv_connector_output: KVConnectorOutput): - """ - KV Connector: update the scheduler state based on the output. + """KV Connector: update the scheduler state based on the output. The Worker side connectors add finished_recving and finished_sending reqs to the output. @@ -2993,7 +3015,6 @@ class Scheduler(SchedulerInterface): # if finished_recving: add to state so we can schedule the request during the next step. """ - if self.connector is not None: self.connector.update_connector_output(kv_connector_output) @@ -3019,19 +3040,13 @@ class Scheduler(SchedulerInterface): num_scheduled_tokens: dict[str, int], evict_blocks: bool = True, ) -> tuple[set[str], int, set[int]]: - """ - Identify and update requests affected by invalid KV cache blocks. + """Identify and update requests affected by invalid KV cache blocks. This method scans the given requests, detects those with invalid blocks and adjusts their `num_computed_tokens` to the longest valid prefix. For observability, it also accumulates the total number of tokens that will need to be recomputed across all affected requests. - A hybrid model spreads one token span over KV cache groups that disagree - on block size, so a block index maps to a different token position in - each group and there is no single longest valid prefix. Those requests - recompute in full instead. - Args: requests: The set of requests to scan for invalid blocks. invalid_block_ids: IDs of invalid blocks. @@ -3047,6 +3062,7 @@ class Scheduler(SchedulerInterface): be recomputed across all affected requests. - blocks_to_evict (set[int]): Block IDs to evict from cache, including invalid blocks and downstream dependent blocks. + """ affected_req_ids: set[str] = set() total_affected_tokens = 0 @@ -3055,49 +3071,19 @@ class Scheduler(SchedulerInterface): # these requests must be rescheduled, but only the first will recompute # it. This set tracks blocks already marked for recomputation. marked_invalid_block_ids: set[int] = set() - # Positions a group holds no KV for -- skipped sliding-window blocks, - # and every Mamba block but the aligned snapshot -- all point at this - # one block, shared by every request. It carries no request's data, so - # it can neither fail for a request nor be evicted on its behalf. - null_block_id = self.kv_cache_manager.block_pool.null_block.block_id for request in requests: is_affected = False marked_invalid_block = False req_id = request.request_id - # One block-id list per KV cache group. - req_block_ids_per_group = self.kv_cache_manager.get_block_ids(req_id) + # One block-id list per KV cache group; single-group layouts are + # enforced by _handle_invalid_blocks. + (req_block_ids,) = self.kv_cache_manager.get_block_ids(req_id) # We iterate only over blocks that may contain externally computed # tokens req_num_computed_tokens = ( request.num_computed_tokens - num_scheduled_tokens.get(req_id, 0) ) - if len(req_block_ids_per_group) > 1: - # No single truncation point exists across groups, so the whole - # request is recomputed. Blocks are still marked, so a request - # sharing one of them is rescheduled too. - request_invalid_block_ids = { - block_id - for req_block_ids in req_block_ids_per_group - for block_id in req_block_ids - if block_id != null_block_id and block_id in invalid_block_ids - } - if request_invalid_block_ids: - marked_invalid_block_ids |= request_invalid_block_ids - total_affected_tokens += req_num_computed_tokens - request.num_computed_tokens = 0 - if evict_blocks: - for req_block_ids in req_block_ids_per_group: - blocks_to_evict.update( - block_id - for block_id in req_block_ids - if block_id != null_block_id - ) - affected_req_ids.add(req_id) - continue - - (req_block_ids,) = req_block_ids_per_group - req_num_computed_blocks = ( req_num_computed_tokens + self.block_size - 1 ) // self.block_size @@ -3169,12 +3155,23 @@ class Scheduler(SchedulerInterface): def _handle_invalid_blocks( self, invalid_block_ids: set[int], num_scheduled_tokens: dict[str, int] ) -> set[str]: - """ - Handle requests affected by invalid KV cache blocks. + """Handle requests affected by invalid KV cache blocks. Returns: Set of affected request IDs to skip in update_from_output main loop. + """ + if len(self.kv_cache_config.kv_cache_groups) > 1: + # Block IDs are only unique within a group, so a flat set of + # invalid block IDs cannot be mapped back to requests. + raise RuntimeError( + "A KV connector reported block-level load failures " + "(invalid_block_ids) on a layout with multiple KV cache " + "groups, where block IDs are only unique within a group. " + "Connectors must report failed requests via " + "KVConnectorTransferResults.failed_recving instead." + ) + should_fail = not self.recompute_kv_load_failures # handle async KV loads (not cached yet, evict_blocks=False) diff --git a/vllm/v1/core/sched/utils.py b/vllm/v1/core/sched/utils.py index 31fa840e77..045dacaa98 100644 --- a/vllm/v1/core/sched/utils.py +++ b/vllm/v1/core/sched/utils.py @@ -30,11 +30,14 @@ def check_sequence_repetition( params: RepetitionDetectionParams, ) -> bool: """Check if a sequence of token IDs has a repetition pattern. + Args: token_ids: List of token IDs params: Repetition detection parameters. + Returns: True if a repetition pattern is found, False otherwise. + """ max_pattern_size = params.max_pattern_size min_pattern_size = params.min_pattern_size @@ -77,6 +80,7 @@ def remove_all(lst: list, items_to_remove: set) -> list: Note: For single item removal, this modifies the original list in-place and returns it. For multiple items, it creates and returns a new list. + """ if not items_to_remove: return lst diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 08eab76287..9ad0a38ed5 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -49,8 +49,7 @@ logger = init_logger(__name__) class SingleTypeKVCacheManager(ABC): - """ - An abstract base class for a manager that handle the kv cache management + """An abstract base class for a manager that handle the kv cache management logic of one specific type of attention layer. """ @@ -81,14 +80,17 @@ class SingleTypeKVCacheManager(ABC): needs_kv_cache_zeroing: bool = False, max_admission_blocks_per_request: int | None = None, ) -> None: - """ - Initializes the SingleTypeKVCacheManager. + """Initializes the SingleTypeKVCacheManager. + Args: kv_cache_spec: The kv_cache_spec for this manager. block_pool: The block pool. + enable_caching: Whether prefix caching is enabled. kv_cache_group_id: The id of the kv cache group of this manager. scheduler_block_size: The scheduling granularity (LCM of all group block sizes); a multiple of this manager's ``block_size``. + dcp_world_size: Decode context parallel world size. + pcp_world_size: Prefill context parallel world size. needs_kv_cache_zeroing: Whether worker-side KV cache zeroing needs newly allocated block IDs from this manager. max_admission_blocks_per_request: Recycling-aware per-request @@ -97,6 +99,7 @@ class SingleTypeKVCacheManager(ABC): chunked-local); `None` (the default) means no cap, which is correct for full-attention-style specs that hold every block until the request finishes. + """ self.scheduler_block_size = scheduler_block_size # Hybrid fine-grained lookup may lower this after all participating @@ -184,8 +187,7 @@ class SingleTypeKVCacheManager(ABC): num_tokens_main_model: int, apply_admission_cap: bool = False, ) -> int: - """ - Get the number of blocks needed to be allocated for the request. + """Get the number of blocks needed to be allocated for the request. Args: request_id: The request ID. @@ -206,8 +208,8 @@ class SingleTypeKVCacheManager(ABC): Returns: The number of blocks to allocate. - """ + """ num_required_blocks = cdiv(num_tokens, self.block_size) if apply_admission_cap and self._max_admission_blocks_per_request is not None: # Recycling-aware specs (SWA, chunked-local) cap the per-request @@ -269,8 +271,7 @@ class SingleTypeKVCacheManager(ABC): num_local_computed_tokens: int, num_external_computed_tokens: int, ) -> None: - """ - Add the locally cached (prefix-hit) blocks to the request: + """Add the locally cached (prefix-hit) blocks to the request: 1. Touch the computed blocks (paired with adding them to `req_blocks`) so their ref_cnt exactly tracks the referencing requests. 1.5. (Optional) For sliding window, skipped blocks are padded with nulls. @@ -282,6 +283,7 @@ class SingleTypeKVCacheManager(ABC): prefix cache. num_local_computed_tokens: The number of local computed tokens. num_external_computed_tokens: The number of external computed tokens. + """ # The coordinator only calls this for first-time allocations (running # requests are short-circuited there), so the request has no blocks yet. @@ -327,8 +329,7 @@ class SingleTypeKVCacheManager(ABC): num_local_computed_tokens: int, num_external_computed_tokens: int, ) -> None: - """ - Allocate new blocks for external (KV-connector) computed tokens. + """Allocate new blocks for external (KV-connector) computed tokens. Must run only after every group's local blocks have been touched via `add_local_computed_blocks`, so this group's `get_new_blocks` cannot @@ -338,6 +339,7 @@ class SingleTypeKVCacheManager(ABC): request_id: The request ID. num_local_computed_tokens: The number of local computed tokens. num_external_computed_tokens: The number of external computed tokens. + """ num_total_computed_tokens = ( num_local_computed_tokens + num_external_computed_tokens @@ -364,8 +366,7 @@ class SingleTypeKVCacheManager(ABC): def allocate_new_blocks( self, request_id: str, num_tokens: int, num_tokens_main_model: int ) -> list[KVCacheBlock]: - """ - Allocate new blocks for the request to give it at least `num_tokens` + """Allocate new blocks for the request to give it at least `num_tokens` token slots. Args: @@ -375,8 +376,10 @@ class SingleTypeKVCacheManager(ABC): num_tokens_main_model: The number of tokens for the main model (aka target model in spec decode). w/o spec decode, it is num_tokens; with spec decode, it is num_tokens - num_lookahead_tokens. + Returns: The new allocated blocks. + """ cow_blocks: list[KVCacheBlock] = [] if request_id in self._partial_hit_reqs: @@ -475,8 +478,7 @@ class SingleTypeKVCacheManager(ABC): *, replay_boundaries: Sequence[int], ) -> None: - """ - Cache the blocks for the request. + """Cache the blocks for the request. Args: request: The request. @@ -488,6 +490,7 @@ class SingleTypeKVCacheManager(ABC): a tail once per that-sized segment. Only SWA acts on it. replay_boundaries: Positions a later request replaying this prompt can resume at, from ``get_replay_boundaries``. + """ num_cached_blocks = self.num_cached_block.get(request.request_id, 0) num_full_blocks = num_tokens // self.block_size @@ -552,8 +555,7 @@ class SingleTypeKVCacheManager(ABC): return None def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: - """ - Pop the request's bookkeeping and return its blocks without yet + """Pop the request's bookkeeping and return its blocks without yet returning them to the block pool. The caller is responsible for eventually passing the returned blocks to `block_pool.free_blocks`, freeing them in reverse order (so that tail blocks are evicted first). @@ -563,6 +565,7 @@ class SingleTypeKVCacheManager(ABC): Returns: The request's blocks in allocation order. + """ # Default to [] in case a request is freed (aborted) before alloc. req_blocks = self.req_to_blocks.pop(request_id, []) @@ -571,19 +574,18 @@ class SingleTypeKVCacheManager(ABC): return req_blocks def free(self, request_id: str) -> None: - """ - Free the blocks for the request. + """Free the blocks for the request. Args: request_id: The request ID. + """ # Free blocks in reverse order so that the tail blocks are freed first. self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id))) @abstractmethod def get_num_common_prefix_blocks(self, running_request_id: str) -> int: - """ - Get the number of common prefix blocks for all requests with allocated + """Get the number of common prefix blocks for all requests with allocated KV cache. Args: @@ -592,8 +594,8 @@ class SingleTypeKVCacheManager(ABC): Returns: The number of common prefix blocks for all requests with allocated KV cache. - """ + """ raise NotImplementedError @classmethod @@ -610,8 +612,7 @@ class SingleTypeKVCacheManager(ABC): dcp_world_size: int = 1, pcp_world_size: int = 1, ) -> tuple[tuple[list[KVCacheBlock], ...], int]: - """ - Get the longest cache hit prefix of the blocks that is not longer than + """Get the longest cache hit prefix of the blocks that is not longer than `max_length`. The prefix should be a common prefix hit for all the kv cache groups in `kv_cache_group_ids`. If no cache hit is found, return an empty list. @@ -647,8 +648,8 @@ class SingleTypeKVCacheManager(ABC): For example, sliding window manager should return a list like ([NULL, NULL, KVCacheBlock(7), KVCacheBlock(8)]) for block size 4 and sliding window 8 and len(kv_cache_group_ids) = 1. - """ + """ raise NotImplementedError def _remove_blocks_in_range( @@ -684,8 +685,8 @@ class SingleTypeKVCacheManager(ABC): processed_computed_tokens: int, num_prompt_tokens: int | None = None, ) -> None: - """ - Remove and free the blocks that are no longer needed for attention computation. + """Remove and free blocks no longer needed for attention computation. + The removed blocks should be replaced by null_block. This function depends on `get_num_skipped_tokens`, which need to be implemented @@ -698,6 +699,7 @@ class SingleTypeKVCacheManager(ABC): num_prompt_tokens: Optional prompt length for attention types (e.g. R-SWA) that evict a middle gap rather than a head prefix. Ignored by the default implementation. + """ del num_prompt_tokens # Remove the blocks that will be skipped during attention computation. @@ -718,14 +720,14 @@ class SingleTypeKVCacheManager(ABC): self._remove_blocks_in_range(request_id, 0, num_skipped_blocks) def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: - """ - Get the number of tokens that will be skipped for attention computation. + """Get the number of tokens that will be skipped for attention computation. Args: num_computed_tokens: The number of tokens that have been computed. Returns: The number of tokens that will be skipped for attention computation. + """ # The default behavior is to not skip any tokens. return 0 @@ -1144,8 +1146,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager): return mask def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: - """ - Get the number of tokens that will be skipped for attention computation. + """Get the number of tokens that will be skipped for attention computation. For sliding window, this corresponds to the tokens that are prior to the current sliding window. @@ -1174,6 +1175,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager): Returns: The number of tokens that will be skipped for attention computation. + """ return max( 0, @@ -1181,8 +1183,7 @@ class SlidingWindowManager(SingleTypeKVCacheManager): ) def get_num_common_prefix_blocks(self, running_request_id: str) -> int: - """ - NOTE(Chen): The prefix blocks are null blocks for sliding window layers. + """NOTE(Chen): The prefix blocks are null blocks for sliding window layers. So it's not correct to count ref_cnt like FullAttentionManager. Return 0 here for correctness. Need to support cascade attention + sliding window in the future. @@ -1301,8 +1302,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): dcp_world_size: int = 1, pcp_world_size: int = 1, ) -> tuple[tuple[list[KVCacheBlock], ...], int]: - """ - For chunked local attention, we need to find the longest cache hit + """For chunked local attention, we need to find the longest cache hit prefix of the blocks that is not longer than `max_length`. The prefix should be a common prefix hit for all the kv cache groups in `kv_cache_group_ids`. If no cache hit is found, return an empty list. @@ -1335,6 +1335,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): Returns: A list of cached blocks + """ assert isinstance(kv_cache_spec, ChunkedLocalAttentionSpec), ( "ChunkedLocalAttentionManager can only be used for " @@ -1389,8 +1390,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): return computed_blocks, hit_length def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: - """ - Get the number of tokens that will be skipped for attention computation. + """Get the number of tokens that will be skipped for attention computation. For chunked local attention, this corresponds to the tokens that are on the left side of the current chunk. @@ -1428,6 +1428,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): Returns: The number of tokens that will be skipped for attention computation. + """ num_skipped_tokens = ( num_computed_tokens // self.attention_chunk_size @@ -1435,9 +1436,7 @@ class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): return num_skipped_tokens def get_num_common_prefix_blocks(self, running_request_id: str) -> int: - """ - cascade attention is not supported by chunked local attention. - """ + """Cascade attention is not supported by chunked local attention.""" return 0 @@ -1674,9 +1673,7 @@ class MambaManager(SingleTypeKVCacheManager): blocks[last_state_block_idx] = self._null_block def get_num_common_prefix_blocks(self, running_request_id: str) -> int: - """ - cascade attention is not supported by mamba - """ + """Cascade attention is not supported by mamba""" return 0 def _needs_internal_checkpoint( @@ -1979,8 +1976,7 @@ class MambaManager(SingleTypeKVCacheManager): return super().pop_blocks_for_free(request_id) def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: - """ - Get the number of tokens whose mamba state are not needed anymore. Mamba only + """Get the number of tokens whose mamba state are not needed anymore. Mamba only need to keep the state of the last computed token, so we return num_computed_tokens - 1. """ @@ -2600,8 +2596,7 @@ def get_manager_for_kv_cache_spec( role: str | None = None, **kwargs, ) -> SingleTypeKVCacheManager: - """ - Get the appropriate manager for a given KVCacheSpec. + """Get the appropriate manager for a given KVCacheSpec. Uses the KVCacheSpecRegistry to look up the manager class, supporting both built-in and custom specs registered via @register_kv_cache_spec @@ -2612,8 +2607,11 @@ def get_manager_for_kv_cache_spec( max_in_flight_tokens: The max tokens scheduled but not yet settled (one batch per concurrent step); see `VllmConfig.max_in_flight_tokens` max_model_len: The maximum context length the model could serve + role: Optional KV cache group role used to select a role-specific + manager Returns: An instance of the appropriate SingleTypeKVCacheManager subclass + """ manager_class = KVCacheSpecRegistry.get_manager_class(kv_cache_spec, role) assert manager_class is not None, ( diff --git a/vllm/v1/cudagraph_dispatcher.py b/vllm/v1/cudagraph_dispatcher.py index 6a48b6282d..0c1fb0b2c4 100644 --- a/vllm/v1/cudagraph_dispatcher.py +++ b/vllm/v1/cudagraph_dispatcher.py @@ -13,8 +13,7 @@ logger = init_logger(__name__) class CudagraphDispatcher: - """ - Runtime cudagraph dispatcher to dispatch keys for multiple set of + """Runtime cudagraph dispatcher to dispatch keys for multiple set of cudagraphs. The dispatcher stores two sets of dispatch keys, one for PIECEWISE and one @@ -109,8 +108,7 @@ class CudagraphDispatcher: ) def _get_lora_cases(self) -> list[int]: - """ - Returns list of has_lora values for CUDA graph capture. + """Returns list of has_lora values for CUDA graph capture. This is the single source of truth for LoRA capture cases. """ lora_config = self.vllm_config.lora_config @@ -241,8 +239,7 @@ class CudagraphDispatcher: valid_modes: AbstractSet[CUDAGraphMode] | None = None, invalid_modes: AbstractSet[CUDAGraphMode] | None = None, ) -> tuple[CUDAGraphMode, BatchDescriptor]: - """ - Given conditions(e.g.,batch descriptor and if using piecewise only), + """Given conditions(e.g.,batch descriptor and if using piecewise only), dispatch to a cudagraph runtime mode and the valid batch descriptor. A new batch descriptor is returned as we might dispatch a uniform batch to a graph that supports a more general batch (uniform to non-uniform). @@ -259,6 +256,7 @@ class CudagraphDispatcher: valid_modes to compute allowed modes. (e.g., {FULL} for features like cascade attention not supported by full cudagraphs). None means no modes are excluded. + """ allowed_modes = valid_modes or CUDAGraphMode.valid_runtime_modes() @@ -324,13 +322,13 @@ class CudagraphDispatcher: return CUDAGraphMode.NONE, BatchDescriptor(num_tokens) def get_capture_descs(self) -> list[tuple[CUDAGraphMode, list[BatchDescriptor]]]: - """ - Returns capture descriptors for cudagraph capturing. + """Returns capture descriptors for cudagraph capturing. Returns: List of (runtime_mode, batch_descriptors) tuples, ordered PIECEWISE first then FULL. Batch descriptors are sorted largest-first for memory efficiency. + """ if not self.keys_initialized or self.cudagraph_mode == CUDAGraphMode.NONE: return [] diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index 7d68b6e797..7c424ea0df 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -45,8 +45,7 @@ class EEPNotificationType(enum.Enum): class FinishReason(enum.IntEnum): - """ - Reason a request finished - stop, length, abort, error, or repetition. + """Reason a request finished - stop, length, abort, error, or repetition. Int rather than Str for more compact serialization. @@ -284,8 +283,7 @@ class EngineCoreOutputs( class EngineCoreRequestType(enum.Enum): - """ - Request types defined as hex byte strings, so it can be sent over sockets + """Request types defined as hex byte strings, so it can be sent over sockets without separate encoding step. """ @@ -310,9 +308,7 @@ class ReconfigureDistributedRequest(msgspec.Struct): class ReconfigureRankType(enum.IntEnum): - """ - Rank type for reconfiguring distributed request. - """ + """Rank type for reconfiguring distributed request.""" KEEP_CURRENT_RANK = -1 SHUTDOWN_CURRENT_RANK = -2 diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index bf3f5f420a..3e399a1a81 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -94,8 +94,7 @@ class AsyncLLM(EngineClient): client_index: int = 0, profiler: TorchProfilerWrapper | None = None, ) -> None: - """ - Create an AsyncLLM. + """Create an AsyncLLM. Args: vllm_config: global configuration. @@ -105,13 +104,21 @@ class AsyncLLM(EngineClient): mm_registry: Multi-modal registry. log_requests: Whether to log requests. start_engine_loop: Whether to start the engine loop. + client_addresses: ZMQ addresses of an externally launched engine, + when not launching one in-process. + client_count: Number of API-server clients sharing the engine. + client_index: Index of this client within ``client_count``. + aggregate_engine_logging: Whether to aggregate per-engine logs + into a single stat logger. stat_loggers: customized stat loggers for the engine. If not provided, default stat loggers will be used. PLEASE BE AWARE THAT STAT LOGGER IS NOT STABLE IN V1, AND ITS BASE CLASS INTERFACE MIGHT CHANGE. + profiler: Torch profiler wrapper used to trace the frontend. Returns: None + """ # Ensure we can serialize custom transformer configs maybe_register_config_serialize_by_value() @@ -255,7 +262,6 @@ class AsyncLLM(EngineClient): stat_loggers: list[StatLoggerFactory] | None = None, ) -> "AsyncLLM": """Create an AsyncLLM from the EngineArgs.""" - # Create the engine configs. vllm_config = engine_args.create_engine_config(usage_context) executor_class = Executor.get_class(vllm_config) @@ -319,6 +325,7 @@ class AsyncLLM(EngineClient): Raises: QueueOverflowError: If ``max_num_queued_reqs`` would be exceeded. MaxQueuedTokensError: If ``max_num_queued_tokens`` would be exceeded. + """ max_num_reqs = self.scheduler_config.max_num_queued_reqs if max_num_reqs is not None: @@ -378,7 +385,6 @@ class AsyncLLM(EngineClient): reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> RequestOutputCollector: """Add new request to the AsyncLLM.""" - if self.errored: raise EngineDeadError() @@ -654,8 +660,7 @@ class AsyncLLM(EngineClient): reasoning_ended: bool | None = None, reasoning_parser_kwargs: dict[str, Any] | None = None, ) -> AsyncGenerator[RequestOutput, None]: - """ - Main function called by the API server to kick off a request + """Main function called by the API server to kick off a request * 1) Making an AsyncStream corresponding to the Request. * 2) Processing the Input. * 3) Adding the Request to the Detokenizer. @@ -678,8 +683,8 @@ class AsyncLLM(EngineClient): >>> params = self.renderer.default_cmpl_tok_params >>> (engine_input,) = self.renderer.render_cmpl([parsed], params) >>> gen = self.generate(engine_input, sampling_params, request_id) - """ + """ q: RequestOutputCollector | None = None try: q = await self.add_request( @@ -763,7 +768,6 @@ class AsyncLLM(EngineClient): def _run_output_handler(self): """Background loop: pulls from EngineCore and pushes to AsyncStreams.""" - if self.output_handler is not None: return @@ -849,7 +853,6 @@ class AsyncLLM(EngineClient): self, request_id: str | Iterable[str], internal: bool = False ) -> None: """Abort RequestId in OutputProcessor and EngineCore.""" - request_ids = ( (request_id,) if isinstance(request_id, str) else as_list(request_id) ) @@ -893,8 +896,7 @@ class AsyncLLM(EngineClient): wait_for_inflight_requests: bool | None = None, clear_cache: bool = True, ) -> None: - """ - Pause generation to allow model weight updates. + """Pause generation to allow model weight updates. All mode handling (abort / wait / keep) and cache clearing is done in the engine. New generation/encoding requests will not be scheduled @@ -910,6 +912,7 @@ class AsyncLLM(EngineClient): wait_for_inflight_requests: DEPRECATED: use mode argument. clear_cache: Whether to clear KV cache and prefix cache after draining. Set to ``False`` to preserve cache for faster resume. + """ if wait_for_inflight_requests: warnings.warn( @@ -950,8 +953,7 @@ class AsyncLLM(EngineClient): tokenization_kwargs: dict[str, Any] | None = None, reasoning_ended: bool | None = None, ) -> AsyncGenerator[PoolingRequestOutput, None]: - """ - Main function called by the API server to kick off a request + """Main function called by the API server to kick off a request * 1) Making an AsyncStream corresponding to the Request. * 2) Processing the Input. * 3) Adding the Request to the EngineCore (separate process). @@ -963,7 +965,6 @@ class AsyncLLM(EngineClient): The caller of generate() iterates the returned AsyncGenerator, returning the RequestOutput back to the caller. """ - q: RequestOutputCollector | None = None try: q = await self.add_request( @@ -1116,9 +1117,7 @@ class AsyncLLM(EngineClient): args: tuple = (), kwargs: dict | None = None, ): - """ - Perform a collective RPC call to the given path. - """ + """Perform a collective RPC call to the given path.""" return await self.engine_core.collective_rpc_async( method, timeout, args, kwargs ) @@ -1198,7 +1197,7 @@ class AsyncLLM(EngineClient): async def handle_fault( self, fault_tolerance_request: FaultToleranceRequest ) -> FaultToleranceResult: - """send fault tolerance instruction to the engine""" + """Send fault tolerance instruction to the engine.""" return await self.engine_core.handle_fault(fault_tolerance_request) async def get_status(self): @@ -1224,11 +1223,11 @@ class AsyncLLM(EngineClient): async def init_weight_transfer_engine( self, request: WeightTransferInitRequest ) -> None: - """ - Initialize weight transfer for RL training. + """Initialize weight transfer for RL training. Args: request: Weight transfer initialization request with backend-specific info + """ await self.collective_rpc( "init_weight_transfer_engine", kwargs={"init_info": request.init_info} @@ -1243,11 +1242,11 @@ class AsyncLLM(EngineClient): await self.collective_rpc("start_draft_weight_update") async def update_weights(self, request: WeightTransferUpdateRequest) -> None: - """ - Batched weight update for RL training. + """Batched weight update for RL training. Args: request: Weight update request with backend-specific update info + """ await self.collective_rpc( "update_weights", kwargs={"update_info": request.update_info} diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index b3d94de3e7..9c84827a63 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -505,7 +505,6 @@ class EngineCore: def abort_requests(self, request_ids: list[str]): """Abort requests from the scheduler.""" - # TODO: The scheduler doesn't really need to know the # specific finish reason, TBD whether we propagate that # (i.e. client-aborted vs stop criteria met). @@ -608,7 +607,6 @@ class EngineCore: Returns tuple of outputs and a flag indicating whether the model was executed. """ - # Check for any requests remaining in the scheduler - unfinished, # or finished and not yet removed from the batch. if not self.scheduler.has_requests(): @@ -659,7 +657,6 @@ class EngineCore: batch in the job queue is finished. 3. Update the scheduler from the output. """ - batch_queue = self.batch_queue assert batch_queue is not None @@ -903,8 +900,8 @@ class EngineCore: - Level 2: Discard all GPU memory. mode: Pause mode - how to deal with any existing requests, see documentation of pause_scheduler method. - """ + """ # Pause scheduler before sleeping. clear_prefix_cache = level >= 1 pause_future = self.pause_scheduler(mode=mode, clear_cache=clear_prefix_cache) @@ -935,6 +932,7 @@ class EngineCore: Args: tags: Tags to wake up. Use ["scheduling"] for level 0 wake up. + """ if tags is not None and "scheduling" in tags: # Remove "scheduling" from tags if there are other tags to process. @@ -1162,8 +1160,7 @@ class EngineCoreProc(EngineCore): vllm_config: VllmConfig, client_handshake_address: str | None, ) -> Generator[EngineZmqAddresses, None, None]: - """ - Perform startup handshakes. + """Perform startup handshakes. For DP=1 or offline mode, this is with the colocated front-end process. @@ -1298,7 +1295,6 @@ class EngineCoreProc(EngineCore): @staticmethod def run_engine_core(*args, dp_rank: int = 0, local_dp_rank: int = 0, **kwargs): """Launch EngineCore busy loop in background process.""" - # Ensure we can serialize transformer config after spawning maybe_register_config_serialize_by_value() @@ -1444,7 +1440,6 @@ class EngineCoreProc(EngineCore): def _process_input_queue(self): """Exits when an engine step needs to be performed.""" - waited = False while not self.has_work() and self.is_running(): # Notify callbacks waiting for engine to become idle. @@ -1475,7 +1470,6 @@ class EngineCoreProc(EngineCore): def _process_engine_step(self) -> bool: """Called only when there are unfinished local requests.""" - # Step the engine core. outputs, model_executed = self.step_fn() # Put EngineCoreOutputs into the output queue. @@ -1549,7 +1543,6 @@ class EngineCoreProc(EngineCore): self, request_type: EngineCoreRequestType, request: Any ) -> None: """Dispatch request from client.""" - if request_type == EngineCoreRequestType.WAKEUP: return elif request_type == EngineCoreRequestType.ADD: @@ -1645,7 +1638,6 @@ class EngineCoreProc(EngineCore): def _send_engine_dead(self): """Send EngineDead status to the EngineCoreClient.""" - # Put ENGINE_CORE_DEAD in the queue. self.output_queue.put_nowait(EngineCoreProc.ENGINE_CORE_DEAD) @@ -1710,7 +1702,6 @@ class EngineCoreProc(EngineCore): ready_event: threading.Event, ): """Input socket IO thread.""" - # Msgpack serialization decoding with optional tensor IPC receiver. add_request_decoder = MsgpackDecoder( EngineCoreRequest, oob_tensor_provider=self.tensor_ipc_receiver @@ -1809,7 +1800,6 @@ class EngineCoreProc(EngineCore): self, output_paths: list[str], coord_output_path: str | None, engine_index: int ): """Output socket IO thread.""" - # Msgpack serialization encoding. encoder = MsgpackEncoder() # Send buffers to reuse. @@ -2199,7 +2189,6 @@ class DPEngineCoreProc(EngineCoreProc): @fault_tolerant_wrapper def run_busy_loop(self): """Core busy loop of the EngineCore for data parallel case.""" - # Loop until process is sent a SIGINT or SIGTERM while self._handle_shutdown(): # 1) Poll the input queue until there is work to do. @@ -2364,8 +2353,7 @@ class DPEngineCoreProc(EngineCoreProc): def _eep_send_engine_core_notification( self, notification_type: EEPNotificationType ): - """ - Send notifications to EngineCoreClient, which can then forward + """Send notifications to EngineCoreClient, which can then forward the notifications to other engine core processes. It is used for: 1) In scale down: removing core engines to notify EngineCoreClient so EngineCoreClient can release their ray placement groups; @@ -2415,9 +2403,7 @@ class DPEngineCoreProc(EngineCoreProc): class EngineCoreActorMixin: - """ - Ray actor for running EngineCore in a data parallel context - """ + """Ray actor for running EngineCore in a data parallel context.""" def __init__( self, @@ -2510,16 +2496,14 @@ class EngineCoreActorMixin: vllm_config: VllmConfig, client_handshake_address: str | None, ): - """ - For Ray, we don't need to actually perform handshake. + """For Ray, we don't need to actually perform handshake. All addresses information is known before the actor creation. Therefore, we simply yield these addresses. """ yield self.addresses def wait_for_init(self): - """ - Wait until the engine core is initialized. + """Wait until the engine core is initialized. This is just an empty method. When ray.get() on this method (or any other method of the actor) returns, it is guaranteed @@ -2528,9 +2512,7 @@ class EngineCoreActorMixin: pass def run(self): - """ - Run the engine core busy loop. - """ + """Run the engine core busy loop.""" try: self.run_busy_loop() # type: ignore[attr-defined] except SystemExit: diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index b63e8016c4..4ca565d56c 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -77,8 +77,7 @@ EngineIdentity = bytes class EngineCoreClient(ABC): - """ - EngineCoreClient: subclasses handle different methods for pushing + """EngineCoreClient: subclasses handle different methods for pushing and pulling from the EngineCore for asyncio / multiprocessing. Subclasses: @@ -331,8 +330,7 @@ class EngineCoreClient(ABC): class InprocClient(EngineCoreClient): - """ - InprocClient: client for in-process EngineCore. Intended + """InprocClient: client for in-process EngineCore. Intended for use in LLMEngine for V0-style add_request() and step() EngineCore setup in this process (no busy loop). @@ -469,7 +467,6 @@ class BackgroundResources: def __call__(self): """Clean up background resources.""" - logger.debug_once("[shutdown] MPClient: background resource cleanup start") self.engine_dead = True if self.engine_manager is not None: @@ -543,16 +540,15 @@ class ElasticScalingCache: class MPClient(EngineCoreClient): - """ - MPClient: base client for multi-proc EngineCore. - EngineCore runs in a background process busy loop, getting - new EngineCoreRequests and returning EngineCoreOutputs + """MPClient: base client for multi-proc EngineCore. + EngineCore runs in a background process busy loop, getting + new EngineCoreRequests and returning EngineCoreOutputs - * pushes EngineCoreRequests via input_socket - * pulls EngineCoreOutputs via output_socket + * pushes EngineCoreRequests via input_socket + * pulls EngineCoreOutputs via output_socket - * AsyncMPClient subclass for AsyncLLM usage - * SyncMPClient subclass for LLM usage + * AsyncMPClient subclass for AsyncLLM usage + * SyncMPClient subclass for LLM usage """ def __init__( @@ -1754,8 +1750,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient): self._prepared_elastic_ep = new_data_parallel_size, num_redundant_experts def _eep_wait_for_setup_switch_complete(self) -> asyncio.Future: - """ - Wait for core engines to switch to the new setup. + """Wait for core engines to switch to the new setup. In eep_process_engine_core_notification(), a dummy UtilityOutput with EEP_NOTIFICATION_CALL_ID will be set when RECONFIGURE_FINISHED diff --git a/vllm/v1/engine/detokenizer.py b/vllm/v1/engine/detokenizer.py index f04fa30a18..9658c030ef 100644 --- a/vllm/v1/engine/detokenizer.py +++ b/vllm/v1/engine/detokenizer.py @@ -94,8 +94,7 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC): self.output_text = "" def update(self, new_token_ids: list[int], stop_terminated: bool) -> str | None: - """ - Update RequestState for the request_id by: + """Update RequestState for the request_id by: 1) Detokenize the new token ids incrementally. 2) Evaluate stop criteria. @@ -149,7 +148,6 @@ class BaseIncrementalDetokenizer(IncrementalDetokenizer, ABC): def get_next_output_text(self, finished: bool, delta: bool) -> str: """If delta is True, only new text since the last call to this method is returned""" - # We return the full output text if the sequence is finished. buffer_length = 0 if finished else self.stop_buffer_length if not delta: diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 3f86f3c8b2..256a72df42 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -206,8 +206,7 @@ class InputProcessor: mm_hash: str, lora_request: LoRARequest | None, ) -> str: - """ - When enable_tower_connector_lora is True, multi-modal embeddings + """When enable_tower_connector_lora is True, multi-modal embeddings vary depending on the LoRA request. Therefore, the mm_hash must be generated based on the LoRA request to prevent incorrect cache hits. """ diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 32096079e0..c6c77f72e7 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -171,7 +171,6 @@ class LLMEngine: enable_multiprocessing: bool = False, ) -> "LLMEngine": """Creates an LLM engine from the engine arguments.""" - # Create the engine configs. vllm_config = engine_args.create_engine_config(usage_context) executor_class = Executor.get_class(vllm_config) @@ -216,7 +215,6 @@ class LLMEngine: def abort_request(self, request_ids: list[str], internal: bool = False) -> None: """Remove request_ids from EngineCore and Detokenizer.""" - request_ids = self.output_processor.abort_requests(request_ids, internal) self.engine_core.abort_requests(request_ids) diff --git a/vllm/v1/engine/logprobs.py b/vllm/v1/engine/logprobs.py index 8eef567b07..ff483e128e 100644 --- a/vllm/v1/engine/logprobs.py +++ b/vllm/v1/engine/logprobs.py @@ -76,7 +76,6 @@ class LogprobsProcessor: logprobs_lists: the lists of logprob tokens, logprobs, and ranks. """ - assert self.num_logprobs is not None assert self.logprobs is not None assert self.cumulative_logprob is not None @@ -129,7 +128,6 @@ class LogprobsProcessor: tensors. """ - # Prompt logprobs are enabled. assert self.num_prompt_logprobs is not None assert self.prompt_logprobs is not None @@ -187,7 +185,7 @@ class LogprobsProcessor: ) def pop_prompt_logprobs(self) -> PromptLogprobs | None: - """Pop and return all request prompt logprobs + """Pop and return all request prompt logprobs. The logprobs processor aggregates prompt chunk logprobs over one or more prefill chunks. This method returns @@ -199,6 +197,7 @@ class LogprobsProcessor: Returns: None if prompt logprobs are disabled for this request. List of all prompt logprobs, otherwise. + """ plp = self.prompt_logprobs if plp: @@ -223,6 +222,7 @@ class LogprobsProcessor: Returns: List of sampled token IDs, oldest first, most recent last. + """ if not logprobs_source: return [] @@ -265,6 +265,7 @@ class LogprobsProcessor: Returns: The corrected decoded string, or empty string if the byte sequence is genuinely incomplete at this point. + """ assert self.tokenizer is not None @@ -325,6 +326,7 @@ class LogprobsProcessor: context_token_ids: Preceding sampled token IDs providing sequential context. If None, extracted from self.logprobs. + """ if context_token_ids is None: context_token_ids = self._get_sampled_context_ids(self.logprobs) diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 257287243d..b4f7338048 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -49,8 +49,7 @@ EMPTY_CPU_TENSOR = torch.empty(0, device="cpu") class RequestOutputCollector: - """ - Collects streamed RequestOutputs per individual request, + """Collects streamed RequestOutputs per individual request, for hand-off to the consuming asyncio generate task. When streaming deltas, RequestOutputs are merged if the @@ -155,6 +154,7 @@ class RequestState: n: int | None = None, temperature: float | None = None, stream_input: bool = False, + remote_prefill_cached_tokens: int | None = None, ): self.request_id = request_id self.external_req_id = external_req_id @@ -179,6 +179,7 @@ class RequestState: self.queue = queue self.num_cached_tokens = 0 self.num_cache_creation_tokens = 0 + self.remote_prefill_cached_tokens = remote_prefill_cached_tokens # Per-sequence spec-decode accumulator; arrives once (on finish) via # EngineCoreOutput, then attached to this sequence's CompletionOutput. self.spec_decode_metrics: RequestSpecDecodeMetrics | None = None @@ -230,7 +231,18 @@ class RequestState: log_stats: bool, stream_interval: int, ) -> "RequestState": + remote_prefill_cached_tokens = None if sampling_params := request.sampling_params: + # In a remote prefill scenario, report cached tokens + # as cache hit rate on the remote P worker. + if sampling_params.extra_args: + kv_transfer_params = sampling_params.extra_args.get( + "kv_transfer_params" + ) + if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"): + cached = kv_transfer_params.get("remote_prefill_cached_tokens") + if isinstance(cached, int): + remote_prefill_cached_tokens = cached if not sampling_params.detokenize: tokenizer = None output_kind = sampling_params.output_kind @@ -281,6 +293,7 @@ class RequestState: log_stats=log_stats, stream_interval=stream_interval, stream_input=request.resumable, + remote_prefill_cached_tokens=remote_prefill_cached_tokens, ) def make_request_output( @@ -492,7 +505,6 @@ class OutputProcessor: def propagate_error(self, e: Exception): """Propagate error to all generate() tasks.""" - for _, state in self.request_states.items(): assert state.queue is not None state.queue.put(e) @@ -632,8 +644,7 @@ class OutputProcessor: engine_core_timestamp: float | None = None, iteration_stats: IterationStats | None = None, ) -> OutputProcessorOutput: - """ - Process the EngineCoreOutputs: + """Process the EngineCoreOutputs: 1) Compute stats for logging 2) Detokenize 3) Create and handle RequestOutput objects: @@ -653,7 +664,6 @@ class OutputProcessor: If you need to touch every element of the batch, do it from within the loop below. """ - request_outputs: list[RequestOutput | PoolingRequestOutput] = [] reqs_to_abort: list[str] = [] for engine_core_output in engine_core_outputs: @@ -687,6 +697,8 @@ class OutputProcessor: req_state.num_cache_creation_tokens = ( engine_core_output.prefill_stats.num_cache_creation_tokens ) + if req_state.remote_prefill_cached_tokens is not None: + req_state.num_cached_tokens = req_state.remote_prefill_cached_tokens req_state.is_prefilling = False if engine_core_output.spec_decode_metrics is not None: diff --git a/vllm/v1/engine/parallel_sampling.py b/vllm/v1/engine/parallel_sampling.py index 8eb6fa057d..9baf785903 100644 --- a/vllm/v1/engine/parallel_sampling.py +++ b/vllm/v1/engine/parallel_sampling.py @@ -53,7 +53,7 @@ class ParentRequest: self, index: int, ) -> SamplingParams: - """Efficiently obtain child `sampling_params` + """Efficiently obtain child `sampling_params`. If `sampling_params.seed` is not `None` then each child request requires a unique clone of @@ -64,6 +64,7 @@ class ParentRequest: Returns: Child `sampling_params` instance. + """ seed = self.sampling_params.seed if self.cached_child_sampling_params: @@ -88,6 +89,7 @@ class ParentRequest: Returns: (request ID, sampling_params) tuple + """ child_req_id = f"{index}_{self.request_id}" self.child_requests.add(child_req_id) diff --git a/vllm/v1/engine/tensor_ipc.py b/vllm/v1/engine/tensor_ipc.py index 2a8391b17e..2b8fbadd97 100644 --- a/vllm/v1/engine/tensor_ipc.py +++ b/vllm/v1/engine/tensor_ipc.py @@ -29,8 +29,7 @@ TensorIpcQueue = MPQueue @dataclasses.dataclass class TensorIpcData: - """ - Data sent via torch.multiprocessing.Queue for zero-copy IPC. + """Data sent via torch.multiprocessing.Queue for zero-copy IPC. Contains the tensor_id and the actual tensor. The tensor is shared in memory (GPU or CPU) for efficient inter-process communication. @@ -130,7 +129,6 @@ class TensorIpcReceiver: the queue, buffering them, until the requested tensor is found. Works for CUDA and CPU. """ - # Create lookup key from handle sender_id: str = meta["sender_id"] message_id: int = meta["message_id"] diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index 9bce2b7687..c1002ba1dc 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -142,8 +142,7 @@ def _node_ip_from_resources(node_resources: dict) -> str | None: class CoreEngineProcManager: - """ - Utility class to handle creation, readiness, and shutdown + """Utility class to handle creation, readiness, and shutdown of background processes used by the AsyncLLM and LLMEngine. """ @@ -255,7 +254,6 @@ class CoreEngineProcManager: def monitor_engine_liveness(self) -> None: """Monitor engine core process liveness.""" - sentinel_to_proc = {proc.sentinel: proc for proc in self.processes} sentinels = set(sentinel_to_proc.keys()) @@ -316,8 +314,7 @@ def set_assigned_physical_gpu_ids_for_dp_rank( local_dp_rank: int, user_assigned_gpu_ids: list[int] | None = None, ) -> None: - """ - Populate assigned_physical_gpu_ids on the config for the given DP rank. + """Populate assigned_physical_gpu_ids on the config for the given DP rank. user_assigned_gpu_ids is the full (un-sharded) --device-ids list, if the user provided one; this DP rank's shard is sliced from it. It is passed @@ -345,8 +342,7 @@ def get_physical_gpu_ids_for_local_dp_rank( local_world_size: int | None = None, user_assigned_gpu_ids: list[int] | None = None, ) -> list[int]: - """ - Returns list of physical GPU IDs for the specified + """Returns list of physical GPU IDs for the specified data parallel rank. For example, if world_size=2 and local_dp_rank=1, and there are 4 devices, @@ -399,8 +395,7 @@ def _apply_dp_identity_suffix(dp_vllm_config, dp_rank: int) -> None: class CoreEngineActorManager: - """ - Utility class to handle creation, readiness, and shutdown + """Utility class to handle creation, readiness, and shutdown of core engine Ray actors used by the AsyncLLM and LLMEngine. Different from CoreEngineProcManager, this class manages @@ -552,10 +547,7 @@ class CoreEngineActorManager: def create_dp_placement_groups( vllm_config: VllmConfig, ) -> tuple[list["PlacementGroup"], list[int]]: - """ - Create placement groups for data parallel. - """ - + """Create placement groups for data parallel.""" import ray from ray._private.state import available_resources_per_node @@ -769,9 +761,7 @@ class CoreEngineActorManager: def add_dp_placement_groups( old_vllm_config: VllmConfig, new_data_parallel_size: int ) -> tuple[list["PlacementGroup"], list[int]]: - """ - Add placement groups for new data parallel size. - """ + """Add placement groups for new data parallel size.""" import ray from ray._private.state import ( available_resources_per_node, @@ -1108,7 +1098,6 @@ def launch_core_engines( addresses: EngineZmqAddresses, ) -> Iterator[CoreEngineLaunch]: """Launch engine and DP coordinator processes as needed.""" - parallel_config = vllm_config.parallel_config dp_size = parallel_config.data_parallel_size local_engine_count = parallel_config.data_parallel_size_local diff --git a/vllm/v1/executor/abstract.py b/vllm/v1/executor/abstract.py index 964f16ed24..6b07bad17c 100644 --- a/vllm/v1/executor/abstract.py +++ b/vllm/v1/executor/abstract.py @@ -36,7 +36,7 @@ FailureCallback = Callable[[], None] class Executor(ABC): - """Abstract base class for vLLM executors." + """Abstract base class for vLLM executors. An executor is responsible for executing the model on one device, or it can be a distributed executor that can execute the model on multiple devices. @@ -139,8 +139,7 @@ class Executor(ABC): ) def register_failure_callback(self, callback: FailureCallback): # noqa: B027 - """ - Register a function to be called if the executor enters a permanent + """Register a function to be called if the executor enters a permanent failed state. """ pass @@ -168,8 +167,29 @@ class Executor(ABC): kwargs: dict | None = None, non_block: Literal[False] = False, ) -> list[_R]: - """ - Execute an RPC call on all workers. + pass + + @overload + def collective_rpc( + self, + method: str | Callable[[WorkerBase], _R], + timeout: float | None = None, + args: tuple = (), + kwargs: dict | None = None, + non_block: Literal[True] = True, + ) -> Future[list[_R]]: + pass + + @abstractmethod + def collective_rpc( + self, + method: str | Callable[[WorkerBase], _R], + timeout: float | None = None, + args: tuple = (), + kwargs: dict | None = None, + non_block: bool = False, + ) -> list[_R] | Future[list[_R]]: + """Execute an RPC call on all workers. Args: method: Name of the worker method to execute, or a callable that @@ -191,24 +211,8 @@ class Executor(ABC): Note: It is recommended to use this API to only pass control messages, and set up data-plane communication to pass data. + """ - pass - - @overload - def collective_rpc( - self, - method: str | Callable[[WorkerBase], _R], - timeout: float | None = None, - args: tuple = (), - kwargs: dict | None = None, - non_block: Literal[True] = True, - ) -> Future[list[_R]]: - pass - - @abstractmethod - def collective_rpc( - self, method, timeout=None, args=(), kwargs=None, non_block: bool = False - ): raise NotImplementedError def get_kv_connector_handshake_metadata( @@ -288,7 +292,7 @@ class Executor(ABC): self.collective_rpc("shutdown") def init_kv_output_aggregator(self, connector: "KVConnectorBase") -> None: - """Init KVOutputAggregator""" + """Init KVOutputAggregator.""" self.kv_output_aggregator = KVOutputAggregator.from_connector( connector, self.parallel_config.world_size ) @@ -381,9 +385,7 @@ class Executor(ABC): @classmethod def supports_async_scheduling(cls) -> bool: - """ - Whether the executor supports async scheduling. - """ + """Whether the executor supports async scheduling.""" return False diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 67a631d3ce..dbaefe2186 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -500,7 +500,7 @@ class MultiprocExecutor(Executor): p.kill() def shutdown(self): - """Properly shut down the executor and its workers""" + """Properly shut down the executor and its workers.""" if not getattr(self, "shutting_down", False): worker_count = len(getattr(self, "workers", None) or []) logger.debug( @@ -853,7 +853,6 @@ class WorkerProc: def worker_main(*args, **kwargs): """Worker initialization and execution loops. This runs a background process""" - # Signal handler used for graceful termination. # SystemExit exception is only raised once to allow this and worker # processes to terminate without error @@ -1010,7 +1009,6 @@ class WorkerProc: def async_output_busy_loop(self): """Entrypoint for the thread which handles outputs asynchronously.""" - # set device to the worker device for the thread. # a thread will not inherit the context of the main thread. # when calling any cuda runtime functions, it will implicitly @@ -1027,7 +1025,7 @@ class WorkerProc: self.enqueue_output(output) def worker_busy_loop(self): - """Main busy loop for Multiprocessing Workers""" + """Main busy loop for Multiprocessing Workers.""" assert self.rpc_broadcast_mq is not None while True: self._execute_worker_rpc(self.rpc_broadcast_mq.dequeue(indefinite=True)) @@ -1099,7 +1097,6 @@ def set_multiprocessing_worker_envs(local_world_size: int = 1): """Set up environment variables that should be used when there are workers in a multiprocessing environment. This should be called by the parent process before worker processes are created""" - _maybe_force_spawn() if current_platform.is_cpu() or "OMP_NUM_THREADS" in os.environ: diff --git a/vllm/v1/executor/ray_executor.py b/vllm/v1/executor/ray_executor.py index b2c385a862..d101885ce2 100644 --- a/vllm/v1/executor/ray_executor.py +++ b/vllm/v1/executor/ray_executor.py @@ -52,8 +52,7 @@ COMPLETED_NONE_FUTURE.set_result(None) @dataclass class RayWorkerMetaData: - """ - Metadata for a Ray worker. + """Metadata for a Ray worker. The order of ray worker creation can be random, and we need to reset the rank after creating all workers. """ @@ -65,7 +64,7 @@ class RayWorkerMetaData: class RayDistributedExecutor(Executor): - """Ray-based distributed executor""" + """Ray-based distributed executor.""" uses_ray: bool = True supports_pp: bool = True @@ -249,8 +248,7 @@ class RayDistributedExecutor(Executor): ip_counts[ip] = ip_counts.get(ip, 0) + 1 def sort_by_driver_then_worker_ip(item: RayWorkerMetaData): - """ - Sort the workers based on 3 properties: + """Sort the workers based on 3 properties: 1. If the worker is on the same node as the driver (vllm engine), it should be placed first. 2. Then, if the worker is on a node with fewer workers, it should @@ -439,6 +437,7 @@ class RayDistributedExecutor(Executor): Returns: The model runner output. + """ scheduler_output = self.scheduler_output if scheduler_output is None: diff --git a/vllm/v1/executor/ray_executor_v2.py b/vllm/v1/executor/ray_executor_v2.py index 8f4b84ed0f..b8a2eec73f 100644 --- a/vllm/v1/executor/ray_executor_v2.py +++ b/vllm/v1/executor/ray_executor_v2.py @@ -72,7 +72,7 @@ class RayWorkerHandle: """run() ObjectRef used as a sentinel for health monitoring""" def run(self): - """Start the worker's busy loop""" + """Start the worker's busy loop.""" self.run_ref = self.actor.run.remote() @@ -190,8 +190,7 @@ class RayWorkerProc(WorkerProc): def _init_message_queues( self, input_shm_handle: Handle, vllm_config: VllmConfig ) -> None: - """ - Workers on the same node as the executor use shared memory for + """Workers on the same node as the executor use shared memory for both the broadcast (input) MQ and the response MQ. Workers on different nodes use TCP (n_local_reader=0). """ diff --git a/vllm/v1/executor/ray_utils.py b/vllm/v1/executor/ray_utils.py index 71d4cf7471..e609fb6e7e 100644 --- a/vllm/v1/executor/ray_utils.py +++ b/vllm/v1/executor/ray_utils.py @@ -68,8 +68,7 @@ try: rpc_rank: int def adjust_rank(self, rank_mapping: dict[int, int]) -> None: - """ - Adjust the rpc_rank based on the given mapping. + """Adjust the rpc_rank based on the given mapping. It is only used during the initialization of the executor, to adjust the rpc_rank of workers after we create all workers. """ @@ -363,8 +362,7 @@ def get_bundles_for_indices( bundle_indices: list[int], world_size: int, ) -> list[tuple[int, str, str]]: - """ - Return GPU bundle indices paired with node IDs and node IPs for + """Return GPU bundle indices paired with node IDs and node IPs for explicit bundle indices specified via VLLM_RAY_BUNDLE_INDICES. """ assert len(bundle_indices) == world_size, ( @@ -391,8 +389,7 @@ def get_bundles_for_indices( def get_bundles_sorted_by_node( placement_group: "PlacementGroup", ) -> list[tuple[int, str, str]]: - """ - Return GPU bundle indices paired with node IDs and node IPs, + """Return GPU bundle indices paired with node IDs and node IPs, sorted driver-first. This utility has to be invoked from the driver node. @@ -533,6 +530,7 @@ def initialize_ray_cluster( on the current (driver) node and pin the first PG bundle to it. Set to False for executors like RayExecutorV2 where all GPU work is delegated to remote Ray actors. + """ assert_ray_available() from vllm.platforms import current_platform diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 7e91300b2f..17771d9ae6 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -154,9 +154,7 @@ class SparseCacheRole(str, Enum): @dataclass(frozen=True) class KVCacheSpec: - """ - A base class for specifying the KV cache format of one layer. - """ + """A base class for specifying the KV cache format of one layer.""" # number of tokens in a block block_size: int @@ -180,11 +178,11 @@ class KVCacheSpec: @property def page_size_bytes(self) -> int: - """ - The size of a page with `block_size` tokens in bytes. + """The size of a page with `block_size` tokens in bytes. Returns: The page size + """ raise NotImplementedError @@ -202,37 +200,33 @@ class KVCacheSpec: return 128 def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - """ - The maximum possible memory usage of this KV cache in bytes. + """The maximum possible memory usage of this KV cache in bytes. Returns: The KV cache size in bytes + """ raise NotImplementedError def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int: - """ - The number of block table entries needed per request, i.e. the row + """The number of block table entries needed per request, i.e. the row length of the worker-side block table for this cache group. Args: vllm_config: The vllm config. max_len: The maximum sequence length to size for, including the encoder length for encoder-decoder models. + """ return cdiv(max_len, self.block_size) def copy_with_new_block_size(self, block_size: int) -> Self: - """ - Create a new KVCacheSpec from self but replacing the block size. - """ + """Create a new KVCacheSpec from self but replacing the block size.""" return replace(self, block_size=block_size) @classmethod def merge(cls, specs: list[Self]) -> Self: - """ - Merge a list of KVCacheSpec objects into a single KVCacheSpec object. - """ + """Merge a list of KVCacheSpec objects into a single KVCacheSpec object.""" if not all(spec == specs[0] for spec in specs[1:]): raise AssertionError( "All layers in the same KV cache group must be the same." @@ -242,9 +236,7 @@ class KVCacheSpec: def is_uniform_with_collection( self, kv_cache_specs: dict[str, KVCacheSpec] ) -> bool: - """ - Whether this KVCacheSpec is uniform with all specs of all layers. - """ + """Whether this KVCacheSpec is uniform with all specs of all layers.""" uniform_type_base_spec = KVCacheSpecRegistry.get_uniform_type_base_spec(self) assert uniform_type_base_spec is not None, ( f"Unsupported KV cache spec type: {type(self)}. " @@ -537,8 +529,7 @@ class AttentionSpec(KVCacheSpec): @dataclass(frozen=True, kw_only=True) class FullAttentionSpec(AttentionSpec): - """ - When hybrid allocator is disabled and the model contains both full + """When hybrid allocator is disabled and the model contains both full attention layers and sliding window attention layers, sliding window attention are regarded as full attention in KV cache manager (blocks are allocated for all tokens), while computed as sliding window @@ -582,8 +573,7 @@ class FullAttentionSpec(AttentionSpec): @classmethod def merge(cls, specs: list[Self]) -> Self: - """ - Merge a list of FullAttentionSpec objects into a single + """Merge a list of FullAttentionSpec objects into a single FullAttentionSpec object. """ assert all(isinstance(spec, FullAttentionSpec) for spec in specs), ( @@ -1124,9 +1114,7 @@ class EncoderOnlyAttentionSpec(AttentionSpec): @dataclass(frozen=True) class CrossAttentionSpec(AttentionSpec): - """ - KV cache spec for cross-attention layers in encoder-decoder models. - """ + """KV cache spec for cross-attention layers in encoder-decoder models.""" def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: # For cross-attention, we need to cache encoder states @@ -1141,8 +1129,7 @@ class SinkFullAttentionSpec(FullAttentionSpec): @classmethod def merge(cls, specs: list[Self]) -> Self: - """ - Merge a list of FullAttentionSpec objects into a single + """Merge a list of FullAttentionSpec objects into a single FullAttentionSpec object. """ assert all(isinstance(spec, FullAttentionSpec) for spec in specs), ( @@ -1192,8 +1179,7 @@ class SinkFullAttentionSpec(FullAttentionSpec): @dataclass(frozen=True) class UniformTypeKVCacheSpecs(KVCacheSpec): - """ - A KV cache spec for multiple layers with the same type of attention. Here, + """A KV cache spec for multiple layers with the same type of attention. Here, same types means always need the same number of token slots. For example, sliding window attentions with different window sizes are not the same type and should not be merged into one UniformTypeKVCacheSpecs. @@ -1241,8 +1227,7 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): @classmethod def is_uniform_type(cls, kv_cache_specs: dict[str, KVCacheSpec]) -> bool: - """ - Whether all layers have the same type of KV cache spec. + """Whether all layers have the same type of KV cache spec. Uses the registry to determine grouping base classes, so custom specs that inherit from FullAttentionSpec are treated as full attention. @@ -1256,8 +1241,7 @@ class UniformTypeKVCacheSpecs(KVCacheSpec): @classmethod def from_specs(cls, kv_cache_specs: dict[str, KVCacheSpec]) -> Self | None: - """ - Return a SameTypeKVCacheSpecs object if all layers have the same type + """Return a SameTypeKVCacheSpecs object if all layers have the same type of KV cache spec. Return None if not. """ if cls.is_uniform_type(kv_cache_specs): @@ -1368,8 +1352,7 @@ def get_kv_cache_spec_sliding_window(kv_cache_spec: KVCacheSpec) -> int | None: @dataclass class KVCacheTensor: - """ - A class for specifying how the workers should initialize the KV cache. + """A class for specifying how the workers should initialize the KV cache. Placement of a set of same-shaped layers in the KV cache allocation. Layer ``layers[l]``'s page for block ``b`` starts at @@ -1399,8 +1382,7 @@ class KVCacheGroupRole(str, Enum): @dataclass class KVCacheGroupSpec: - """ - Represents a group of model layers that share the same KV cache block table. + """Represents a group of model layers that share the same KV cache block table. These layers are regarded as one layer in the KV cache manager. """ @@ -1421,9 +1403,7 @@ class KVCacheGroupSpec: @dataclass class KVCacheConfig: - """ - The KV cache configuration of a model. - """ + """The KV cache configuration of a model.""" num_blocks: int """The number of KV cache blocks""" diff --git a/vllm/v1/kv_cache_spec_registry.py b/vllm/v1/kv_cache_spec_registry.py index 62cb291892..3a03fb376c 100644 --- a/vllm/v1/kv_cache_spec_registry.py +++ b/vllm/v1/kv_cache_spec_registry.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Registry for KVCacheSpec types and their associated managers. +"""Registry for KVCacheSpec types and their associated managers. This module provides a pluggable architecture for registering custom KVCacheSpec subclasses without modifying vLLM core code. Out-of-tree platforms can define @@ -42,9 +41,7 @@ class KVCacheSpecRegistry: @classmethod def _ensure_registered(cls, vllm_config=None) -> None: - """ - Run full KVCacheSpec registration if the registration is not done. - """ + """Run full KVCacheSpec registration if the registration is not done.""" if _REGISTRY_KVCACHESPEC_LIST: return @@ -67,8 +64,7 @@ class KVCacheSpecRegistry: manager_class: type["SingleTypeKVCacheManager"] | None = None, uniform_type_base_spec: type["KVCacheSpec"] | None = None, ) -> None: - """ - Register a KVCacheSpec class with its manager and base spec. + """Register a KVCacheSpec class with its manager and base spec. Args: kvcache_spec_cls: The KVCacheSpec subclass to register @@ -77,6 +73,7 @@ class KVCacheSpecRegistry: instead of being grouped to different kvcache group, `kvcache_spec_cls` and `uniform_type_base_spec` will be trated as uniform type. If None, defaults to kvcache_spec_cls itself (for built-in base specs). + """ assert manager_class is not None, "manager_class is required" if uniform_type_base_spec is None: @@ -118,8 +115,7 @@ class KVCacheSpecRegistry: def get_manager_class( cls, kvcache_spec: "KVCacheSpec", role: str | None = None ) -> type["SingleTypeKVCacheManager"] | None: - """ - Get the single type kvcache manager class for a given kvcache spec instance. + """Get the single type kvcache manager class for a given kvcache spec instance. Args: kvcache_spec: A KVCacheSpec instance @@ -127,6 +123,7 @@ class KVCacheSpecRegistry: Returns: The SingleTypeKVCacheManager class to use for this kvcache_spec + """ cls._ensure_registered() if role is not None and (manager_class := _REGISTRY_ROLE_MANAGERS.get(role)): @@ -144,8 +141,7 @@ class KVCacheSpecRegistry: def get_uniform_type_base_spec( cls, kvcache_spec: "KVCacheSpec" ) -> type["KVCacheSpec"] | None: - """ - Get the base kvcache spec class for grouping compatibility checks. + """Get the base kvcache spec class for grouping compatibility checks. KVCacheSpecs with uniform_type_base_spec will be trated as one group. Args: @@ -153,6 +149,7 @@ class KVCacheSpecRegistry: Returns: The base KVCacheSpec class for checking uniform type kvcache specs + """ cls._ensure_registered() kvcache_spec_cls = type(kvcache_spec) @@ -168,9 +165,7 @@ class KVCacheSpecRegistry: def check_kv_cache_spec_registry( cls, kv_cache_spec: dict[str, "KVCacheSpec"] ) -> None: - """ - Check if the KVCacheSpecs of each layer are registered as expected. - """ + """Check if the KVCacheSpecs of each layer are registered as expected.""" cls._ensure_registered() for layer_name, spec in kv_cache_spec.items(): # use raise instead of assert to make it effective in production environment @@ -192,8 +187,7 @@ def register_kv_cache_spec( manager_class: type["SingleTypeKVCacheManager"] | None = None, uniform_type_base_spec: type["KVCacheSpec"] | None = None, ): - """ - Decorator to register a custom KVCacheSpec class. + """Decorator to register a custom KVCacheSpec class. Args: manager_class: The SingleTypeKVCacheManager to use for this spec. @@ -211,6 +205,7 @@ def register_kv_cache_spec( @dataclass(frozen=True, kw_only=True) class CustomFullAttentionSpec(FullAttentionSpec): pass + """ def decorator(kvcache_spec_cls: type["KVCacheSpec"]) -> type["KVCacheSpec"]: diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 988747e06e..1144477628 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Core abstractions for KV cache offloading in vLLM v1. -""" +"""Core abstractions for KV cache offloading in vLLM v1.""" from abc import ABC, abstractmethod from collections.abc import Collection, Iterable, Sequence @@ -149,8 +147,7 @@ class ScheduleEndContext(NamedTuple): class LoadStoreSpec: - """ - Metadata that encapsulates information allowing a worker + """Metadata that encapsulates information allowing a worker to load, and optionally also to store, blocks of KV data. """ @@ -235,8 +232,7 @@ class OffloadingKVEventsConfig: class OffloadingManager(ABC): @abstractmethod def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: - """ - Checks whether a single block is offloaded and ready to be read. + """Checks whether a single block is offloaded and ready to be read. Args: key: the key identifying the block to lookup. @@ -246,6 +242,7 @@ class OffloadingManager(ABC): HIT if the block is offloaded and ready, MISS if not found, HIT_PENDING if found but not yet readable, or RETRY if the lookup should be retried later. + """ pass @@ -255,8 +252,7 @@ class OffloadingManager(ABC): keys: Collection[OffloadKey], req_context: ReqContext, ) -> LoadStoreSpec: - """ - Prepare the given blocks to be read. + """Prepare the given blocks to be read. The given blocks will be protected from eviction until complete_load is called. It assumes all given blocks are offloaded. @@ -268,27 +264,28 @@ class OffloadingManager(ABC): Returns: A LoadStoreSpec that can be used by a worker to locate and load the actual offloaded KV data. + """ pass def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): - """ - Mark the given blocks as recently used. + """Mark the given blocks as recently used. This could in practice mean moving them to the end of an LRU list. Args: keys: the keys identifying the blocks. req_context: per-request context (e.g. kv_transfer_params). + """ return def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext): - """ - Marks previous blocks that were prepared to load as done loading. + """Marks previous blocks that were prepared to load as done loading. Args: keys: the keys identifying the blocks. req_context: per-request context (e.g. kv_transfer_params). + """ return @@ -298,8 +295,7 @@ class OffloadingManager(ABC): keys: Collection[OffloadKey], req_context: ReqContext, ) -> PrepareStoreOutput | None: - """ - Prepare the given blocks to be offloaded. + """Prepare the given blocks to be offloaded. The given blocks will be protected from eviction until complete_store is called. @@ -312,6 +308,7 @@ class OffloadingManager(ABC): where to store them (LoadStoreSpec), and list of blocks that were evicted as a result. None is returned if the blocks cannot be stored. + """ pass @@ -321,8 +318,7 @@ class OffloadingManager(ABC): req_context: ReqContext, success: bool = True, ): - """ - Marks blocks which were previously prepared to be stored, as stored. + """Marks blocks which were previously prepared to be stored, as stored. Following this call, the blocks become loadable. If success is False, blocks that were not marked as stored will be removed. @@ -331,25 +327,25 @@ class OffloadingManager(ABC): keys: the keys identifying the blocks. req_context: per-request context (e.g. kv_transfer_params). success: whether the blocks were stored successfully. + """ return @abstractmethod def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: - """ - Called when a new request is first seen by the scheduler. + """Called when a new request is first seen by the scheduler. Returns a RequestOffloadingContext indicating how this request's blocks should be offloaded. Args: req_context: per-request context. + """ pass def on_request_finished(self, req_context: ReqContext) -> None: - """ - Called when a request has finished. + """Called when a request has finished. By the time this is called, the scheduler will issue no more submit-side calls for this request, such as prepare_store() and @@ -364,12 +360,12 @@ class OffloadingManager(ABC): Args: req_context: per-request context. + """ return def take_events(self) -> Iterable[OffloadingEvent]: - """ - Take the offloading events from the manager. + """Take the offloading events from the manager. A tier manager emits only events for storage state it owns. A composing manager may aggregate child event streams, but should not @@ -377,6 +373,7 @@ class OffloadingManager(ABC): Yields: New OffloadingEvents collected since the last call. + """ return () @@ -410,8 +407,7 @@ class OffloadingManager(ABC): class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC): - """ - Spec for loading/storing KV blocks from given block numbers. + """Spec for loading/storing KV blocks from given block numbers. Subclass semantics differ: GPULoadStoreSpec.block_ids are GPU block indices; CPULoadStoreSpec.block_ids are CPU cache chunk indices. @@ -425,8 +421,7 @@ class BlockIDsLoadStoreSpec(LoadStoreSpec, ABC): class GPULoadStoreSpec(BlockIDsLoadStoreSpec): - """ - Spec for loading/storing a KV block to GPU memory. + """Spec for loading/storing a KV block to GPU memory. If there are multiple KV groups, the blocks are expected to be ordered by the group index. @@ -460,8 +455,7 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec): @dataclass class CanonicalKVCacheTensor: - """ - A canonicalized KV cache tensor whose first dimension is num_blocks. + """A canonicalized KV cache tensor whose first dimension is num_blocks. With standardized layouts (RFC #42082) num_blocks is always the leading logical dimension. @@ -517,8 +511,7 @@ class CanonicalPageMapping: @dataclass class CanonicalKVCacheRef: - """ - Per-layer (or group of layers) reference to a specific (by index) + """Per-layer (or group of layers) reference to a specific (by index) CanonicalKVCacheTensor and records the un-padded page size used by that layer. """ @@ -532,8 +525,7 @@ class CanonicalKVCacheRef: @dataclass class CanonicalKVCaches: - """ - Canonicalized block-level representation of the KV caches. + """Canonicalized block-level representation of the KV caches. Composed of: - Unique list of KV cache data tensors, @@ -620,8 +612,7 @@ class OffloadingSpec(ABC): @abstractmethod def get_manager(self) -> OffloadingManager: - """ - Get an OffloadingManager that will be used + """Get an OffloadingManager that will be used by the scheduler-side offloading connector to track offloaded blocks and manage evictions. """ @@ -629,13 +620,13 @@ class OffloadingSpec(ABC): @abstractmethod def get_worker(self, kv_caches: CanonicalKVCaches) -> OffloadingWorker: - """ - Get an OffloadingWorker that handles async KV transfers for this spec. + """Get an OffloadingWorker that handles async KV transfers for this spec. Args: kv_caches: Canonicalized KV caches. Returns: An OffloadingWorker instance for this medium. + """ pass diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 8db9e805ff..037e3631e4 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -80,8 +80,7 @@ def compute_sub_block_ptrs( tensor: torch.Tensor, skip_count: int = 0, ): - """ - Compute byte pointers for sub-blocks of the given block IDs. + """Compute byte pointers for sub-blocks of the given block IDs. Each block in block_ids contains blocks_per_chunk sub-blocks. The pointer for sub-block j of block b is: @@ -98,6 +97,7 @@ def compute_sub_block_ptrs( output: pre-allocated pointer array to write pointers into. tensor: the source or destination tensor. skip_count: sub-blocks to skip in the first block. + """ assert skip_count < blocks_per_chunk @@ -234,8 +234,7 @@ def _new_descriptor_buffers( class SingleDirectionOffloadingHandler: - """ - Handles transfers for a single direction, either CPU->GPU or GPU->CPU. + """Handles transfers for a single direction, either CPU->GPU or GPU->CPU. Transfers are guaranteed to be executed in order of their submission. Each transfer uses a unique CUDA stream, and its stream will start executing only after the streams of previous transfers have finished. @@ -250,8 +249,7 @@ class SingleDirectionOffloadingHandler: gpu_to_cpu: bool, canonical_layout: bool = False, ): - """ - Initialize a SingleDirectionOffloadingHandler. + """Initialize a SingleDirectionOffloadingHandler. Args: gpu_tensors: list of GPU KV cache tensors. @@ -259,10 +257,12 @@ class SingleDirectionOffloadingHandler: cpu_tensors: list of CPU KV cache tensors. Each of shape (num_cpu_chunks, cpu_page_size_bytes) with dtype int8. Order should match gpu_tensors. + blocks_per_chunk: number of blocks transferred per chunk. layer_refs_per_group: list of CanonicalKVCacheRef per group. gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU. canonical_layout: if True, CPU pages use the canonical layout described by the refs' mappings. + """ assert len(gpu_tensors) == len(cpu_tensors) assert len(gpu_tensors) > 0 diff --git a/vllm/v1/kv_offload/cpu/manager.py b/vllm/v1/kv_offload/cpu/manager.py index 9d85090f2d..e42cb47f25 100644 --- a/vllm/v1/kv_offload/cpu/manager.py +++ b/vllm/v1/kv_offload/cpu/manager.py @@ -44,8 +44,7 @@ class _RequestCacheAccess: class CPUOffloadingManager(OffloadingManager): - """ - An OffloadingManager with a pluggable CachePolicy, resolved by name via + """An OffloadingManager with a pluggable CachePolicy, resolved by name via CachePolicyFactory (built in: "lru", "arc"; external policies can either register their own or be loaded out-of-tree via cache_policy_module_path). diff --git a/vllm/v1/kv_offload/cpu/policies/arc.py b/vllm/v1/kv_offload/cpu/policies/arc.py index 882ee02219..c3942e90d8 100644 --- a/vllm/v1/kv_offload/cpu/policies/arc.py +++ b/vllm/v1/kv_offload/cpu/policies/arc.py @@ -14,8 +14,7 @@ from vllm.v1.kv_offload.cpu.policies.base import ( class ARCCachePolicy(CachePolicy): - """ - ARC (Adaptive Replacement Cache) cache policy. + """ARC (Adaptive Replacement Cache) cache policy. Data Structures: T1: Recent cache containing chunks accessed once. diff --git a/vllm/v1/kv_offload/cpu/policies/base.py b/vllm/v1/kv_offload/cpu/policies/base.py index 3efffbd396..d3ef8b859a 100644 --- a/vllm/v1/kv_offload/cpu/policies/base.py +++ b/vllm/v1/kv_offload/cpu/policies/base.py @@ -23,8 +23,7 @@ def order_request_keys( class ChunkStatus(ctypes.Structure): - """ - Offloading status for a single chunk of KV data. + """Offloading status for a single chunk of KV data. Holds the following information: ref_cnt - the current number of transfers using this chunk as a source. @@ -42,15 +41,12 @@ class ChunkStatus(ctypes.Structure): @property def is_ready(self) -> bool: - """ - Returns whether the chunk is ready to be read. - """ + """Returns whether the chunk is ready to be read.""" return self.ref_cnt >= 0 class CachePolicy(ABC): - """ - Encapsulates both chunk organization (data structures) and replacement + """Encapsulates both chunk organization (data structures) and replacement decisions (which chunk to evict). LRU and ARC differ in both dimensions — ARC's ghost lists and target_t1_size live at the intersection of storage and eviction, so they cannot be separated cleanly. @@ -73,12 +69,12 @@ class CachePolicy(ABC): @abstractmethod def touch(self, keys: Iterable[OffloadKey], req_context: ReqContext) -> None: - """ - Mark chunks as recently used. + """Mark chunks as recently used. Args: keys: Chunks to mark as recently used. req_context: Per-request context for the request touching these chunks. + """ def on_store_miss( @@ -120,8 +116,7 @@ class CachePolicy(ABC): def evict( self, n: int, protected: set[OffloadKey] ) -> list[tuple[OffloadKey, ChunkStatus]] | None: - """ - Evict exactly n chunks, skipping any in protected. + """Evict exactly n chunks, skipping any in protected. Returns a list of (key, chunk) for the evicted chunks, or None if n evictions cannot be satisfied. The operation is atomic: @@ -133,8 +128,7 @@ class CachePolicy(ABC): @abstractmethod def clear(self) -> None: - """ - Remove ALL chunks regardless of ref_cnt. + """Remove ALL chunks regardless of ref_cnt. Ghost lists and adaptive state are also reset. """ diff --git a/vllm/v1/kv_offload/cpu/policies/factory.py b/vllm/v1/kv_offload/cpu/policies/factory.py index eedcc68076..e4f811d49f 100644 --- a/vllm/v1/kv_offload/cpu/policies/factory.py +++ b/vllm/v1/kv_offload/cpu/policies/factory.py @@ -57,6 +57,7 @@ class CachePolicyFactory: Raises ValueError if the cache policy is neither registered nor resolvable via `module_path`. + """ if name in cls._registry: return cls._registry[name]() diff --git a/vllm/v1/kv_offload/cpu/policies/lru.py b/vllm/v1/kv_offload/cpu/policies/lru.py index 5b5f65dec7..6ca9cb32f4 100644 --- a/vllm/v1/kv_offload/cpu/policies/lru.py +++ b/vllm/v1/kv_offload/cpu/policies/lru.py @@ -14,8 +14,7 @@ from vllm.v1.kv_offload.cpu.policies.base import ( class LRUCachePolicy(CachePolicy): - """ - LRU caching policy with logical recency independent of transfer pinning. + """LRU caching policy with logical recency independent of transfer pinning. Evictable chunks live in a lazy-invalidating min-heap. A chunk's recency can therefore be updated while it is pinned; when it later becomes diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index 284f1de68e..af183fbc4f 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -69,8 +69,7 @@ def _get_populate_write_fn( class SharedOffloadRegion: - """ - Single mmap-backed memory region shared across all workers for a + """Single mmap-backed memory region shared across all workers for a vLLM instance. Workers coordinate via the filesystem: the first worker to open the file with O_EXCL becomes the creator and calls ftruncate; the rest open the existing file and wait until it reaches the expected @@ -264,6 +263,7 @@ class SharedOffloadRegion: Args: tensor_page_size: Bytes per chunk for this tensor. + """ assert self.rank is not None new_offset = self._worker_offset + tensor_page_size @@ -313,6 +313,7 @@ class SharedOffloadRegion: Args: tensor_page_size: Canonical bytes per chunk for this tensor. + """ new_offset = self._canonical_offset + tensor_page_size assert new_offset <= self._row_stride diff --git a/vllm/v1/kv_offload/file_mapper.py b/vllm/v1/kv_offload/file_mapper.py index 82ebe31e2d..1ae0d2d1fb 100644 --- a/vllm/v1/kv_offload/file_mapper.py +++ b/vllm/v1/kv_offload/file_mapper.py @@ -19,9 +19,7 @@ _CONFIG_FILENAME = "config.json" class FileMapper: - """ - FileMapper maps KV blocks (given by their hash) to file names. - """ + """FileMapper maps KV blocks (given by their hash) to file names.""" def __init__( self, @@ -41,8 +39,7 @@ class FileMapper: replicated_layout: bool = False, canonical_format: str | None = None, ): - """ - Initialize the file mapper. Each worker constructs its own, but + """Initialize the file mapper. Each worker constructs its own, but `config.json` is shared across workers since rank lives outside the hash. When `parallel_agnostic=True`, tp/pp/pcp/dcp are forced to 1 and rank to 0 so multiple parallelism layouts collapse into the same folder. @@ -136,8 +133,7 @@ class FileMapper: @staticmethod def _compute_base_path(root_dir: str, fields: dict) -> str: - """ - Layout: /_/. + """Layout: /_/. safe_model_name replaces '/' with '_' so HuggingFace IDs don't nest. """ canonical = json.dumps(fields, sort_keys=True, separators=(",", ":")) diff --git a/vllm/v1/kv_offload/tiering/async_lookup.py b/vllm/v1/kv_offload/tiering/async_lookup.py index c08c63d018..a6879d8c44 100644 --- a/vllm/v1/kv_offload/tiering/async_lookup.py +++ b/vllm/v1/kv_offload/tiering/async_lookup.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -AsyncLookupManager: per-tier async lookup manager for secondary tier +"""AsyncLookupManager: per-tier async lookup manager for secondary tier existence checks. Each secondary tier that wants non-blocking lookups composes its own @@ -65,8 +64,7 @@ class LookupState: class AsyncLookupManager(ABC): - """ - Per-tier async lookup manager for secondary tier existence checks. + """Per-tier async lookup manager for secondary tier existence checks. Each secondary tier that wants non-blocking lookups composes its own AsyncLookupManager instance internally. The manager maintains lookup @@ -126,8 +124,7 @@ class AsyncLookupManager(ABC): def batch_lookup( self, keys: list[OffloadKey], req_context: ReqContext ) -> Iterable[bool]: - """ - Check whether a batch of blocks exist in this tier. + """Check whether a batch of blocks exist in this tier. Called from the worker thread — must be synchronous and must not touch the primary tier or scheduler state. @@ -141,13 +138,13 @@ class AsyncLookupManager(ABC): # ------------------------------------------------------------------ def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None: - """ - Non-blocking lookup called from the scheduler thread. + """Non-blocking lookup called from the scheduler thread. Returns: True — block is present in this tier. False — block is not present in this tier. None — result not yet available; retry next step. + """ if self._need_to_drain: self.drain_results() diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index 34aa6ddf78..77300fa707 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Abstract interfaces and data types for the secondary tiering layer. -""" +"""Abstract interfaces and data types for the secondary tiering layer.""" from abc import ABC, abstractmethod from collections.abc import Collection, Iterable @@ -119,8 +117,7 @@ class ParentManager(ABC): class SecondaryTierManager(ABC): - """ - Abstract interface for managing a single non-primary offloading tier. + """Abstract interface for managing a single non-primary offloading tier. Secondary tiers cannot directly access GPU memory. All data transfers must go through the CPU (primary) tier: @@ -140,12 +137,12 @@ class SecondaryTierManager(ABC): primary_kv_view: memoryview, tier_type: str, ) -> None: - """ - Args: - offloading_spec: Offloading configuration. - primary_kv_view: Memoryview of the primary tier's CPU KV cache. - tier_type: Tier type identifier, set by SecondaryTierFactory - from the registered tier type. + """Args: + offloading_spec: Offloading configuration. + primary_kv_view: Memoryview of the primary tier's CPU KV cache. + tier_type: Tier type identifier, set by SecondaryTierFactory + from the registered tier type. + """ self._offloading_spec = offloading_spec self._primary_kv_view: memoryview = primary_kv_view @@ -154,8 +151,7 @@ class SecondaryTierManager(ABC): @abstractmethod def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: - """ - Check whether a chunk exists in this secondary tier. + """Check whether a chunk exists in this secondary tier. Args: key: Offload key to look up. @@ -165,13 +161,13 @@ class SecondaryTierManager(ABC): HIT if the chunk is present and ready, MISS if not found, or RETRY if the chunk is being transferred (retry later). + """ pass @abstractmethod def submit_store(self, job_metadata: TransferJob) -> None: - """ - Submit an async job to store chunks from the primary tier to this + """Submit an async job to store chunks from the primary tier to this secondary tier. This method must be lightweight and non-blocking: allocate metadata @@ -193,13 +189,13 @@ class SecondaryTierManager(ABC): Args: job_metadata: Job metadata including job_id, keys, and chunk_ids identifying the primary-tier slots to read from. + """ pass @abstractmethod def submit_load(self, job_metadata: TransferJob) -> None: - """ - Submit an async job to load chunks from this secondary tier to the + """Submit an async job to load chunks from this secondary tier to the primary tier. This method must be lightweight and non-blocking: mark chunks as @@ -218,13 +214,13 @@ class SecondaryTierManager(ABC): Args: job_metadata: Job metadata including job_id, keys, and chunk_ids identifying the primary-tier slots to write into. + """ pass @abstractmethod def get_finished_jobs(self) -> Iterable[JobResult]: - """ - Return all jobs (loads and stores) that completed since the last call. + """Return all jobs (loads and stores) that completed since the last call. The framework uses these results to release resources and finalize transfers. @@ -232,6 +228,7 @@ class SecondaryTierManager(ABC): Returns: Iterable of JobResult objects for jobs finished since the last call. + """ pass @@ -248,31 +245,30 @@ class SecondaryTierManager(ABC): return () def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): - """ - Mark chunks as recently used for eviction policy. + """Mark chunks as recently used for eviction policy. Args: keys: Offload keys to mark as recently used. req_context: Per-request context. + """ return @abstractmethod def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: - """ - Called when a new request is first seen by the scheduler. + """Called when a new request is first seen by the scheduler. Returns a RequestOffloadingContext expressing this tier's preference for how chunks should be offloaded for this request. Args: req_context: Per-request context. + """ pass def on_request_finished(self, req_context: ReqContext) -> None: - """ - Called when a request has finished. + """Called when a request has finished. By the time this is called, all per-request calls for this request (submit_store, submit_load, touch) have already been issued, and none @@ -283,6 +279,7 @@ class SecondaryTierManager(ABC): Args: req_context: per-request context. + """ return @@ -300,6 +297,7 @@ class SecondaryTierManager(ABC): Args: context: Per-step context from the scheduler. + """ return diff --git a/vllm/v1/kv_offload/tiering/example/manager.py b/vllm/v1/kv_offload/tiering/example/manager.py index 967f7cc498..5ec13f62da 100644 --- a/vllm/v1/kv_offload/tiering/example/manager.py +++ b/vllm/v1/kv_offload/tiering/example/manager.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -ExampleSecondaryTierManager: A simple in-memory secondary tier. +"""ExampleSecondaryTierManager: A simple in-memory secondary tier. This implementation provides a minimal secondary tier that stores chunks in memory (using a dictionary) with immediate completion. It serves as a @@ -35,8 +34,7 @@ if TYPE_CHECKING: class ExampleSecondaryTierManager(SecondaryTierManager): - """ - A simple in-memory secondary tier. + """A simple in-memory secondary tier. This implementation: - Stores chunks in a dictionary (key -> True) @@ -52,11 +50,14 @@ class ExampleSecondaryTierManager(SecondaryTierManager): tier_type: str, custom_param: int = 0, ): - """ - Initialize the example secondary tier. + """Initialize the example secondary tier. Args: + offloading_spec: The offloading spec this tier belongs to. + primary_kv_view: Memoryview over the primary tier's KV buffer. + tier_type: Name identifying this tier type. custom_param: Dummy parameter demonstrating custom args. + """ super().__init__( offloading_spec=offloading_spec, @@ -78,8 +79,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager): @override def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: - """ - Check whether a chunk exists in this secondary tier. + """Check whether a chunk exists in this secondary tier. Args: key: Offload key to look up. @@ -87,17 +87,18 @@ class ExampleSecondaryTierManager(SecondaryTierManager): Returns: HIT if the chunk is present, MISS if not found. + """ return LookupResult.HIT if key in self.chunks else LookupResult.MISS @override def submit_store(self, job_metadata: TransferJob) -> None: - """ - Submit a job to store chunks from primary tier to this tier. + """Submit a job to store chunks from primary tier to this tier. Args: job_metadata: Job metadata including job_id, keys, and spec for reading chunks from the primary tier. + """ keys = job_metadata.keys chunk_ids = job_metadata.chunk_ids @@ -118,12 +119,12 @@ class ExampleSecondaryTierManager(SecondaryTierManager): @override def submit_load(self, job_metadata: TransferJob) -> None: - """ - Submit a job to load chunks from this tier to primary tier. + """Submit a job to load chunks from this tier to primary tier. Args: job_metadata: Job metadata including job_id, keys, and spec for writing chunks into the primary tier. + """ keys = job_metadata.keys chunk_ids = job_metadata.chunk_ids @@ -149,12 +150,12 @@ class ExampleSecondaryTierManager(SecondaryTierManager): @override def get_finished_jobs(self) -> Iterable[JobResult]: - """ - Poll for finished jobs. + """Poll for finished jobs. Returns: Iterable of JobResult objects for all jobs that have finished since the last call. + """ result = self.completed_jobs self.completed_jobs = [] diff --git a/vllm/v1/kv_offload/tiering/factory.py b/vllm/v1/kv_offload/tiering/factory.py index 3310167b87..086392a406 100644 --- a/vllm/v1/kv_offload/tiering/factory.py +++ b/vllm/v1/kv_offload/tiering/factory.py @@ -71,6 +71,7 @@ class SecondaryTierFactory: Raises: ValueError: If 'type' is missing, or is neither registered nor resolvable via module_path. + """ tier_type = tier_config.get("type") if not tier_type: diff --git a/vllm/v1/kv_offload/tiering/fs/io.py b/vllm/v1/kv_offload/tiering/fs/io.py index 9943c90b54..cda7e46e59 100644 --- a/vllm/v1/kv_offload/tiering/fs/io.py +++ b/vllm/v1/kv_offload/tiering/fs/io.py @@ -94,9 +94,7 @@ def _store_block( block_size: int, use_o_direct: bool = True, ) -> None: - """ - Store callback: Writes to a temp file then atomically replaces the destination. - """ + """Store callback: write to a temp file then atomically replace the target.""" # Check if block already exists to avoid redundant writes if os.path.exists(dest_path): return @@ -172,8 +170,7 @@ def batch_store_block( block_size: int, use_o_direct: bool = True, ) -> None: - """ - Store a batch of KV blocks from a shared buffer to disk in one call. + """Store a batch of KV blocks from a shared buffer to disk in one call. Each block buffer[offsets[i] : offsets[i]+block_size] is written atomically to dest_paths[i] via a temp-file rename. Raises on first error. @@ -197,8 +194,7 @@ def batch_load_block( block_size: int, use_o_direct: bool = True, ) -> None: - """ - Load a batch of KV blocks from disk into a shared buffer in one call. + """Load a batch of KV blocks from disk into a shared buffer in one call. Block i is read from source_paths[i] into view[offsets[i] : offsets[i]+block_size]. Raises on first error (see _load_block for the delete-on-short-read policy). diff --git a/vllm/v1/kv_offload/tiering/fs/manager.py b/vllm/v1/kv_offload/tiering/fs/manager.py index f73f70e77f..3d51d95774 100644 --- a/vllm/v1/kv_offload/tiering/fs/manager.py +++ b/vllm/v1/kv_offload/tiering/fs/manager.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -FileSystemTierManager: Pure-Python file system secondary tier for KV cache offloading. +"""FileSystemTierManager: pure-Python filesystem tier for KV cache offloading. Store path: Data is written to a temp file () via os.write, @@ -84,8 +83,7 @@ class FsAsyncLookupManager(AsyncLookupManager): class FileSystemTierManager(SecondaryTierManager): - """ - Pure-Python disk-backed secondary tier. + """Pure-Python disk-backed secondary tier. Read-priority threads service load jobs preferentially; write-priority threads service store jobs preferentially. Both groups can drain either @@ -118,20 +116,20 @@ class FileSystemTierManager(SecondaryTierManager): enable_kv_events: bool = False, locality: str | None = None, ): - """ - Args: - offloading_spec: Contains normalized offloading configuration and - blocks_per_chunk. - primary_kv_view: Memoryview of the primary tier's CPU KV cache. - tier_type: Tier type identifier, set by SecondaryTierFactory. - root_dir: Root directory for block files. - n_read_threads: Number of read-priority I/O threads. - n_write_threads: Number of write-priority I/O threads. - enable_kv_events: Emit BlockStored KV events for blocks - successfully stored to this tier. Effective only when KV - cache events are enabled globally (kv_events_config). - locality: Whether this tier's storage is LOCAL or REMOTE relative - to the publishing vLLM instance. + """Args: + offloading_spec: Contains normalized offloading configuration and + blocks_per_chunk. + primary_kv_view: Memoryview of the primary tier's CPU KV cache. + tier_type: Tier type identifier, set by SecondaryTierFactory. + root_dir: Root directory for block files. + n_read_threads: Number of read-priority I/O threads. + n_write_threads: Number of write-priority I/O threads. + enable_kv_events: Emit BlockStored KV events for blocks + successfully stored to this tier. Effective only when KV + cache events are enabled globally (kv_events_config). + locality: Whether this tier's storage is LOCAL or REMOTE relative + to the publishing vLLM instance. + """ super().__init__(offloading_spec, primary_kv_view, tier_type) self.locality = Locality(locality) if locality is not None else None @@ -333,8 +331,7 @@ class FileSystemTierManager(SecondaryTierManager): @override def shutdown(self) -> None: - """ - Release resources held by this tier. + """Release resources held by this tier. Shuts down the lookup manager and the thread pool, clearing pending tasks and waiting for active threads to complete. diff --git a/vllm/v1/kv_offload/tiering/fs/thread_pool.py b/vllm/v1/kv_offload/tiering/fs/thread_pool.py index 9db0629192..3c246708cf 100644 --- a/vllm/v1/kv_offload/tiering/fs/thread_pool.py +++ b/vllm/v1/kv_offload/tiering/fs/thread_pool.py @@ -1,11 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Thread pool: - Two queues (load, store) and two sets of threads: - - Load-priority threads: drain the load queue first, then the store queue. - - Store-priority threads: drain the store queue first, then the load queue. - Load jobs are enqueued to the load queue; store jobs to the store queue. +"""Thread pool: +Two queues (load, store) and two sets of threads: +- Load-priority threads: drain the load queue first, then the store queue. +- Store-priority threads: drain the store queue first, then the load queue. +Load jobs are enqueued to the load queue; store jobs to the store queue. """ import threading @@ -20,8 +19,7 @@ logger = init_logger(__name__) class JobState: - """ - Thread-safe completion tracker for a set of per-block I/O tasks. + """Thread-safe completion tracker for a set of per-block I/O tasks. Each task calls task_done(success) when it finishes. """ @@ -50,7 +48,7 @@ class JobState: def task_done( self, success: bool, transfer_time: float ) -> tuple[bool, bool, float]: - """Returns if job completed and success flag""" + """Returns if job completed and success flag.""" with self._lock: self._completed += 1 self._transfer_time += transfer_time @@ -60,8 +58,7 @@ class JobState: class DualQueueThreadPool: - """ - Thread pool with two task queues (load and store) and two thread groups. + """Thread pool with two task queues (load and store) and two thread groups. Load-priority threads drain the load queue first, then fall back to the store queue. Store-priority threads do the reverse. Both queues share diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index f19616828d..6e0bf90a33 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -TieringOffloadingManager: Multi-tier KV cache offloading orchestrator. +"""TieringOffloadingManager: Multi-tier KV cache offloading orchestrator. This manager coordinates between a CPU primary tier (with direct GPU access) and zero or more secondary tiers (Storage, Network, etc.) to provide @@ -174,8 +173,7 @@ class _SecondaryTierFacingParent(ParentManager): class TieringOffloadingManager(OffloadingManager): - """ - Orchestrates multi-tier KV cache offloading. + """Orchestrates multi-tier KV cache offloading. This manager coordinates between a CPU primary tier (with direct GPU access) and zero or more secondary tiers (Storage, Network, etc.) to provide @@ -194,13 +192,13 @@ class TieringOffloadingManager(OffloadingManager): primary_tier: CPUPrimaryTierOffloadingManager, secondary_tiers: list[SecondaryTierManager] | None = None, ): - """ - Initialize the TieringOffloadingManager. + """Initialize the TieringOffloadingManager. Args: primary_tier: The primary tier manager (CPU-based). secondary_tiers: List of secondary tier managers (e.g., Storage, Network). Can be None or empty list. + """ self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier self.secondary_tiers = secondary_tiers or [] @@ -259,8 +257,7 @@ class TieringOffloadingManager(OffloadingManager): return self._jobs.pop(job_id, None) def _maybe_process_finished_jobs(self): - """ - Poll secondary tiers for completed jobs (at most once per step). + """Poll secondary tiers for completed jobs (at most once per step). Guarded by _processed_jobs_this_step: the first call in an engine step does the actual polling; subsequent calls are no-ops. The flag is reset @@ -306,8 +303,7 @@ class TieringOffloadingManager(OffloadingManager): ) def _process_finished_jobs(self): - """ - Unconditionally poll all secondary tiers for completed jobs. + """Unconditionally poll all secondary tiers for completed jobs. This method: 1. Calls get_finished_jobs() on each secondary tier @@ -350,8 +346,7 @@ class TieringOffloadingManager(OffloadingManager): *, exclude_tier_idx: int | None = None, ) -> LookupResult: - """ - Check whether a single chunk is offloaded and ready. + """Check whether a single chunk is offloaded and ready. Algorithm: 1. Process any completed async jobs first. @@ -362,6 +357,7 @@ class TieringOffloadingManager(OffloadingManager): Args: key: Chunk hash to look up. req_context: Per-request context. + exclude_tier_idx: Skip this tier index during the lookup. Returns: HIT — chunk is ready in the primary tier. @@ -370,6 +366,7 @@ class TieringOffloadingManager(OffloadingManager): RETRY — promotion started or a secondary tier is busy. MISS — chunk not found in any tier, or primary is full and cannot accept a promotion. + """ # Poll first so a promotion that finished since the last call is # already reflected as HIT (not stale HIT_PENDING/MISS) below, and @@ -432,8 +429,7 @@ class TieringOffloadingManager(OffloadingManager): key: OffloadKey, req_context: ReqContext, ) -> bool: - """ - Queue a chunk for promotion from a secondary tier to the primary tier. + """Queue a chunk for promotion from a secondary tier to the primary tier. Allocates space in the primary tier immediately (sets ref_cnt=-1 so subsequent lookups within the same step see the slot as in-flight), @@ -448,6 +444,7 @@ class TieringOffloadingManager(OffloadingManager): Returns: True if promotion was initiated, False if primary tier is full. + """ # Allocate space in primary tier for promoted chunk. # Must happen immediately so primary.lookup() returns None (in-flight) @@ -505,8 +502,7 @@ class TieringOffloadingManager(OffloadingManager): def prepare_load( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> LoadStoreSpec: - """ - Prepare chunks to be loaded from primary tier to GPU. + """Prepare chunks to be loaded from primary tier to GPU. Callers only pass keys already confirmed HIT by lookup() earlier this step. @@ -520,17 +516,18 @@ class TieringOffloadingManager(OffloadingManager): Returns: LoadStoreSpec for reading from primary tier. + """ return self.primary_tier.prepare_load(keys, req_context) @override def touch(self, keys: Collection[OffloadKey], req_context: ReqContext): - """ - Mark chunks as recently used in all tiers. + """Mark chunks as recently used in all tiers. Args: keys: Chunks to mark as recently used. req_context: Per-request context. + """ self.primary_tier.touch(keys, req_context) for tier in self.secondary_tiers: @@ -538,8 +535,7 @@ class TieringOffloadingManager(OffloadingManager): @override def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext): - """ - Mark chunks as done loading from primary tier to GPU. + """Mark chunks as done loading from primary tier to GPU. This decrements ref_cnt on the chunks in the primary tier, allowing them to be evicted again. @@ -547,6 +543,7 @@ class TieringOffloadingManager(OffloadingManager): Args: keys: Chunks that finished loading. req_context: Per-request context. + """ self.primary_tier.complete_load(keys, req_context) @@ -554,8 +551,7 @@ class TieringOffloadingManager(OffloadingManager): def prepare_store( self, keys: Collection[OffloadKey], req_context: ReqContext ) -> PrepareStoreOutput | None: - """ - Prepare chunks to be stored from GPU to primary tier. + """Prepare chunks to be stored from GPU to primary tier. CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure that any completed async transfers have their ref_cnt decremented @@ -571,6 +567,7 @@ class TieringOffloadingManager(OffloadingManager): Returns: PrepareStoreOutput describing where to store chunks and what was evicted, or None if store cannot proceed. + """ # Step 1: Poll for completed async jobs FIRST # _process_finished_jobs() handles two kinds of completions here: @@ -617,8 +614,7 @@ class TieringOffloadingManager(OffloadingManager): req_context: ReqContext, request_level_tiers: set[int], ) -> None: - """ - For tiers that requested request-level policy, submit_store() for + """For tiers that requested request-level policy, submit_store() for chunks that are already present in the primary tier. A key whose primary write is still in flight (HIT_PENDING) cannot be @@ -674,8 +670,7 @@ class TieringOffloadingManager(OffloadingManager): req_context: ReqContext, success: bool = True, ) -> None: - """ - Mark chunks as done storing from GPU to primary tier. + """Mark chunks as done storing from GPU to primary tier. This is where secondary tier cascading happens — after chunks are confirmed to be in the primary tier, they are cascaded to ALL @@ -691,6 +686,7 @@ class TieringOffloadingManager(OffloadingManager): keys: Chunks that finished storing. success: Whether the GPU→primary transfer succeeded. req_context: Per-request context forwarded to primary.prepare_read(). + """ # Step 1: Complete store in primary tier (makes chunks loadable) self.primary_tier.complete_store(keys, req_context, success) @@ -748,8 +744,7 @@ class TieringOffloadingManager(OffloadingManager): *, exclude_tier_idx: int | None = None, ) -> RequestOffloadingContext: - """ - Query each secondary tier for its offload policy preference. + """Query each secondary tier for its offload policy preference. Returns REQUEST_LEVEL if ANY secondary tier wants request-level. Only stores REQUEST_LEVEL tier decisions for use in prepare_store. @@ -858,6 +853,7 @@ class TieringOffloadingManager(OffloadingManager): Yields: New OffloadingEvents collected by each tier since the last call. + """ yield from self.primary_tier.take_events() for tier in self.secondary_tiers: diff --git a/vllm/v1/kv_offload/tiering/obj/manager.py b/vllm/v1/kv_offload/tiering/obj/manager.py index 50a7e611f6..3d6c66c483 100644 --- a/vllm/v1/kv_offload/tiering/obj/manager.py +++ b/vllm/v1/kv_offload/tiering/obj/manager.py @@ -113,19 +113,19 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager): enable_kv_events: bool = False, locality: str | None = None, ): - """ - Args: - offloading_spec: Offloading configuration. - primary_kv_view: Memoryview of the primary tier's CPU KV cache. - tier_type: Tier type identifier, set by SecondaryTierFactory. - store_config: Object store connection parameters (see ObjStoreConfig). - prefix: Key prefix prepended to all object keys. - io_threads: Number of NIXL I/O threads. - enable_kv_events: Emit BlockStored KV events for blocks - successfully stored to this tier. Effective only when KV - cache events are enabled globally (kv_events_config). - locality: Whether this tier's storage is LOCAL or REMOTE relative - to the publishing vLLM instance. + """Args: + offloading_spec: Offloading configuration. + primary_kv_view: Memoryview of the primary tier's CPU KV cache. + tier_type: Tier type identifier, set by SecondaryTierFactory. + store_config: Object store connection parameters (see ObjStoreConfig). + prefix: Key prefix prepended to all object keys. + io_threads: Number of NIXL I/O threads. + enable_kv_events: Emit BlockStored KV events for blocks + successfully stored to this tier. Effective only when KV + cache events are enabled globally (kv_events_config). + locality: Whether this tier's storage is LOCAL or REMOTE relative + to the publishing vLLM instance. + """ super().__init__(offloading_spec, primary_kv_view, tier_type) self.locality = Locality(locality) if locality is not None else None diff --git a/vllm/v1/kv_offload/tiering/p2p/control/base.py b/vllm/v1/kv_offload/tiering/p2p/control/base.py index 7016cfb162..5a7030737b 100644 --- a/vllm/v1/kv_offload/tiering/p2p/control/base.py +++ b/vllm/v1/kv_offload/tiering/p2p/control/base.py @@ -139,6 +139,7 @@ class ControlTransport(ABC): A connection to peer_id that is registered but no longer alive is retired and replaced; a live one is a duplicate and must not be replaced. + """ ... @@ -158,6 +159,7 @@ class ControlTransport(ABC): The returned sequence is read-only — callers must not mutate it. The caller is responsible for creating sessions for these connections. + """ ... diff --git a/vllm/v1/kv_offload/tiering/p2p/control/zmq.py b/vllm/v1/kv_offload/tiering/p2p/control/zmq.py index 42dc2c6984..b28ed76dac 100644 --- a/vllm/v1/kv_offload/tiering/p2p/control/zmq.py +++ b/vllm/v1/kv_offload/tiering/p2p/control/zmq.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -ZMQ-based transport layer for P2P KV cache sharing. +"""ZMQ-based transport layer for P2P KV cache sharing. Provides ZmqConnection (per-peer messaging) and ZmqTransport (connection management). Message-content agnostic. diff --git a/vllm/v1/kv_offload/tiering/p2p/data/base.py b/vllm/v1/kv_offload/tiering/p2p/data/base.py index ffe609d71c..62304b7544 100644 --- a/vllm/v1/kv_offload/tiering/p2p/data/base.py +++ b/vllm/v1/kv_offload/tiering/p2p/data/base.py @@ -87,6 +87,7 @@ class PollResult(NamedTuple): Attributes: done: Transfer IDs that completed successfully. failed: Transfer IDs that failed (error, timeout, etc.). + """ done: Sequence[int] @@ -174,6 +175,7 @@ class DataTransport(ABC): base_addr: Base address of the peer's block memory region. num_blocks: Number of blocks in the peer's region. block_len: Size of each block (must match local block_len). + """ ... @@ -203,6 +205,7 @@ class DataTransport(ABC): Returns: A unique transfer_id (int) to track this transfer, or None if the peer is not registered or submission failed. + """ ... @@ -224,6 +227,7 @@ class DataTransport(ABC): Completed/failed transfers are removed from the inflight set. Must be called periodically to drive progress checking. + """ ... @@ -255,6 +259,7 @@ class DataTransport(ABC): For mode="wait", the subset of *transfer_ids* still tracked as inflight after the cancel attempt. For mode="immediate", always []. + """ ... diff --git a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py index bf3adf4b60..e8fb66dd6e 100644 --- a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py +++ b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL. -""" +"""NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL.""" from __future__ import annotations diff --git a/vllm/v1/kv_offload/tiering/p2p/manager.py b/vllm/v1/kv_offload/tiering/p2p/manager.py index ba1973b767..703c632899 100644 --- a/vllm/v1/kv_offload/tiering/p2p/manager.py +++ b/vllm/v1/kv_offload/tiering/p2p/manager.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -P2PSecondaryTierManager: Secondary tier for P2P KV cache sharing. +"""P2PSecondaryTierManager: Secondary tier for P2P KV cache sharing. Owns transports and a single bidirectional P2PSession per remote peer. """ @@ -264,6 +263,7 @@ class P2PSecondaryTierManager(SecondaryTierManager): Raises: ValueError: If ``unbound_store_timeout_s`` is not a positive number, or anything convertible to one. + """ super().__init__(offloading_spec, primary_kv_view, tier_type) try: diff --git a/vllm/v1/kv_offload/tiering/p2p/session/protocol.py b/vllm/v1/kv_offload/tiering/p2p/session/protocol.py index ec4153e25b..8369bbffe8 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/protocol.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/protocol.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -P2P KV cache sharing protocol constants and documentation. +"""P2P KV cache sharing protocol constants and documentation. Protocol Overview ================= diff --git a/vllm/v1/kv_offload/tiering/p2p/session/session.py b/vllm/v1/kv_offload/tiering/p2p/session/session.py index ba2aca6438..f43e1a5aab 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/session.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/session.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -P2PSession — bidirectional session combining client + server roles. +"""P2PSession — bidirectional session combining client + server roles. A single P2PSession per remote peer handles BOTH directions of the P2P protocol on one ControlConnection: it can request blocks from the peer diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index 9922a4095c..9adac0f787 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -TieringOffloadingSpec: Spec for multi-tier KV cache offloading. +"""TieringOffloadingSpec: Spec for multi-tier KV cache offloading. This spec creates a TieringOffloadingManager with a CPU primary tier and configurable secondary tiers (e.g., Storage, Network). @@ -82,8 +81,7 @@ logger = init_logger(__name__) class TieringOffloadingSpec(CPUOffloadingSpec): - """ - Spec for multi-tier KV cache offloading. + """Spec for multi-tier KV cache offloading. Creates a TieringOffloadingManager with: - Primary tier: CPU (LRU or ARC eviction policy) @@ -276,8 +274,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec): @override def get_manager(self) -> OffloadingManager: - """ - Get the TieringOffloadingManager. + """Get the TieringOffloadingManager. Creates a TieringOffloadingManager with: - Primary tier: CPU (LRU or ARC) @@ -285,6 +282,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec): Returns: TieringOffloadingManager instance + """ if not self._manager: if int(self.extra_config.get("store_threshold", 0)) >= 2: diff --git a/vllm/v1/metrics/buckets.py b/vllm/v1/metrics/buckets.py index 375d730729..7b9efb8f6d 100644 --- a/vllm/v1/metrics/buckets.py +++ b/vllm/v1/metrics/buckets.py @@ -177,6 +177,7 @@ def build_buckets(mantissa_lst: list[int], max_value: int) -> list[float]: Returns: Bucket values in increasing order, capped at `max_value`. + """ exponent = 0 buckets: list[float] = [] @@ -196,6 +197,7 @@ def build_1_2_5_buckets(max_value: int) -> list[float]: Example: >>> build_1_2_5_buckets(100) [1, 2, 5, 10, 20, 50, 100] + """ return build_buckets([1, 2, 5], max_value) @@ -218,6 +220,7 @@ def histogram_buckets( Raises: ValueError: If `family` is `request_tokens` and `max_model_len` is None. + """ if family == "request_tokens": if max_model_len is None: diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 3a8adb6e32..50e606d3b3 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -1202,16 +1202,15 @@ class PrometheusStatLogger(AggregateStatLoggerBase): class StatLoggerManager: - """ - StatLoggerManager: - Logging happens at the level of the EngineCore (per scheduler). - * DP: >1 EngineCore per AsyncLLM - loggers for each EngineCore. - * With Local Logger, just make N copies for N EngineCores. - * With Prometheus, we need a single logger with N "labels" + """StatLoggerManager: + Logging happens at the level of the EngineCore (per scheduler). + * DP: >1 EngineCore per AsyncLLM - loggers for each EngineCore. + * With Local Logger, just make N copies for N EngineCores. + * With Prometheus, we need a single logger with N "labels" - This class abstracts away this implementation detail from - the AsyncLLM, allowing the AsyncLLM to just call .record() - and .log() to a simple interface. + This class abstracts away this implementation detail from + the AsyncLLM, allowing the AsyncLLM to just call .record() + and .log() to a simple interface. """ def __init__( diff --git a/vllm/v1/metrics/perf.py b/vllm/v1/metrics/perf.py index ebb676e66d..dd046a8250 100644 --- a/vllm/v1/metrics/perf.py +++ b/vllm/v1/metrics/perf.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Analytic flops/memory estimation module for transformer components, +"""Analytic flops/memory estimation module for transformer components, to help derive MFU (Model Flops Utilization) stats for a running model. """ @@ -33,8 +32,7 @@ logger = init_logger(__name__) class InvalidComponent(Exception): - """ - Custom exception to indicate that a certain ComponentMetric is not + """Custom exception to indicate that a certain ComponentMetric is not applicable to the given VllmConfig. """ @@ -99,8 +97,7 @@ class PerfStats: @dataclass class ExecutionContext: - """ - Represents an execution context for a batch of requests. + """Represents an execution context for a batch of requests. This class aggregates statistics across multiple requests in a batch, separately tracking prefill and decode phases. @@ -202,8 +199,7 @@ class ExecutionContext: class ParsedArgs: - """ - Syntactic sugar so that Parsers can use dot notations + """Syntactic sugar so that Parsers can use dot notations to access/update the parsed arguments. e.g.) @@ -227,16 +223,14 @@ class ParsedArgs: class Parser(Protocol): def parse(self, args: ParsedArgs, vllm_config: VllmConfig) -> ParsedArgs: - """ - Parse the vllm config and update the current ParsedArgs and pass it on. + """Parse the vllm config and update the current ParsedArgs and pass it on. If the parser isn't applicable to the vllm_config, it will do nothing. """ ... class ParserChain: - """ - Applies chain of parser in a sequential order. + """Applies chain of parser in a sequential order. Later parsers might overwrite results from previous parsers, so parsers should be chained in the appropriate order if they are not mutually exclusive. @@ -260,8 +254,7 @@ _COMPONENT_METRICS_REGISTRY: dict[str, type["ComponentMetrics"]] = {} class ComponentMetrics(BaseModel, ABC): - """ - Each concrete ComponentMetrics class is associated with: + """Each concrete ComponentMetrics class is associated with: - fields that are required for metric derivation (fields are specified/validated through pydantic model) - parser to parse VllmConfig into fields @@ -277,8 +270,7 @@ class ComponentMetrics(BaseModel, ABC): @classmethod @abstractmethod def get_parser(cls) -> ParserChain: - """ - Return a ParserChain that provides values for all required fields. + """Return a ParserChain that provides values for all required fields. The returned parser chain must populate ParsedArgs with values for every field defined on this ComponentMetrics class. Missing fields will cause a ValidationError when from_vllm_config() is called. @@ -292,11 +284,9 @@ class ComponentMetrics(BaseModel, ABC): @classmethod def from_vllm_config(cls, vllm_config: VllmConfig) -> Self: - """ - Instantiate this class from VllmConfig. + """Instantiate this class from VllmConfig. Raises ValidationError if parsing fails. """ - parser = cls.get_parser() parsed_args = parser.parse(vllm_config) try: @@ -343,8 +333,7 @@ class ComponentMetrics(BaseModel, ABC): class BaseConfigParser(Parser): - """ - Parses base model configuration. + """Parses base model configuration. Provides: vocab_size, hidden_size, num_attention_heads, num_hidden_layers, weight_byte_size, activation_byte_size, dp_size, tp_size, pp_size, enable_ep """ @@ -396,8 +385,7 @@ class BaseConfigParser(Parser): class BaseAttentionConfigParser(Parser): - """ - Parses attention-specific configuration. + """Parses attention-specific configuration. Provides: num_key_value_heads, head_dim, cache_byte_size """ @@ -418,8 +406,7 @@ class BaseAttentionConfigParser(Parser): class SlidingWindowAttentionParser(Parser): - """ - Parses sliding window attention configuration and layer breakdown. + """Parses sliding window attention configuration and layer breakdown. Provides: sliding_window, num_swa_layers """ @@ -466,8 +453,7 @@ class SlidingWindowAttentionParser(Parser): class AttentionQuantizationConfigParser(Parser): - """ - Parses quantization configuration for attention layers. + """Parses quantization configuration for attention layers. Overrides: weight_byte_size """ @@ -490,8 +476,7 @@ class AttentionQuantizationConfigParser(Parser): class AttentionDetectionParser(Parser): - """ - Prevents standard AttentionMetrics from being instantiated for MLA models. + """Prevents standard AttentionMetrics from being instantiated for MLA models. MLA models should use MLAAttentionMetrics instead. """ @@ -589,6 +574,7 @@ class AttentionMetrics(ComponentMetrics): Returns: Dict with keys: qkv_proj, out_proj, attn_qk, attn_av. + """ D, q, kv, d = ( self.hidden_size, @@ -642,6 +628,7 @@ class AttentionMetrics(ComponentMetrics): Returns: Dict with keys: qkv_input, qkv_weight, attn_input (conditional), out_input, out_weight. + """ D, q, kv, d = ( self.hidden_size, @@ -747,8 +734,7 @@ class AttentionMetrics(ComponentMetrics): class MLADetectionParser(Parser): - """ - Validates that the model uses MLA attention. + """Validates that the model uses MLA attention. Raises InvalidComponent if the model does not use MLA, so MLAAttentionMetrics is silently skipped for non-MLA models. """ @@ -761,8 +747,7 @@ class MLADetectionParser(Parser): class MLAConfigParser(Parser): - """ - Parses MLA-specific configuration fields. + """Parses MLA-specific configuration fields. Provides: kv_lora_rank, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, q_lora_rank """ @@ -787,8 +772,7 @@ class MLAConfigParser(Parser): class MLAAttentionMetrics(ComponentMetrics): - """ - Performance metrics for Multi-Latent Attention (MLA) layers. + """Performance metrics for Multi-Latent Attention (MLA) layers. MLA uses a compressed latent representation for KV cache: - KV cache stores a single compressed vector of size @@ -1025,8 +1009,7 @@ class MLAAttentionMetrics(ComponentMetrics): class BaseFfnConfigParser(Parser): - """ - Parses FFN and MoE configuration. + """Parses FFN and MoE configuration. Provides: intermediate_size, num_experts, num_experts_per_tok, moe_intermediate_size, num_shared_experts, num_moe_layers """ @@ -1056,8 +1039,7 @@ class BaseFfnConfigParser(Parser): class FfnParallelParser(Parser): - """ - Parses FFN parallelism configuration. + """Parses FFN parallelism configuration. Provides: ffn_tp_size, ffn_ep_size """ @@ -1077,8 +1059,7 @@ class FfnParallelParser(Parser): class InterleaveMoeLayerStepParser(Parser): - """ - Parses interleave_moe_layer_step field for models like Llama4. + """Parses interleave_moe_layer_step field for models like Llama4. Overrides: num_moe_layers """ @@ -1104,8 +1085,7 @@ class InterleaveMoeLayerStepParser(Parser): class MoeLayerFreqParser(Parser): - """ - Parses moe_layer_freq and first_k_dense_replace fields for models like Deepseek. + """Parses moe_layer_freq and first_k_dense_replace fields for models like Deepseek. Overrides: num_moe_layers """ @@ -1129,8 +1109,7 @@ class MoeLayerFreqParser(Parser): class FfnQuantizationConfigParser(Parser): - """ - Parses quantization configuration for FFN layers. + """Parses quantization configuration for FFN layers. Overrides: weight_byte_size """ @@ -1486,11 +1465,9 @@ class UnembedMetrics(ComponentMetrics): class ModelMetrics: def __init__(self, vllm_config: VllmConfig) -> None: - """ - Parse vllm_config to instantiate metrics for each component. + """Parse vllm_config to instantiate metrics for each component. is_enabled() will return False if no component metrics could be instantiated. """ - self.vllm_config = vllm_config self.metrics: list[ComponentMetrics] = [] @@ -1558,10 +1535,7 @@ class ModelMetrics: def get_step_perf_stats_per_gpu( self, scheduler_output: SchedulerOutput ) -> PerfStats: - """ - Calculate perf stats for the current step based on scheduled tokens. - """ - + """Calculate perf stats for the current step based on scheduled tokens.""" t0 = time.monotonic() # Build a single batch context diff --git a/vllm/v1/metrics/prometheus.py b/vllm/v1/metrics/prometheus.py index c874027671..91bb88310d 100644 --- a/vllm/v1/metrics/prometheus.py +++ b/vllm/v1/metrics/prometheus.py @@ -42,6 +42,7 @@ def get_prometheus_registry() -> CollectorRegistry: Returns: Registry: A prometheus registry + """ if os.getenv("PROMETHEUS_MULTIPROC_DIR") is not None: logger.debug("Using multiprocess registry for prometheus metrics") @@ -70,7 +71,6 @@ def unregister_vllm_metrics(): def shutdown_prometheus(): """Shutdown prometheus metrics.""" - path = _prometheus_multiproc_dir if path is None: return diff --git a/vllm/v1/metrics/ray_wrappers.py b/vllm/v1/metrics/ray_wrappers.py index 7e2100546e..dedc60ba54 100644 --- a/vllm/v1/metrics/ray_wrappers.py +++ b/vllm/v1/metrics/ray_wrappers.py @@ -68,8 +68,7 @@ class RayPrometheusMetric: @staticmethod def _get_sanitized_opentelemetry_name(name: str) -> str: - """ - For compatibility with Ray + OpenTelemetry, the metric name must be + """For compatibility with Ray + OpenTelemetry, the metric name must be sanitized. In particular, this replaces disallowed character (e.g., ':') with '_' in the metric name. Allowed characters: a-z, A-Z, 0-9, _ @@ -78,7 +77,6 @@ class RayPrometheusMetric: Ref: https://github.com/open-telemetry/opentelemetry-cpp/blob/main/sdk/src/metrics/instrument_metadata_validator.cc#L22-L23 Ref: https://github.com/ray-project/ray/blob/master/src/ray/stats/metric.cc#L107 """ - return re.sub(r"[^a-zA-Z0-9_]", "_", name) @@ -169,8 +167,7 @@ class RayHistogramWrapper(RayPrometheusMetric): class RaySpecDecodingProm(SpecDecodingProm): - """ - RaySpecDecodingProm is used by RayMetrics to log to Ray metrics. + """RaySpecDecodingProm is used by RayMetrics to log to Ray metrics. Provides the same metrics as SpecDecodingProm but uses Ray's util.metrics library. """ @@ -179,8 +176,7 @@ class RaySpecDecodingProm(SpecDecodingProm): class RayKVConnectorProm(KVConnectorProm): - """ - RayKVConnectorProm is used by RayMetrics to log Ray + """RayKVConnectorProm is used by RayMetrics to log Ray metrics. Provides the same metrics as KV connectors but uses Ray's util.metrics library. """ @@ -191,8 +187,7 @@ class RayKVConnectorProm(KVConnectorProm): class RayPerfMetricsProm(PerfMetricsProm): - """ - RayPerfMetricsProm is used by RayMetrics to log Ray + """RayPerfMetricsProm is used by RayMetrics to log Ray metrics. Provides the same MFU metrics as PerfMetricsProm uses Ray's util.metrics library. """ diff --git a/vllm/v1/metrics/reader.py b/vllm/v1/metrics/reader.py index 48c88e5b61..e847e86253 100644 --- a/vllm/v1/metrics/reader.py +++ b/vllm/v1/metrics/reader.py @@ -82,6 +82,7 @@ def get_metrics_snapshot() -> list[Metric]: ... print(f" count = {metric.count}") ... for bucket_le, value in metrics.buckets.items(): ... print(f" {bucket_le} = {value}") + """ collected: list[Metric] = [] for metric in REGISTRY.collect(): diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index 24e07e57cc..fe17208ce4 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -34,9 +34,10 @@ class BaseCacheStats: class CachingMetrics: """Metrics for caching with a hit rate of the most recent N requests. + Args: - interval: The number of the most recent requests to aggregate. - Defaults to 1000. + max_recent_requests: The number of the most recent requests to aggregate. + """ def __init__(self, max_recent_requests: int = 1000) -> None: @@ -62,6 +63,7 @@ class CachingMetrics: Args: stats: The prefix cache stats. + """ # reset_prefix_cache was invoked before the current update. # Reset the metrics before aggregating the current stats. @@ -113,8 +115,7 @@ class CachingMetrics: @dataclass class PrefixCacheStats(BaseCacheStats): - """ - Stores prefix cache hit statistics. + """Stores prefix cache hit statistics. - `reset`: Whether `reset_prefix_cache` was invoked. - `queries`: Refers to the number of tokens that were queried. """ @@ -144,8 +145,7 @@ class PrefixCacheStats(BaseCacheStats): @dataclass class MultiModalCacheStats(BaseCacheStats): - """ - Stores multi-modal cache hit statistics. + """Stores multi-modal cache hit statistics. - `reset`: Whether `reset_mm_cache` was invoked. - `queries`: Refers to the number of multi-modal data items that were queried. diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index d8924b2738..35780dce9c 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -166,7 +166,6 @@ class LogprobsTensors(NamedTuple): num_positions: int, num_tokens_per_position: int ) -> "LogprobsTensors": """Create empty LogprobsTensors on CPU.""" - logprob_token_ids = torch.empty( (num_positions, num_tokens_per_position), dtype=torch.int32, @@ -437,8 +436,7 @@ class DraftTokenIds: def make_empty_encoder_model_runner_output( scheduler_output: "SchedulerOutput", ) -> ModelRunnerOutput: - """ - Create a ModelRunnerOutput stub that contains the correct + """Create a ModelRunnerOutput stub that contains the correct per-request bookkeeping but no generated data yet. """ if not scheduler_output.num_scheduled_tokens: diff --git a/vllm/v1/request.py b/vllm/v1/request.py index 8b453a0906..ad875264af 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -348,8 +348,7 @@ class Request: return prefill_stats def __lt__(self, other: "Request") -> bool: - """ - Compare two requests based on priority, arrival time, and request ID. + """Compare two requests based on priority, arrival time, and request ID. Used in priority scheduling. """ if self.priority != other.priority: diff --git a/vllm/v1/sample/logits_processor/__init__.py b/vllm/v1/sample/logits_processor/__init__.py index f319097495..ff9a6428c5 100644 --- a/vllm/v1/sample/logits_processor/__init__.py +++ b/vllm/v1/sample/logits_processor/__init__.py @@ -55,8 +55,7 @@ BUILTIN_LOGITS_PROCESSORS: list[type[LogitsProcessor]] = [ def _load_logitsprocs_plugins() -> list[type[LogitsProcessor]]: - """Load all installed logit processor plugins""" - + """Load all installed logit processor plugins.""" from importlib.metadata import entry_points installed_logitsprocs_plugins = entry_points(group=LOGITSPROCS_GROUP) @@ -171,6 +170,7 @@ def _load_custom_logitsprocs( Returns: A list of all loaded logitproc types + """ from vllm.platforms import current_platform @@ -238,7 +238,7 @@ def validate_logits_processors_parameters( class AdapterLogitsProcessor(LogitsProcessor): - """Wrapper for per-request logits processors + """Wrapper for per-request logits processors. To wrap a specific per-request logits processor, * Subclass `AdapterLogitsProcessor` @@ -264,7 +264,6 @@ class AdapterLogitsProcessor(LogitsProcessor): these arguments are used, the vLLM logits processor interface requires all three arguments to be present. """ - # Map req index -> logits processor state # # State representation is a partial[Tensor] comprising a request-level @@ -301,7 +300,7 @@ class AdapterLogitsProcessor(LogitsProcessor): prompt_ids: list[int] | None, output_ids: list[int], ) -> partial[torch.Tensor] | None: - """Return state representation for new request + """Return state representation for new request. Returns None if logits processor is not applicable to request diff --git a/vllm/v1/sample/logits_processor/builtin.py b/vllm/v1/sample/logits_processor/builtin.py index 45b5c47711..99dd157e78 100644 --- a/vllm/v1/sample/logits_processor/builtin.py +++ b/vllm/v1/sample/logits_processor/builtin.py @@ -45,7 +45,7 @@ class MinPLogitsProcessor(LogitsProcessor): self.min_p: torch.Tensor = self.min_p_device[:0] def is_argmax_invariant(self) -> bool: - """Min-p never impacts greedy sampling""" + """Min-p never impacts greedy sampling.""" return True def get_min_p_by_index(self, index: int) -> float: @@ -377,7 +377,6 @@ def process_dict_updates( new_state: Callable[[SamplingParams, list[int] | None, list[int]], T | None], ) -> bool: """Utility function to update dict state for sparse LogitsProcessors.""" - if not batch_update: # Nothing to do. return False diff --git a/vllm/v1/sample/logits_processor/interface.py b/vllm/v1/sample/logits_processor/interface.py index dce0706335..355ab6a6f8 100644 --- a/vllm/v1/sample/logits_processor/interface.py +++ b/vllm/v1/sample/logits_processor/interface.py @@ -35,7 +35,7 @@ MovedRequest = tuple[int, int, MoveDirectionality] @dataclass(frozen=True) class BatchUpdate: - """Persistent batch state change info for logitsprocs""" + """Persistent batch state change info for logitsprocs.""" batch_size: int # Current num reqs in batch @@ -104,5 +104,6 @@ class LogitsProcessor(ABC): Args: batch_update: Non-None iff there have been changes to the batch makeup. + """ raise NotImplementedError diff --git a/vllm/v1/sample/logits_processor/state.py b/vllm/v1/sample/logits_processor/state.py index 41cbba8dff..7988e8c6e9 100644 --- a/vllm/v1/sample/logits_processor/state.py +++ b/vllm/v1/sample/logits_processor/state.py @@ -81,6 +81,7 @@ class BatchUpdateBuilder: Args: index: request index + """ if self._is_removed_sorted: raise RuntimeError( @@ -93,14 +94,14 @@ class BatchUpdateBuilder: return bool(self._removed) def peek_removed(self) -> int | None: - """Return lowest removed request index""" + """Return lowest removed request index.""" if self.has_removed(): self._ensure_removed_sorted() return self._removed[-1] return None def pop_removed(self) -> int | None: - """Pop lowest removed request index""" + """Pop lowest removed request index.""" if self.has_removed(): self._ensure_removed_sorted() return self._removed.pop() @@ -125,6 +126,7 @@ class BatchUpdateBuilder: Returns: Frozen logitsprocs batch update instance; `None` if no updates + """ # Reset removal-sorting logic self._is_removed_sorted = False diff --git a/vllm/v1/sample/ops/logprobs.py b/vllm/v1/sample/ops/logprobs.py index 9cdcfa456a..359a36a816 100644 --- a/vllm/v1/sample/ops/logprobs.py +++ b/vllm/v1/sample/ops/logprobs.py @@ -9,8 +9,7 @@ from vllm.platforms import current_platform @torch.compile(backend=current_platform.simple_compile_backend) def batched_count_greater_than(x: torch.Tensor, values: torch.Tensor) -> torch.Tensor: - """ - Counts elements in each row of x that are greater than the corresponding + """Counts elements in each row of x that are greater than the corresponding value in values. Use torch.compile to generate an optimized kernel for this function. otherwise, it will create additional copies of the input tensors and cause memory issues. @@ -21,6 +20,7 @@ def batched_count_greater_than(x: torch.Tensor, values: torch.Tensor) -> torch.T Returns: torch.Tensor: A 1D tensor of shape (batch_size,) with the counts. + """ torch._check(x.shape[0] >= 1) torch._check(x.shape[0] == values.shape[0]) diff --git a/vllm/v1/sample/ops/penalties.py b/vllm/v1/sample/ops/penalties.py index 7bc6ec7ab8..2e2981bc4c 100644 --- a/vllm/v1/sample/ops/penalties.py +++ b/vllm/v1/sample/ops/penalties.py @@ -15,9 +15,7 @@ def apply_all_penalties( repetition_penalties: torch.Tensor, output_token_ids: list[list[int]], ) -> torch.Tensor: - """ - Applies presence, frequency and repetition penalties to the logits. - """ + """Applies presence, frequency and repetition penalties to the logits.""" _, vocab_size = logits.shape output_tokens_t = _convert_to_tensors(output_token_ids, vocab_size, logits.device) @@ -41,9 +39,7 @@ def apply_all_penalties( def _convert_to_tensors( output_token_ids: list[list[int]], vocab_size: int, device: torch.device ) -> torch.Tensor: - """ - Convert the different list data structures to tensors. - """ + """Convert the different list data structures to tensors.""" output_tokens_tensor = make_tensor_with_pad( output_token_ids, # Use the value of vocab_size as a pad since we don't have a diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index aeb82dec42..06b3fc3423 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -91,8 +91,7 @@ def flashinfer_sampler_supported() -> bool: class TopKTopPSampler(nn.Module): - """ - Module that performs optional top-k and top-p filtering followed by + """Module that performs optional top-k and top-p filtering followed by weighted random sampling of logits. Implementations may update the logits tensor in-place. @@ -154,8 +153,7 @@ class TopKTopPSampler(nn.Module): k: torch.Tensor | None, p: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: - """ - PyTorch-native implementation of top-k and top-p sampling. + """PyTorch-native implementation of top-k and top-p sampling. The logits tensor may be updated in-place. """ @@ -207,8 +205,7 @@ class TopKTopPSampler(nn.Module): k: torch.Tensor | None, p: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: - """ - PyTorch-native implementation of top-k and top-p sampling for CPU. + """PyTorch-native implementation of top-k and top-p sampling for CPU. The logits tensor may be updated in-place. """ @@ -423,8 +420,7 @@ def apply_top_k_top_p_pytorch( def apply_top_k_only(logits: torch.Tensor, k: torch.Tensor) -> torch.Tensor: - """ - Apply top-k mask to the logits. + """Apply top-k mask to the logits. This implementation doesn't involve sorting the entire vocab. Note however that it involves a GPU->CPU sync which can be detrimental for diff --git a/vllm/v1/sample/ops/topk_topp_triton.py b/vllm/v1/sample/ops/topk_topp_triton.py index 695ba624f7..20b636e745 100755 --- a/vllm/v1/sample/ops/topk_topp_triton.py +++ b/vllm/v1/sample/ops/topk_topp_triton.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Combined Top-K and Top-P Triton kernels. +"""Combined Top-K and Top-P Triton kernels. Based on the paper "Qrita: High-performance Top-k and Top-p Algorithm for GPUs using Pivot-based Truncation and Selection" By Park et al. @@ -1617,8 +1616,7 @@ def apply_top_k_top_p_triton( p: torch.Tensor | None, mask_value: float = float("-inf"), ) -> torch.Tensor: - """ - Apply combined top-k and top-p masking using Triton. + """Apply combined top-k and top-p masking using Triton. Top-k is applied first (by logit value), then top-p is applied to the remaining k values (by probability). @@ -1634,6 +1632,7 @@ def apply_top_k_top_p_triton( Returns: The masked logits tensor. It may or may not be modified in-place. + """ assert logits.ndim == 2 assert logits.dtype == torch.float32 diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py index 85b623bd84..d5f01f0fb4 100644 --- a/vllm/v1/sample/rejection_sampler.py +++ b/vllm/v1/sample/rejection_sampler.py @@ -43,8 +43,7 @@ MAX_SPEC_LEN = 128 class RejectionSampler(nn.Module): - """ - The implementation strictly follows the algorithm described in + """The implementation strictly follows the algorithm described in https://arxiv.org/abs/2211.17192. However, we want to clarify the terminology used in the implementation: accepted tokens: tokens that are accepted based on the relationship @@ -113,8 +112,7 @@ class RejectionSampler(nn.Module): logits: torch.Tensor, sampling_metadata: SamplingMetadata, ) -> SamplerOutput: - """ - Args: + """Args: metadata: Metadata for spec decoding. draft_probs (Optional[torch.Tensor]): @@ -130,10 +128,12 @@ class RejectionSampler(nn.Module): sampling_metadata (vllm.v1.sample.metadata.SamplingMetadata): Additional metadata needed for sampling, such as temperature, top-k/top-p parameters, or other relevant information. + Returns: SamplerOutput: Contains the final output token IDs and their logprobs if requested. + """ assert metadata.max_spec_len <= MAX_SPEC_LEN @@ -272,6 +272,7 @@ class RejectionSampler(nn.Module): logprobs_tensors: LogprobsTensors | None = None, ) -> tuple[list[list[int]], LogprobsLists | None]: """Parse the output of the rejection sampler. + Args: output_token_ids: The sampled token IDs in shape [batch_size, max_spec_len + 1]. The rejected tokens are @@ -280,8 +281,10 @@ class RejectionSampler(nn.Module): vocab_size: The size of the vocabulary. discard_req_indices: Optional row indices to discard tokens in. logprobs_tensors: Optional logprobs tensors to filter. + Returns: A list of lists of token IDs. + """ output_token_ids_np = output_token_ids.cpu().numpy() # Create mask for valid tokens. @@ -536,6 +539,7 @@ def apply_sampling_constraints( Returns: torch.Tensor: Processed logits if non-greedy sampling is used, otherwise returns the original logits. + """ assert logits.ndim == 2 assert cu_num_draft_tokens.ndim == 1 @@ -597,8 +601,10 @@ def expand_batch_to_tokens( Value to be replaced if it is found in x. replace_to: int = 0 Value to replace with when replace_from is found. + Returns: expanded_x: [num_tokens] tensor. + """ batch_size = x.shape[0] assert cu_num_tokens.shape[0] == batch_size @@ -619,8 +625,7 @@ def generate_uniform_probs( generators: dict[int, torch.Generator], device: torch.device, ) -> torch.Tensor: - """ - Generates a batch of uniform random samples, with optional seeding + """Generates a batch of uniform random samples, with optional seeding if available. This method creates a tensor of shape `(num_tokens, )` filled @@ -639,10 +644,12 @@ def generate_uniform_probs( `torch.Generator` objects. device: torch.device The device on which to allocate the tensor. + Returns: uniform_rand: torch.Tensor A tensor of shape `(num_tokens, )` containing uniform random values in the range [0, 1). + """ # NOTE(woosuk): We deliberately use float64 instead of float32 here # because when using float32, there's a non-negligible chance that diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py index 3efaf33d98..7e8d597d88 100644 --- a/vllm/v1/sample/sampler.py +++ b/vllm/v1/sample/sampler.py @@ -19,8 +19,7 @@ _SAMPLING_EPS = 1e-5 class Sampler(nn.Module): - """ - A layer that samples the next tokens from the model's outputs + """A layer that samples the next tokens from the model's outputs with the following steps in order: 1. If logprobs are requested: @@ -170,6 +169,7 @@ class Sampler(nn.Module): Returns: LogprobsTensors with logprobs for the specified tokens, or None if no requests have logprob_token_ids. + """ if not logprob_token_ids: return None @@ -252,7 +252,6 @@ class Sampler(nn.Module): The various logits processing functions called in this method may update the logits tensor in-place. """ - logprobs_mode = logprobs_mode_override or self.logprobs_mode assert not (sampling_metadata.all_greedy and sampling_metadata.all_random) if sampling_metadata.all_random: @@ -312,8 +311,7 @@ class Sampler(nn.Module): num_logprobs: int, token_ids: torch.Tensor, ) -> LogprobsTensors: - """ - Gather logprobs for topk and sampled/prompt token. + """Gather logprobs for topk and sampled/prompt token. Args: logprobs: (num tokens) x (vocab) tensor @@ -329,6 +327,7 @@ class Sampler(nn.Module): Top-k int indices tensor, (num tokens) x (num_logprobs + 1) Top-k float logprobs tensor, (num tokens) x (num_logprobs + 1) Sampled token rank tensor, (num tokens) + """ assert token_ids.dtype == torch.int64 # Find the topK values. diff --git a/vllm/v1/serial_utils.py b/vllm/v1/serial_utils.py index bc4619a7eb..882bc44917 100644 --- a/vllm/v1/serial_utils.py +++ b/vllm/v1/serial_utils.py @@ -57,8 +57,7 @@ bytestr: TypeAlias = bytes | bytearray | memoryview | zmq.Frame class OOBTensorConsumer(ABC): @abstractmethod def __call__(self, tensor: torch.Tensor) -> dict | None: - """ - Called with tensors for the current message. + """Called with tensors for the current message. Returns None to reject the tensor (falls back to regular serialization), otherwise a dict with arbitrary placeholder data to be included in the serialized message. @@ -489,8 +488,7 @@ def run_method( args: tuple[Any, ...], kwargs: dict[str, Any], ) -> Any: - """ - Run a method of an object with the given arguments and keyword arguments. + """Run a method of an object with the given arguments and keyword arguments. If the method is string, it will be converted to a method using getattr. If the method is serialized bytes and will be deserialized using cloudpickle. @@ -528,8 +526,7 @@ class PydanticMsgspecMixin: def __get_pydantic_core_schema__( cls, source_type: Any, handler: GetCoreSchemaHandler ) -> core_schema.CoreSchema: - """ - Make msgspec.Struct compatible with Pydantic, respecting defaults. + """Make msgspec.Struct compatible with Pydantic, respecting defaults. Handle JSON=>msgspec.Struct. Used when exposing msgspec.Struct to the API as input or in `/docs`. Note this is cached by Pydantic and not called on every validation. diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index bc7e6c3a13..85dde88f2d 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -321,7 +321,6 @@ class SimpleCPUOffloadScheduler: self, request: "Request", num_computed_tokens: int ) -> tuple[int | None, bool]: """Return (num_new_tokens, is_async) from consecutive CPU cache hits.""" - # Pins found CPU blocks so they survive LRU eviction until # update_state_after_alloc() consumes them. Any pin from an earlier # call on the same request (e.g. retry after a failed allocate_slots) @@ -636,8 +635,8 @@ class SimpleCPUOffloadScheduler: Returns: (gpu_block_ids, cpu_block_ids, req_ids, block_meta) for the store event. ``block_meta`` is None when kv cache events are disabled. - """ + """ merged_gpu_block_ids: list[int] = [] merged_cpu_block_ids: list[int] = [] req_ids: list[str] = [] @@ -1330,7 +1329,6 @@ class SimpleCPUOffloadScheduler: the transfer finished, then release refs without caching abandoned store results. """ - self._abandoned_store_event_to_blocks.update(self._store_event_to_blocks) for transfer in self._pending_finished_stores: self._release_transfer_refs(transfer) diff --git a/vllm/v1/simple_kv_offload/metadata.py b/vllm/v1/simple_kv_offload/metadata.py index 8c8d4511ee..68ced9b89c 100644 --- a/vllm/v1/simple_kv_offload/metadata.py +++ b/vllm/v1/simple_kv_offload/metadata.py @@ -14,8 +14,7 @@ INVALID_JOB_ID = -1 @dataclass class SimpleCPUOffloadMetadata(KVConnectorMetadata): - """ - Metadata passed from scheduler to worker for CPU offload operations. + """Metadata passed from scheduler to worker for CPU offload operations. The worker receives flat block lists keyed by a monotonic event_idx. Job->req_id translation is handled by the scheduler-side manager diff --git a/vllm/v1/simple_kv_offload/worker.py b/vllm/v1/simple_kv_offload/worker.py index fc950af40f..9fc16014c2 100644 --- a/vllm/v1/simple_kv_offload/worker.py +++ b/vllm/v1/simple_kv_offload/worker.py @@ -91,6 +91,7 @@ class SimpleCPUOffloadWorker: tensor (attention layers) or a list of tensors (Mamba layers in hybrid models). All values are included for offloading by resolving to their underlying raw storage. + """ if not kv_caches: logger.warning("No KV caches to offload.") @@ -313,6 +314,7 @@ class SimpleCPUOffloadWorker: tuple of (finished_sending, finished_recving). - finished_sending: always None (stores use worker metadata). - finished_recving: req_ids whose loads have completed. + """ metadata = self._connector_metadata finished_recving: set[str] = set() diff --git a/vllm/v1/spec_decode/dflash.py b/vllm/v1/spec_decode/dflash.py index 284db7dee7..d7d3e3adb0 100644 --- a/vllm/v1/spec_decode/dflash.py +++ b/vllm/v1/spec_decode/dflash.py @@ -219,8 +219,7 @@ class DFlashProposer(SpecDecodeBaseProposer): is_graph_capturing: bool = False, slot_mappings: dict[str, torch.Tensor] | None = None, ) -> None: - """ - Key differences to default dummy_run: + """Key differences to default dummy_run: - Only one forward pass due to parallel drafting - DFlash uses context states as unpadded metadata, so hidden_states will use the unpadded num_tokens instead of num_input_tokens diff --git a/vllm/v1/spec_decode/extract_hidden_states.py b/vllm/v1/spec_decode/extract_hidden_states.py index 6325c1bf5d..18ae2700e9 100644 --- a/vllm/v1/spec_decode/extract_hidden_states.py +++ b/vllm/v1/spec_decode/extract_hidden_states.py @@ -106,6 +106,7 @@ class ExtractHiddenStatesProposer: The main purpose is to cache hidden states, not to speculate. Args: + num_speculative_tokens: Number of draft tokens to return per request sampled_token_ids: Sampled token IDs from the target model target_hidden_states: List of hidden state tensors from target model (one per aux hidden state layer) @@ -117,6 +118,7 @@ class ExtractHiddenStatesProposer: Tuple of: - Draft tokens matching sampled tokens, shape [batch_size, 1] - KV connector output (if KV transfer is active), else None + """ assert num_speculative_tokens == self.num_speculative_tokens assert self.model is not None and isinstance(target_hidden_states, list) @@ -318,14 +320,12 @@ class ExtractHiddenStatesProposer: gpu_input_batch: InputBatch, discard_request_mask: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Prepare next token IDs for speculative decoding. + """Prepare next token IDs for speculative decoding. Since num_speculative_tokens == 1, sampled_token_ids has shape (batch_size, 1). For each request we either use the sampled token (if valid and not discarded) or a backup token from the request state. """ - # Precompute backup token IDs for discarded requests. num_reqs = gpu_input_batch.num_reqs for i in range(num_reqs): @@ -359,6 +359,7 @@ class ExtractHiddenStatesProposer: Args: target_model: The target model (passed for compatibility with EagleProposer interface, but not used here) + """ # Get the target model's attention layers before loading draft model target_attn_layer_names = set( diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 1bd1bdc45b..9f7ad68a88 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -1024,8 +1024,7 @@ class SpecDecodeBaseProposer: gpu_input_batch: InputBatch, num_scheduled_tokens: dict[str, int], ) -> torch.Tensor: - """ - This function is used to prepare the inputs for speculative decoding. + """This function is used to prepare the inputs for speculative decoding. It calculates the next token ids for each request based on the sampled token ids from the CPU. If a request has no sampled token ids (e.g., during the initial decoding steps), it falls back to using the request @@ -1057,8 +1056,7 @@ class SpecDecodeBaseProposer: gpu_input_batch: InputBatch, discard_request_mask: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - This function is used to prepare the inputs for speculative decoding. + """This function is used to prepare the inputs for speculative decoding. It calculates the next token ids and the number of valid sampled tokens for each request, considering the "discarded" requests whose next token is not sampled and comes from `request.get_token_id()` instead. This is denoted @@ -1107,8 +1105,7 @@ class SpecDecodeBaseProposer: spec_decode_metadata: SpecDecodeMetadata, valid_sampled_tokens_count: torch.Tensor, ) -> tuple[CommonAttentionMetadata, torch.Tensor, torch.Tensor]: - """ - This function is used to prepare the inputs for speculative decoding + """This function is used to prepare the inputs for speculative decoding It updates the common_attn_metadata for speculative decoding, but does not consider the rejected tokens. Instead, all tokens are included as inputs to the speculator, with the rejected tokens @@ -1166,8 +1163,7 @@ class SpecDecodeBaseProposer: sampled_token_ids: list[list[int]], num_draft_tokens: list[int], ) -> tuple[CommonAttentionMetadata, torch.Tensor]: - """ - This function is used to prepare the inputs for speculative decoding. + """This function is used to prepare the inputs for speculative decoding. It updates to the common_attn_metadata to account for the rejected tokens (and newly sampled tokens). It also returns the token indices of the tokens that should be fed to the speculator. @@ -1309,8 +1305,7 @@ class SpecDecodeBaseProposer: return base def _get_model(self) -> nn.Module: - """ - Default method to call get_model(). Can be overridden by subclasses which + """Default method to call get_model(). Can be overridden by subclasses which need to customize model loading. """ from vllm.compilation.backends import set_model_tag @@ -1422,8 +1417,7 @@ class SpecDecodeBaseProposer: self.parallel_drafting_hidden_state_tensor.copy_(flat_mask) def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None: - """ - Some draft models may not have their own embedding layers, and some may + """Some draft models may not have their own embedding layers, and some may have a duplicate copy of the target model's embedding layers. In these cases, we share the target model's embedding layers with the draft model to save memory. @@ -1515,8 +1509,7 @@ class SpecDecodeBaseProposer: ) def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None: - """ - Some draft models may not have their own LM head, and some may have a + """Some draft models may not have their own LM head, and some may have a duplicate copy of the target model's LM head. In these cases, we share the target model's LM head with the draft model to save memory. """ @@ -1680,8 +1673,7 @@ class SpecDecodeBaseProposer: self.model(**kwargs) def _get_eagle3_use_aux_hidden_state_from_config(self) -> bool: - """ - Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary + """Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary hidden states and directly uses the last layer output just like eagle1. They might indicate this by setting "use_aux_hidden_state" to False inside the "eagle_config" dict of their hf_config. @@ -1696,8 +1688,7 @@ class SpecDecodeBaseProposer: return use_aux_hidden_state def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None: - """ - Validate that all drafting layers belong to the same KVCacheGroup. + """Validate that all drafting layers belong to the same KVCacheGroup. Need this assumption to ensure all drafting layers can use the same AttentionMetadata. May extend to multiple AttentionMetadata in the future. @@ -1723,8 +1714,7 @@ class SpecDecodeBaseProposer: kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int] | None = None, ) -> None: - """ - Initialize AttentionGroups for draft layers using kv_cache_config. + """Initialize AttentionGroups for draft layers using kv_cache_config. Called from the model runner's initialize_metadata_builders. """ all_attn_layers = get_layers_from_vllm_config( diff --git a/vllm/v1/spec_decode/medusa.py b/vllm/v1/spec_decode/medusa.py index 7adf7cff5f..7c926ba837 100644 --- a/vllm/v1/spec_decode/medusa.py +++ b/vllm/v1/spec_decode/medusa.py @@ -16,9 +16,7 @@ logger = init_logger(__name__) class MedusaProposer: - """ - Medusa proposer class for generating token sequences - """ + """Medusa proposer class for generating token sequences.""" def __init__( self, diff --git a/vllm/v1/spec_decode/ngram_proposer.py b/vllm/v1/spec_decode/ngram_proposer.py index e0240d0e66..5e0d2cfc3c 100644 --- a/vllm/v1/spec_decode/ngram_proposer.py +++ b/vllm/v1/spec_decode/ngram_proposer.py @@ -72,6 +72,8 @@ class NgramProposer: """Batch version of ngram proposer using numba for acceleration. Args: + num_requests: + Number of requests in the batch. valid_ngram_requests: Set of indices of requests that need ngram proposals. num_tokens_no_spec: @@ -87,6 +89,7 @@ class NgramProposer: list[list[int]]: A list where each element is a list of proposed token IDs for the corresponding request. + """ draft_token_ids: list[list[int]] = [] @@ -211,8 +214,7 @@ def _find_longest_matched_ngram_and_propose_tokens( max_model_len: int, k: int, ) -> np.ndarray: - """ - Find the longest n-gram which matches the suffix of the given tokens + """Find the longest n-gram which matches the suffix of the given tokens whose length is within [min_ngram, max_ngram] (inclusive). If found, we will extract k right after the matched ngram. diff --git a/vllm/v1/spec_decode/ngram_proposer_gpu.py b/vllm/v1/spec_decode/ngram_proposer_gpu.py index 2de5c3be00..061a8f5ea9 100644 --- a/vllm/v1/spec_decode/ngram_proposer_gpu.py +++ b/vllm/v1/spec_decode/ngram_proposer_gpu.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GPU-accelerated N-gram proposer using fully async PyTorch tensor operations. +"""GPU-accelerated N-gram proposer using fully async PyTorch tensor operations. This version uses a fully vectorized approach with unfold and argmax for finding the first match across all sequences in parallel. @@ -52,8 +51,7 @@ class NgramGPUKernel(nn.Module): max_ngram_len: int, num_draft_tokens: int, ) -> torch.Tensor: - """ - Find suffix n-gram matches and extract following tokens. + """Find suffix n-gram matches and extract following tokens. Searches for the earliest prior occurrence of the trailing n-gram, tries multiple lengths, and picks the longest valid match. @@ -66,6 +64,7 @@ class NgramGPUKernel(nn.Module): Returns: Draft token predictions; -1 means invalid/no match. + """ batch_size = token_ids.shape[0] max_seq_len = token_ids.shape[1] @@ -165,8 +164,7 @@ class NgramGPUKernel(nn.Module): token_ids_gpu: torch.Tensor, combined_mask: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass for N-gram proposal using GPU tensor operations. + """Forward pass for N-gram proposal using GPU tensor operations. Args: num_tokens_no_spec: Number of tokens for each sequence [batch_size] @@ -177,8 +175,8 @@ class NgramGPUKernel(nn.Module): draft_tokens: [batch_size, k] on GPU num_valid_draft_tokens: [batch_size] int32 on GPU, count of leading valid (non -1) tokens per request. - """ + """ device = token_ids_gpu.device # Infer batch size to preserve dynamic shape. @@ -282,8 +280,7 @@ class NgramProposerGPU: pattern_len: int, device: str = "cuda", ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Generate random test data with n-gram repetitions. + """Generate random test data with n-gram repetitions. Args: batch_size: Number of sequences in the batch @@ -296,6 +293,7 @@ class NgramProposerGPU: num_tokens: [batch_size] tensor sampled_flags: [batch_size] bool tensor valid_mask: [batch_size] bool tensor + """ token_ids = torch.zeros( batch_size, @@ -321,8 +319,7 @@ class NgramProposerGPU: valid_sampled_token_ids_gpu: torch.Tensor, # [batch_size, num_spec_tokens + 1] valid_sampled_tokens_count: torch.Tensor, # [batch_size] ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Propose draft tokens using GPU-accelerated n-gram matching. + """Propose draft tokens using GPU-accelerated n-gram matching. Scatter sampled tokens into `token_ids_gpu`, compute temporary updated lengths, then run the kernel. @@ -338,6 +335,7 @@ class NgramProposerGPU: draft_tokens: Proposed draft token IDs [batch_size, k] num_valid_draft_tokens: Count of leading valid draft tokens per request [batch_size] + """ assert num_speculative_tokens == self.k assert token_ids_gpu.device == self.device @@ -398,8 +396,7 @@ class NgramProposerGPU: num_tokens_no_spec: torch.Tensor, discard_request_mask: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Prepare speculative decoding inputs on device: + """Prepare speculative decoding inputs on device: compute next token ids and valid counts, honoring discarded requests and rejected tokens, without CPU-GPU sync. """ @@ -485,6 +482,7 @@ def update_scheduler_for_invalid_drafts( num_valid_draft_tokens_cpu: CPU buffer of valid draft counts. scheduler_output: Scheduler metadata to update in-place. req_id_to_index: Request-id to batch-index mapping. + """ req_data = scheduler_output.scheduled_cached_reqs num_valid_draft_tokens_event.synchronize() @@ -652,9 +650,7 @@ def copy_num_valid_draft_tokens( num_valid_draft_tokens: torch.Tensor | None, batch_size: int, ) -> None: - """ - Async D2H copy of per-request valid draft counts. - """ + """Async D2H copy of per-request valid draft counts.""" if num_valid_draft_tokens is None: return diff --git a/vllm/v1/spec_decode/step3p5.py b/vllm/v1/spec_decode/step3p5.py index 109d326356..7a73473adc 100644 --- a/vllm/v1/spec_decode/step3p5.py +++ b/vllm/v1/spec_decode/step3p5.py @@ -155,7 +155,6 @@ class Step3p5MTPProposer(EagleProposer): def _maybe_share_lm_head(self, target_language_model: torch.nn.Module) -> None: """Step3.5 MTP uses the lm_head stored in each MTP layer.""" - # The base MTP path shares target lm_head into shared_head.head. # Step3.5 checkpoints carry per-MTP-layer shared_head weights. return diff --git a/vllm/v1/spec_decode/suffix_decoding.py b/vllm/v1/spec_decode/suffix_decoding.py index 66137a0063..5d100ce9ed 100644 --- a/vllm/v1/spec_decode/suffix_decoding.py +++ b/vllm/v1/spec_decode/suffix_decoding.py @@ -7,8 +7,7 @@ from vllm.v1.worker.gpu_input_batch import InputBatch class SuffixDecodingProposer: - """ - Speculative decoding proposer for Suffix Decoding (https://arxiv.org/pdf/2411.04975). + """Speculative decoding proposer for Suffix Decoding (https://arxiv.org/pdf/2411.04975). This class imports and uses the official implementation from Arctic Inference (https://github.com/snowflakedb/ArcticInference). """ diff --git a/vllm/v1/spec_decode/utils.py b/vllm/v1/spec_decode/utils.py index fbfe443842..471d51d355 100644 --- a/vllm/v1/spec_decode/utils.py +++ b/vllm/v1/spec_decode/utils.py @@ -50,8 +50,7 @@ def eagle_step_slot_mapping_metadata_kernel( PAD_ID: tl.constexpr, batch_size, ): - """ - Fused kernel for EAGLE autoregressive step: updates positions, slot mapping, + """Fused kernel for EAGLE autoregressive step: updates positions, slot mapping, and sequence lengths in a single kernel to reduce launch overhead. Launched with input_batch_size threads. Threads with req_idx >= batch_size @@ -149,8 +148,7 @@ def eagle_step_update_slot_mapping_and_metadata( out_slot_mapping: torch.Tensor, input_batch_size: int | None = None, ) -> None: - """ - Fused update of slot mapping and metadata for one EAGLE autoregressive step. + """Fused update of slot mapping and metadata for one EAGLE autoregressive step. Updates seq_lens in place. Writes to out_clamped_positions and out_slot_mapping. When input_batch_size > batch_size, threads beyond batch_size write @@ -166,6 +164,7 @@ def eagle_step_update_slot_mapping_and_metadata( out_slot_mapping: [input_batch_size] output buffer for slot mapping input_batch_size: total batch size including cudagraph padding; defaults to batch_size (no padding) + """ batch_size = positions_1d.shape[0] if input_batch_size is None: @@ -197,8 +196,7 @@ def eagle_prepare_inputs_padded_kernel( num_rejected_tokens_gpu_ptr, # [num_reqs] (output) num_reqs, # tl.int32 ): - """ - Fused kernel for Eagle prepare_input_padded. This kernel computes the + """Fused kernel for Eagle prepare_input_padded. This kernel computes the token index to sample for each request, taking into account the number of draft tokens and the number of valid sampled tokens (which is one more than the number of accepted tokens). @@ -270,8 +268,7 @@ def eagle_prepare_next_token_padded_kernel( stride_sampled_token_ids, # tl.int32 (stride for dim 0) BLOCK_SIZE_TOKENS: tl.constexpr, # Power-of-2 >= num_sampled_tokens_per_req ): - """ - Fused kernel for Eagle prepare_next_token_ids_padded. This kernel computes the + """Fused kernel for Eagle prepare_next_token_ids_padded. This kernel computes the number of valid (1 + accepted) tokens for each request, and the corresponding "next" token id to sample from during speculative decoding. This is the "last accepted token" from the sampled tokens, or the backup token if no @@ -401,8 +398,7 @@ def extend_all_queries_by_N( arange: torch.Tensor, new_slot_mapping: torch.Tensor, ) -> CommonAttentionMetadata: - """ - Creates a new CommonAttentionMetadata with all query lengths increased by N. + """Creates a new CommonAttentionMetadata with all query lengths increased by N. Also all seq lens are increased by N. This is useful e.g. in speculative decoding with parallel drafting, where we extend each sequence by N tokens and predict all tokens in one pass. @@ -453,8 +449,7 @@ def copy_and_expand_eagle_inputs_kernel( shift_input_ids, # tl.bool BLOCK_SIZE_TOKENS: tl.constexpr, # Blocks along token dim to handle prefills ): - """ - Copy and expand inputs from the target model to the drafting buffers for Eagle + """Copy and expand inputs from the target model to the drafting buffers for Eagle speculative decoding. This kernel handles padding slots and parallel drafting tokens, if enabled. """ @@ -668,8 +663,7 @@ def copy_and_expand_dflash_inputs_kernel( BLOCK_SIZE: tl.constexpr, HAS_NUM_REJECTED: tl.constexpr = False, ): - """ - Fused kernel for DFlash first-pass input setup. + """Fused kernel for DFlash first-pass input setup. Per request, this kernel: 1. Copies context positions from target_positions to diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 0d8b7377c5..5f43c5a890 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -161,7 +161,6 @@ class GuidanceGrammar(StructuredOutputGrammar): Returns True if the parser was advanced successfully. Returns False if the parser failed to advance. """ - if self.ll_tokenizer.eos_token in tokens: if self.ll_matcher.is_stopped() and not self.terminated: self.rollback_lag = 1 diff --git a/vllm/v1/structured_output/backend_outlines.py b/vllm/v1/structured_output/backend_outlines.py index dd460f96ec..bb49fb189b 100644 --- a/vllm/v1/structured_output/backend_outlines.py +++ b/vllm/v1/structured_output/backend_outlines.py @@ -283,7 +283,7 @@ def _prefix_needs_context(parsed) -> bool: def _check_unsupported(parsed) -> None: - """Check for regex features unsupported by regex-automata""" + """Check for regex features unsupported by regex-automata.""" tokens = parsed.data if hasattr(parsed, "data") else parsed for ttype, tval in tokens: # backreference @@ -310,8 +310,7 @@ def _check_unsupported(parsed) -> None: def validate_regex_is_buildable(pattern: str) -> None: - """ - Validates that the input regex is not using unsupported features + """Validates that the input regex is not using unsupported features of the `regex-automata` crate (outlines_core regex engine) and has a universal start state. definition of universal start state used can be found at: diff --git a/vllm/v1/structured_output/backend_types.py b/vllm/v1/structured_output/backend_types.py index d9826c18f5..b3695efd00 100644 --- a/vllm/v1/structured_output/backend_types.py +++ b/vllm/v1/structured_output/backend_types.py @@ -33,8 +33,7 @@ class StructuredOutputGrammar(ABC): @abstractmethod def accept_tokens(self, request_id: str, tokens: list[int]) -> bool: - """ - Determines whether the provided tokens are accepted for the + """Determines whether the provided tokens are accepted for the given request. Args: @@ -43,12 +42,12 @@ class StructuredOutputGrammar(ABC): Returns: bool: True if the tokens are accepted, False otherwise. + """ @abstractmethod def validate_tokens(self, tokens: list[int]) -> list[int]: - """ - Validates the provided tokens against the grammar. + """Validates the provided tokens against the grammar. Will not advance the FSM. Args: @@ -57,42 +56,41 @@ class StructuredOutputGrammar(ABC): Returns: list[int]: A list of accepted token IDs. Will be a prefix of the input tokens, and empty if none are accepted. + """ @abstractmethod def rollback(self, num_tokens: int) -> None: - """ - Rolls back the state of the grammar by a specified number of tokens. + """Rolls back the state of the grammar by a specified number of tokens. Will also revert counters for the number of processed tokens. Args: num_tokens (int): The number of tokens to roll back. + """ @abstractmethod def fill_bitmask(self, bitmask: "torch.Tensor", batch_index: int) -> None: - """ - Fills the bitmask for a specific batch index. + """Fills the bitmask for a specific batch index. Args: bitmask (torch.Tensor): The bitmask to fill batch_index (int): The index in the bitmask to fill + """ @abstractmethod def is_terminated(self) -> bool: - """ - Checks whether the structured output process has terminated. + """Checks whether the structured output process has terminated. Returns: bool: True if the process is terminated, False otherwise. + """ @abstractmethod def reset(self): - """ - Resets the state of the structured output grammar. - """ + """Resets the state of the structured output grammar.""" @dataclass @@ -110,8 +108,7 @@ class StructuredOutputBackend(ABC): grammar_spec: str, stop_token_ids: set[int] | None = None, ) -> StructuredOutputGrammar: - """ - Compiles a grammar specification into a structured output grammar. + """Compiles a grammar specification into a structured output grammar. Args: request_type (StructuredOutputOptions): The type of structured @@ -123,20 +120,19 @@ class StructuredOutputBackend(ABC): Returns: StructuredOutputGrammar: The compiled structured output grammar. + """ @abstractmethod def allocate_token_bitmask(self, max_num_seqs: int) -> "torch.Tensor": - """ - Allocates a token bitmask for the specified maximum number of sequences. + """Allocates a token bitmask for the specified maximum number of sequences. Args: max_num_seqs (int): The maximum number of sequences for which to allocate the bitmask. + """ @abstractmethod def destroy(self): - """ - Backend-specific cleanup. - """ + """Backend-specific cleanup.""" diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 92122f85d0..4969789434 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -74,6 +74,7 @@ def compile_regex_with_timeout(fn: Callable[[str], _T], pattern: str) -> _T: Raises: ValueError: If compilation exceeds the configured timeout. + """ timeout = envs.VLLM_REGEX_COMPILATION_TIMEOUT_S if timeout <= 0: @@ -103,13 +104,14 @@ def apply_grammar_bitmask( input_batch: InputBatch, logits: torch.Tensor, ) -> None: - """ - Apply grammar bitmask to output logits of the model with xgrammar function. + """Apply grammar bitmask to output logits of the model with xgrammar function. Args: scheduler_output (SchedulerOutput): The result of engine scheduling. + grammar_output (GrammarOutput): The grammar bitmask to apply. input_batch (InputBatch): The input of model runner. logits (torch.Tensor): The output logits of model forward. + """ # Serialization of np.ndarray is much more efficient than a tensor, # so we receive it in that format. @@ -190,8 +192,7 @@ def apply_grammar_bitmask( class OutlinesVocabulary: - """ - Wrapper class for `outlines_core.Vocabulary`, + """Wrapper class for `outlines_core.Vocabulary`, which allows us to store a hash with the vocabulary """ @@ -206,7 +207,7 @@ class OutlinesVocabulary: def get_outlines_cache_path() -> str: - """Get the context object that contains previously-computed return values""" + """Get the context object that contains previously-computed return values.""" outlines_cache_dir = os.getenv("OUTLINES_CACHE_DIR") xdg_cache_home = os.getenv("XDG_CACHE_HOME") home_dir = os.path.expanduser("~") @@ -294,8 +295,7 @@ class OutlinesDiskCache: def get_outlines_cache(): - """Get the Cache instance to be used for index caching""" - + """Get the Cache instance to be used for index caching.""" cache_dir = get_outlines_cache_path() if envs.VLLM_V1_USE_OUTLINES_CACHE: logger.warning( @@ -324,6 +324,7 @@ def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]: Returns: A Dict of token string -> equivalent token ids + """ eos_token_id = tokenizer.eos_token_id @@ -403,8 +404,7 @@ def get_outlines_vocabulary(tokenizer: TokenizerLike) -> oc.Vocabulary: def grammar_is_likely_lark(grammar_str: str) -> bool: - """ - Check if grammar appears to use Lark syntax. + """Check if grammar appears to use Lark syntax. Args: grammar_str: Input grammar string @@ -417,6 +417,7 @@ def grammar_is_likely_lark(grammar_str: str) -> bool: True >>> grammar_is_likely_lark("rule ::= 'abc'") False + """ if not grammar_str or not isinstance(grammar_str, str): return False @@ -435,8 +436,7 @@ def grammar_is_likely_lark(grammar_str: str) -> bool: def convert_lark_to_ebnf(grammar_str: str) -> str: - """ - Convert a Lark grammar string to EBNF format. + """Convert a Lark grammar string to EBNF format. EBNF reference: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md @@ -453,6 +453,7 @@ def convert_lark_to_ebnf(grammar_str: str) -> str: >>> print(convert_lark_to_ebnf("rule: 'hello'")) root ::= rule rule ::= "hello" + """ if not isinstance(grammar_str, str): raise ValueError(f"Grammar must be a string, got {type(grammar_str)}") diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index facafd2860..a89ccbb9ed 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -201,6 +201,7 @@ class APIServerProcessManager: output_addresses: Output addresses for each API server stats_update_address: Optional stats update address tensor_queue: Optional tensor IPC queue for sharing MM tensors + """ self.listen_address = listen_address self.sock = sock @@ -519,7 +520,6 @@ def run_api_server_worker_proc( listen_address, sock, args, client_config=None, **uvicorn_kwargs ) -> None: """Entrypoint for individual API server worker processes.""" - from vllm.entrypoints.launchers.api_server.entry import run_server_worker client_config = client_config or {} @@ -550,8 +550,8 @@ def wait_for_completion_or_failure( If CoreEngineProcManager, it manages local engines; if CoreEngineActorManager, it manages all engines. coordinator: The coordinator for data parallel. - """ + """ try: logger.info("Waiting for API servers to complete ...") # Create a mapping of sentinels to their corresponding processes @@ -613,6 +613,7 @@ def shutdown(procs: list[BaseProcess], timeout: float | None = None) -> None: Args: procs: List of processes to shutdown timeout: Maximum time in seconds to wait for graceful shutdown + """ if timeout is None: # Keep a small grace period for best-effort cleanup paths that do not @@ -667,8 +668,7 @@ def shutdown(procs: list[BaseProcess], timeout: float | None = None) -> None: def copy_slice( from_tensor: torch.Tensor, to_tensor: torch.Tensor, length: int ) -> torch.Tensor: - """ - Copy the first length elements of a tensor into another tensor in a + """Copy the first length elements of a tensor into another tensor in a non-blocking manner. Used to copy pinned CPU tensor data to pre-allocated GPU tensors. @@ -682,7 +682,6 @@ def report_usage_stats( vllm_config, usage_context: UsageContext = UsageContext.ENGINE_CONTEXT ) -> None: """Report usage statistics if enabled.""" - if not is_usage_stats_enabled(): return @@ -803,6 +802,7 @@ def tensor_data(tensor: torch.Tensor) -> memoryview: Returns: A memoryview of the tensor data as uint8. + """ return tensor.flatten().cpu().contiguous().view(torch.uint8).numpy().data @@ -826,8 +826,7 @@ class IterationDetails: def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDetails: - """ - Compute the number of context/generation requests and tokens + """Compute the number of context/generation requests and tokens for the current iteration's scheduler output. A requests is regarded as a context request if its output tokens are still 0, an extended chunk of chunked prefill falls into this category. @@ -838,6 +837,7 @@ def compute_iteration_details(scheduler_output: SchedulerOutput) -> IterationDet Returns: An IterationDetails object containing the number of context/generation requests and tokens. + """ num_context_requests = 0 num_context_tokens = 0 diff --git a/vllm/v1/worker/block_table.py b/vllm/v1/worker/block_table.py index 33b52652a1..c63908dad0 100644 --- a/vllm/v1/worker/block_table.py +++ b/vllm/v1/worker/block_table.py @@ -67,20 +67,20 @@ class BlockTable: cp_kv_cache_interleave_size: int, slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT, ): - """ - Args: - block_size: Block size used for KV cache memory allocation - max_num_reqs: Maximum number of concurrent requests supported. - max_num_blocks_per_req: Maximum number of blocks per request. - max_num_batched_tokens: Maximum number of tokens in a batch. - pin_memory: Whether to pin memory for faster GPU transfers. - device: Target device for the block table. - kernel_block_size: The block_size of underlying attention kernel. - Will be the same as `block_size` if `block_size` is supported - by the attention kernel. - slot_mapping_mode: How this cache group maps scheduled tokens to - cache slots. Mamba-like state caches do not use token slot - mappings and should use SlotMappingMode.NONE. + """Args: + block_size: Block size used for KV cache memory allocation + max_num_reqs: Maximum number of concurrent requests supported. + max_num_blocks_per_req: Maximum number of blocks per request. + max_num_batched_tokens: Maximum number of tokens in a batch. + pin_memory: Whether to pin memory for faster GPU transfers. + device: Target device for the block table. + kernel_block_size: The block_size of underlying attention kernel. + Will be the same as `block_size` if `block_size` is supported + by the attention kernel. + slot_mapping_mode: How this cache group maps scheduled tokens to + cache slots. Mamba-like state caches do not use token slot + mappings and should use SlotMappingMode.NONE. + """ self.max_num_reqs = max_num_reqs self.max_num_batched_tokens = max_num_batched_tokens @@ -254,6 +254,7 @@ class BlockTable: # kv_manager_block_id 0 → kernel block id [0, 1] # kv_manager_block_id 1 → kernel block id [2, 3] # kv_manager_block_id 2 → kernel block id [4, 5] + """ if blocks_per_kv_block == 1: return kv_manager_block_ids diff --git a/vllm/v1/worker/dp_utils.py b/vllm/v1/worker/dp_utils.py index fe82ec1427..b2f2fe1506 100644 --- a/vllm/v1/worker/dp_utils.py +++ b/vllm/v1/worker/dp_utils.py @@ -112,8 +112,7 @@ def _post_process_dp_padding(tensor: torch.Tensor, should_dp_pad: bool) -> torch def _post_process_cudagraph_mode(tensor: torch.Tensor) -> int: - """ - Synchronize cudagraph_mode across DP ranks by taking the minimum. + """Synchronize cudagraph_mode across DP ranks by taking the minimum. If any rank has NONE (0), all ranks use NONE. This ensures all ranks send consistent values (all padded or all unpadded). """ @@ -127,8 +126,7 @@ def _synchronize_dp_ranks( cudagraph_mode: int, parallel_config: ParallelConfig, ) -> tuple[bool, torch.Tensor | None, int]: - """ - 1. Decides if each DP rank is going to microbatch. Either all ranks + """1. Decides if each DP rank is going to microbatch. Either all ranks run with microbatching or none of them do. 2. Determines the total number of tokens that each rank will run. @@ -197,8 +195,7 @@ def coordinate_batch_across_dp( uniform_decode: bool | None = None, cudagraph_mode: int = 0, ) -> tuple[bool, torch.Tensor | None, int]: - """ - Coordinates amongst all DP ranks to determine if and how the full batch + """Coordinates amongst all DP ranks to determine if and how the full batch should be split into microbatches. Args: diff --git a/vllm/v1/worker/ec_connector_model_runner_mixin.py b/vllm/v1/worker/ec_connector_model_runner_mixin.py index be676666e8..8a32f99660 100644 --- a/vllm/v1/worker/ec_connector_model_runner_mixin.py +++ b/vllm/v1/worker/ec_connector_model_runner_mixin.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Define EC connector functionality mixin for model runners. -""" +"""Define EC connector functionality mixin for model runners.""" from collections.abc import Generator from contextlib import AbstractContextManager, contextmanager, nullcontext diff --git a/vllm/v1/worker/encoder_cudagraph.py b/vllm/v1/worker/encoder_cudagraph.py index a60e0d1892..3b587ad71f 100644 --- a/vllm/v1/worker/encoder_cudagraph.py +++ b/vllm/v1/worker/encoder_cudagraph.py @@ -351,6 +351,7 @@ class EncoderCudaGraphManager: Returns: Token budget if found, None if no fitting budget. + """ budgets = budgets if budgets is not None else self.token_budgets for budget in budgets: @@ -411,8 +412,11 @@ class EncoderCudaGraphManager: mm_kwargs: Multimodal inputs for the batch. token_budget: Token budget to use. path: Configured encoder path. + axis_keys: Resolved capture-axis keys for this batch. + Returns: Encoder outputs, or None if graph not captured. + """ graph_set = self._get_graph_set(path) num_items = len(self._get_item_specs(mm_kwargs)) @@ -562,6 +566,7 @@ class EncoderCudaGraphManager: images_per_rank: Number of items per rank. max_output_tokens_per_rank: Max output tokens across all ranks (for padding during all_gather). + """ tp_size = get_tensor_model_parallel_world_size() current_rank = get_tensor_model_parallel_rank() @@ -687,6 +692,7 @@ class EncoderCudaGraphManager: Returns: List of encoder outputs (one per item). + """ if self.use_dp: per_item_out_tokens = self._get_per_item_out_tokens(mm_kwargs) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 9657c28fe6..4690c7eae7 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -408,6 +408,8 @@ class CudaGraphManager: it is invoked once with warmup=True and again with warmup=False because attention backends may mutate or lazily initialize metadata during warmup. + progress_bar_desc: Description shown on the capture progress bar. + """ with graph_capture(device=self.device), ExitStack() as stack: if self.ubatch_runner is not None: @@ -503,7 +505,6 @@ class CudaGraphManager: num_ubatches: int = 1, ) -> BatchExecutionDescriptor: """Find matching cudagraph descriptor from priority-ordered candidates.""" - effective_loras = self._resolve_effective_loras(num_active_loras) key = (num_tokens, effective_loras) if self._graphs_captured and num_tokens > 0 and key in self._candidates: diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index 80309bc009..8eabc3adbf 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -55,8 +55,7 @@ def sync_cudagraph_and_dp_padding( allow_ubatching: bool = False, uniform_decode: bool = False, ) -> tuple[BatchExecutionDescriptor, DPSyncState | None]: - """ - Coordinates the batch descriptor and DP padding across all ranks. + """Coordinates the batch descriptor and DP padding across all ranks. `parallel_config` is only needed to decide whether to microbatch, so callers that never do (`allow_ubatching=False`) can leave it out. @@ -248,6 +247,12 @@ def dispatch_cg_and_sync_dp( need_eager: Force `CUDAGraphMode.NONE` instead of dispatching. num_active_loras: Active LoRA count for this rank. Does not need cross-rank agreement; it never changes a bucket's token count. + parallel_config: Only needed to decide whether to microbatch, so + callers that never do (`allow_ubatching=False`) can leave it out. + allow_ubatching: Whether this rank's batch may be split into + microbatches. Agreed across ranks before it takes effect. + uniform_decode: Whether this rank's batch is a uniform decode, used + to pick the microbatching split. dp_sync: Agreement from a prior dispatch over this same batch, to reuse. Must come from a batch with this same padded `num_tokens` and the same `uniform_token_count`; `num_reqs` may differ, as neither @@ -257,6 +262,7 @@ def dispatch_cg_and_sync_dp( Returns: (batch_desc, sync), where `sync` is this batch's agreement for a later dispatch to reuse. It is None when `dp_size` is 1 or no rank has work. + """ reuse_eager = dp_sync is not None and dp_sync.eager diff --git a/vllm/v1/worker/gpu/lora_utils.py b/vllm/v1/worker/gpu/lora_utils.py index 6a00b35e4b..eb81a83662 100644 --- a/vllm/v1/worker/gpu/lora_utils.py +++ b/vllm/v1/worker/gpu/lora_utils.py @@ -21,8 +21,7 @@ def get_lora_capture_cases( lora_config: "LoRAConfig | None", compilation_config: "CompilationConfig", ) -> list[int]: - """ - Return num_active_loras values for cudagraph capture. + """Return num_active_loras values for cudagraph capture. When cudagraph_specialize_lora=True: powers of 2 up to max_loras, plus max_loras+1. When False: [0, max_loras+1]. When LoRA disabled: [0]. diff --git a/vllm/v1/worker/gpu/mm/encoder_cache.py b/vllm/v1/worker/gpu/mm/encoder_cache.py index 21d9218182..b9aba7b31a 100644 --- a/vllm/v1/worker/gpu/mm/encoder_cache.py +++ b/vllm/v1/worker/gpu/mm/encoder_cache.py @@ -24,8 +24,7 @@ class EncoderCache: self.mm_features.pop(req_id, None) def reset_mm_cache(self) -> None: - """ - Clear the multi-modal cache that was used during profiling, + """Clear the multi-modal cache that was used during profiling, but no longer needed during inference. """ # NOTE: v2 encoder cache profiling skips the multi-modal cache diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 82cc697e73..97571df8a9 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -NOTE: Coding style guide for this file: +"""NOTE: Coding style guide for this file: This model runner is shared by all models: text and multimodal, generative and embedding, public and private. As a result, this file must only contain code that is common to every model. Model-specific behavior belongs in the diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index ab1b991d1a..c2aaa1fba2 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -39,7 +39,6 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None: that produce a sampled token this step, and therefore must have that token (and the draft block proposed from it) propagated to the earlier PP stages. Returns None if no request in the batch produces a sample.""" - old_computed = input_batch.num_computed_tokens_np prefill_len = input_batch.prefill_len_np # Exclude non-final prefill chunks (they don't produce a sample). diff --git a/vllm/v1/worker/gpu/sample/watermark.py b/vllm/v1/worker/gpu/sample/watermark.py index 0a9c64cbf7..b70d8ae271 100644 --- a/vllm/v1/worker/gpu/sample/watermark.py +++ b/vllm/v1/worker/gpu/sample/watermark.py @@ -163,6 +163,7 @@ def repeated_context_mask( include_prompt: Search the prompt as well as the generated tokens. skip_partial_context: Mark contexts containing start padding so they use ordinary sampling. + """ if max_history is not None and max_history < 1: raise ValueError("max_history must be positive or None") diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 245f304ac4..7e829cf309 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -88,9 +88,14 @@ def build_cost_tables_from_curves( """Build cost tables: graph-padded below the capture limit, smooth above. Args: + draft_curve: (size, cost) samples for the draft model. + verify_curve: (size, cost) samples for the verify model. + max_num_reqs: Largest request count to build a table for. + max_batch_tokens: Largest token count to build a table for. cudagraph_limit: Largest cudagraph-captured size. At or below it, execution pads up to the next captured size, so cost is a step function. Above it there is no padding, so cost is continuous. + """ def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray: diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 3d243b6ee5..f166d63e8e 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -73,8 +73,7 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): @property def advance_draft_positions(self) -> bool: - """ - Whether to increment positions and seq_lens between draft steps. + """Whether to increment positions and seq_lens between draft steps. True for Eagle/standard MTP (each step produces new KV). False for Gemma4 MTP (Q-only, shares target KV, constant positions). diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py index b93a79e455..eb0f4793bf 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -13,7 +13,6 @@ from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool: """Share when the draft has no own copy, or its copy matches the target.""" - if not getattr(eagle, flag, False) or draft is None: return True if target is None: diff --git a/vllm/v1/worker/gpu_input_batch.py b/vllm/v1/worker/gpu_input_batch.py index 01b6b5b0cc..79db440cff 100644 --- a/vllm/v1/worker/gpu_input_batch.py +++ b/vllm/v1/worker/gpu_input_batch.py @@ -323,7 +323,6 @@ class InputBatch: """Track add-request operations for logits processors. Not applicable to pooling models. """ - # Fill the next empty index if there is one. if (new_req_index := self.batch_update_builder.pop_removed()) is None: # Append to end otherwise. @@ -531,8 +530,8 @@ class InputBatch: Returns: Removed request index, or `None` if `req_id` not recognized - """ + """ req_index = self.req_id_to_index.pop(req_id, None) if req_index is None: return None @@ -710,6 +709,7 @@ class InputBatch: Returns: swaps: list of (from,to) swap tuples for moved requests empty_req_indices: indices not filled by condensation + """ num_reqs = self.num_reqs @@ -835,7 +835,6 @@ class InputBatch: def refresh_metadata(self): """Apply any batch updates to sampling metadata.""" - if self.is_pooling_model: batch_changed = self.batch_update_builder.reset() if batch_changed: @@ -1001,18 +1000,19 @@ class InputBatch: def make_lora_inputs( self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray ) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]: - """ - Given the num_scheduled_tokens for each request in the batch, return + """Given the num_scheduled_tokens for each request in the batch, return datastructures used to activate the current LoRAs. + Returns: 1. prompt_lora_mapping: A tuple of size np.sum(num_sampled_tokens) - where, prompt_lora_mapping[i] is the LoRA id to use for the ith - sampled token. + where, prompt_lora_mapping[i] is the LoRA id to use for the ith + sampled token. 2. token_lora_mapping: A tuple of size np.sum(num_scheduled_tokens) - where, token_lora_mapping[i] is the LoRA id to use for ith token. + where, token_lora_mapping[i] is the LoRA id to use for ith + token. 3. lora_requests: Set of relevant LoRA requests. - """ + """ req_lora_mapping = self.request_lora_mapping[: self.num_reqs] prompt_lora_mapping = tuple(req_lora_mapping.repeat(num_sampled_tokens)) token_lora_mapping = tuple(req_lora_mapping.repeat(num_scheduled_tokens)) @@ -1028,8 +1028,7 @@ class InputBatch: sampled_token_ids_cpu: torch.Tensor, async_copy_ready_event: torch.Event, ) -> None: - """ - In async scheduling case, store ref to sampled_token_ids_cpu + """In async scheduling case, store ref to sampled_token_ids_cpu tensor and corresponding copy-ready event. Used to repair output_token_ids prior to sampling, if needed by logits processors. """ @@ -1041,8 +1040,7 @@ class InputBatch: self.async_copy_ready_event = None def update_async_output_token_ids(self) -> None: - """ - In async scheduling case, update output_token_ids in sampling metadata + """In async scheduling case, update output_token_ids in sampling metadata from prior steps sampled token ids once they've finished copying to CPU. This is called right before they are needed by the logits processors. """ @@ -1088,8 +1086,7 @@ class InputBatch: # ^ Implicitly resizes to (first_placeholder + num_to_replace) def update_async_spec_token_ids(self, draft_token_ids: list[list[int]]) -> None: - """ - In async scheduling case, update spec_token_ids in sampling metadata with + """In async scheduling case, update spec_token_ids in sampling metadata with real draft token ids from prior step. This is called right before they are needed by the rejection sampler for penalty/bad_words computation. """ diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 97a59bd273..5c85257fa3 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1003,8 +1003,7 @@ class GPUModelRunner( self.effective_drafter_max_model_len = self.max_model_len def reset_mm_cache(self) -> None: - """ - Clear the multi-modal cache that was used during profiling, + """Clear the multi-modal cache that was used during profiling, but no longer needed during inference. """ if self.mm_budget: @@ -1136,14 +1135,14 @@ class GPUModelRunner( return model_kwargs def _may_reorder_batch(self, scheduler_output: "SchedulerOutput") -> None: - """ - Update the order of requests in the batch based on the attention + """Update the order of requests in the batch based on the attention backend's needs. For example, some attention backends (namely MLA) may want to separate requests based on if the attention computation will be compute-bound or memory-bound. Args: scheduler_output: The scheduler output. + """ # Attention free models have zero kv_cache_groups, however models # like Mamba are also attention free but use the kv_cache for @@ -1182,7 +1181,6 @@ class GPUModelRunner( # Note: used for model runner override. def _init_device_properties(self) -> None: """Initialize attributes from torch.cuda.get_device_properties""" - self.num_sms = num_compute_units(self.device.index) # Note: used for model runner override. @@ -1803,7 +1801,6 @@ class GPUModelRunner( Uses self.prev_positions[:num_reqs] which maps current pos -> prev pos (-1 for new requests). """ - # Sync the async PP broadcast before reading sampled tokens. if self._pp_recv_work is not None: self._pp_recv_work.wait() @@ -1977,9 +1974,9 @@ class GPUModelRunner( SpecDecodeMetadata | None, int, ]: - """ - Returns: - tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens] + """Returns: + tuple[logits_indices, spec_decode_metadata, max_num_sampled_tokens] + """ total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens assert total_num_scheduled_tokens > 0 @@ -2310,9 +2307,9 @@ class GPUModelRunner( cascade_attn_prefix_lens: list[list[int]] | None = None, slot_mappings: dict[int, torch.Tensor] | None = None, ) -> tuple[PerLayerAttnMetadata, CommonAttentionMetadata | None]: - """ - Returns: - tuple[attn_metadata, spec_decode_common_attn_metadata] + """Returns: + tuple[attn_metadata, spec_decode_common_attn_metadata] + """ # Attention metadata is not needed for attention free models if len(self.kv_cache_config.kv_cache_groups) == 0: @@ -2651,14 +2648,13 @@ class GPUModelRunner( num_computed_tokens: np.ndarray, num_common_prefix_blocks: list[int], ) -> list[list[int]] | None: - """ - Returns: - Optional[cascade_attn_prefix_lens] - cascade_attn_prefix_lens is 2D: - ``[kv_cache_group_id][attn_group_idx]``, - None if we should not use cascade attention - """ + """Returns: + Optional[cascade_attn_prefix_lens] + cascade_attn_prefix_lens is 2D: + ``[kv_cache_group_id][attn_group_idx]``, + None if we should not use cascade attention + """ use_cascade_attn = False num_kv_cache_groups = len(self.kv_cache_config.kv_cache_groups) cascade_attn_prefix_lens: list[list[int]] = [ @@ -2703,12 +2699,16 @@ class GPUModelRunner( Args: num_scheduled_tokens: Number of tokens scheduled per request. + num_computed_tokens: Number of tokens already computed per request. num_common_prefix_blocks: Number of shared KV cache blocks. + kv_cache_spec: The KV cache spec of the group being considered. + attn_metadata_builder: Builder used to decide whether cascade + attention can be used. Returns: int: Length of common prefix in tokens. - """ + """ common_prefix_len = num_common_prefix_blocks * kv_cache_spec.block_size if common_prefix_len == 0: # Common case. @@ -2947,6 +2947,7 @@ class GPUModelRunner( - mm_hashes: List of multimodal hashes for each item - mm_kwargs: List of multimodal kwargs for each item - mm_lora_refs: List of (req_id, placeholder_range) for each item + """ scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs if not scheduled_encoder_inputs: @@ -3404,9 +3405,7 @@ class GPUModelRunner( ) def eplb_step(self, is_dummy: bool = False, is_profile: bool = False) -> None: - """ - Step for the EPLB (Expert Parallelism Load Balancing) state. - """ + """Step for the EPLB (Expert Parallelism Load Balancing) state.""" if not self.parallel_config.enable_eplb or self.eep_eplb_suppressed: return @@ -3875,6 +3874,7 @@ class GPUModelRunner( Returns: Model output tensor + """ return self.model( input_ids=input_ids, @@ -3892,8 +3892,7 @@ class GPUModelRunner( num_reqs: int, force_uniform_decode: bool | None = None, ) -> bool: - """ - Checks if it's a decode batch with same amount scheduled tokens + """Checks if it's a decode batch with same amount scheduled tokens across all requests. """ return ( @@ -4065,11 +4064,9 @@ class GPUModelRunner( ) def _register_layerwise_nvtx_hooks(self) -> None: - """ - Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled + """Register layerwise NVTX hooks if --enable-layerwise-nvtx-tracing is enabled to trace detailed information of each layer or module in the model. """ - if ( self.vllm_config.observability_config.enable_layerwise_nvtx_tracing and not self.layerwise_nvtx_hooks_registered @@ -4110,8 +4107,7 @@ class GPUModelRunner( dict[int, torch.Tensor] | None, dict[str, torch.Tensor] | list[dict[str, torch.Tensor]] | None, ]: - """ - Build slot mappings in both formats needed by the system. + """Build slot mappings in both formats needed by the system. Args: num_tokens_padded: Total number of tokens (padded) @@ -4123,6 +4119,7 @@ class GPUModelRunner( A tuple of: - slot_mappings_by_gid: dict[int, torch.Tensor] for attention metadata - slot_mappings_by_layer: dict[str, torch.Tensor] or list for ForwardContext + """ if not ( hasattr(self, "kv_cache_config") @@ -5310,9 +5307,9 @@ class GPUModelRunner( @instrument(span_name="Loading (GPU)") def load_model(self, load_dummy_weights: bool = False) -> None: - """ - Args: - load_dummy_weights: load dummy weights instead of real weights. + """Args: + load_dummy_weights: load dummy weights instead of real weights. + """ logger.info_once( "Starting to load model %s...", @@ -5525,6 +5522,7 @@ class GPUModelRunner( Returns: Tuple of layer indices if found in draft model config, None otherwise. + """ if not (self.speculative_config and self.speculative_config.draft_model_config): return None @@ -5556,8 +5554,7 @@ class GPUModelRunner( weights_path: str | None = None, is_checkpoint_format: bool = True, ) -> None: - """ - Reload weights from a weights iterator or from disk + """Reload weights from a weights iterator or from disk Args: weights_iterator: weights to load into model @@ -5565,6 +5562,7 @@ class GPUModelRunner( provided. Use path of original model if neither is provided. is_checkpoint_format: set to False if weights have already been processed into kernel format (repacking, renaming, etc.) + """ # TODO(@kylesayrs): generalize to all runners and loaders # argument validation @@ -5774,13 +5772,11 @@ class GPUModelRunner( inputs_embeds: torch.Tensor | None, randomize_inputs: bool = False, ): - """ - Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set. + """Randomize input_ids if VLLM_RANDOMIZE_DP_DUMMY_INPUTS is set. This is to help balance expert-selection - during profile_run - during DP rank dummy run """ - dp_size = self.vllm_config.parallel_config.data_parallel_size randomize_inputs = randomize_inputs or ( envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1 @@ -5863,8 +5859,7 @@ class GPUModelRunner( profile_seq_lens: int | None = None, randomize_inputs: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Run a dummy forward pass to warm up/profile run or capture the + """Run a dummy forward pass to warm up/profile run or capture the CUDA graph for the model. Args: @@ -5889,6 +5884,12 @@ class GPUModelRunner( profile_seq_lens: If provided, use this value for seq_lens instead of max_query_len. Used to profile attention workspace that scales with context length. + allow_microbatching: Whether the dummy batch may be split into + microbatches. + is_graph_capturing: Whether this run is a cudagraph capture. + randomize_inputs: Whether to fill the dummy inputs with random + values rather than zeros. + """ mm_config = self.vllm_config.model_config.multimodal_config if mm_config and mm_config.mm_encoder_only: @@ -7038,9 +7039,7 @@ class GPUModelRunner( kv_cache_config: KVCacheConfig, is_profiling: bool = False, ) -> None: - """ - Initialize the attention backends and attention metadata builders. - """ + """Initialize the attention backends and attention metadata builders.""" assert len(self.attn_groups) == 0, "Attention backends are already initialized" class AttentionGroupKey(NamedTuple): @@ -7143,9 +7142,7 @@ class GPUModelRunner( def initialize_metadata_builders( self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int] ) -> None: - """ - Create the metadata builders for all KV cache groups and attn groups. - """ + """Create the metadata builders for all KV cache groups and attn groups.""" for kv_cache_group_id in range(len(kv_cache_config.kv_cache_groups)): for attn_group in self.attn_groups[kv_cache_group_id]: attn_group.create_metadata_builders( @@ -7180,8 +7177,7 @@ class GPUModelRunner( kv_cache_groups: list[KVCacheGroupSpec], is_profiling: bool = False, ) -> None: - """ - Resolve the cudagraph_mode when there are multiple attention + """Resolve the cudagraph_mode when there are multiple attention groups with potential conflicting CUDA graph support. Then initialize the cudagraph_dispatcher based on the resolved cudagraph_mode. @@ -7234,8 +7230,7 @@ class GPUModelRunner( self.drafter.initialize_cudagraph_keys(cudagraph_mode) def calculate_reorder_batch_threshold(self) -> None: - """ - Choose the minimum reorder batch threshold from all attention groups. + """Choose the minimum reorder batch threshold from all attention groups. Backends should be able to support lower threshold then what they request just may have a performance penalty due to that backend treating decodes as prefills. @@ -7256,8 +7251,7 @@ class GPUModelRunner( def may_reinitialize_input_batch( self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int] ) -> None: - """ - Re-initialize the input batch if the block sizes are different from + """Re-initialize the input batch if the block sizes are different from what it was originally created with. This happens when the final block size (determined after model loading) differs from the placeholder used during __init__, or when there are multiple @@ -7266,6 +7260,7 @@ class GPUModelRunner( Args: kv_cache_config: The KV cache configuration. kernel_block_sizes: The kernel block sizes for each KV cache group. + """ block_sizes = [] max_num_blocks = [] @@ -7347,18 +7342,19 @@ class GPUModelRunner( kernel_block_sizes: list[int], kv_cache_allocation_context: AbstractContextManager | None = None, ) -> dict[str, torch.Tensor]: - """ - Initialize the memory buffer for KV cache. + """Initialize the memory buffer for KV cache. Args: kv_cache_config: The KV cache config kernel_block_sizes: The kernel block sizes for each KV cache group. + kv_cache_allocation_context: Context manager the allocation runs + inside, e.g. to place the KV cache in a specific memory pool. Returns: Dict[str, torch.Tensor]: A map between layer names to their corresponding memory buffer for KV cache. - """ + """ allocation_context = kv_cache_allocation_context or nullcontext() with allocation_context: kv_caches = allocate_kv_cache( @@ -7388,8 +7384,7 @@ class GPUModelRunner( def maybe_add_kv_sharing_layers_to_kv_cache_groups( self, kv_cache_config: KVCacheConfig ) -> None: - """ - Add layers that re-use KV cache to KV cache group of its target layer. + """Add layers that re-use KV cache to KV cache group of its target layer. Mapping of KV cache tensors happens in `initialize_kv_cache_tensors()` """ if not self.shared_kv_cache_layers: @@ -7419,11 +7414,15 @@ class GPUModelRunner( is_profiling: bool = False, kv_cache_allocation_context: AbstractContextManager | None = None, ) -> None: - """ - Initialize KV cache based on `kv_cache_config`. + """Initialize KV cache based on `kv_cache_config`. + Args: kv_cache_config: Configuration for the KV cache, including the KV - cache size of each layer + cache size of each layer + is_profiling: Whether this call is part of a profiling run. + kv_cache_allocation_context: Context manager the allocation runs + inside, e.g. to place the KV cache in a specific memory pool. + """ kv_cache_config = deepcopy(kv_cache_config) self.kv_cache_config = kv_cache_config @@ -7530,9 +7529,7 @@ class GPUModelRunner( self.routed_experts_initialized = True def may_add_encoder_only_layers_to_kv_cache_config(self) -> None: - """ - Add encoder-only layers to the KV cache config. - """ + """Add encoder-only layers to the KV cache config.""" block_size = self.vllm_config.cache_config.block_size encoder_only_attn_specs: dict[AttentionSpec, list[str]] = defaultdict(list) attn_layers = get_layers_from_vllm_config(self.vllm_config, Attention) @@ -7556,12 +7553,13 @@ class GPUModelRunner( ) def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]: - """ - Generates the KVCacheSpec by parsing the kv cache format from each + """Generates the KVCacheSpec by parsing the kv cache format from each Attention module in the static forward context. + Returns: KVCacheSpec: A dictionary mapping layer names to their KV cache format. Layers that do not need KV cache are not included. + """ if has_ec_transfer() and not get_ec_transfer().is_consumer: return {} @@ -7605,11 +7603,11 @@ class GPUModelRunner( return pinned.tolist() def get_encoder_timing_stats(self) -> dict[str, dict[str, float | int]]: - """ - Get encoder timing stats for all requests and clear the registry. + """Get encoder timing stats for all requests and clear the registry. Returns: Dictionary mapping request_id to stats dict. + """ with self._encoder_timing_lock: stats = { @@ -7627,14 +7625,14 @@ class GPUModelRunner( current_item_idx: int, num_items: int, ): - """ - Context manager to time encoder forward operations. + """Context manager to time encoder forward operations. Args: should_time: Whether timing is enabled group_lora_refs: Full list of (request_id, pos_info) tuples current_item_idx: Starting index for this group num_items: Number of items in this group + """ if not should_time: yield diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index 0735b39f81..a0f2643052 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -119,8 +119,7 @@ class UBatchWrapper: return self.runnable def _capture_ubatches(self, ubatch_metadata, model) -> torch.Tensor: - """ - Capture a cudagraph for a microbatched run. + """Capture a cudagraph for a microbatched run. The logic here is somewhat complicated because we need to make sure that each of the ubatch threads initialize the cuda context before we start diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 2a3531b61b..acba1d9b67 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -536,6 +536,7 @@ class Worker(WorkerBase): Tip: You may limit the usage of GPU memory by adjusting the `gpu_memory_utilization` parameter. + """ maybe_apply_startup_plan(self) @@ -694,7 +695,6 @@ class Worker(WorkerBase): Returned dict is keyed by `(pp_rank, tp_rank)`. """ - if not has_kv_transfer_group(): return None @@ -734,7 +734,6 @@ class Worker(WorkerBase): @instrument(span_name="Allocate KV cache") def initialize_from_config(self, kv_cache_config: KVCacheConfig) -> None: """Allocate GPU KV cache with the specified kv_cache_config.""" - # Update local config with adjusted num blocks after profiling, # so that it's available to the warmup stage. self.cache_config.num_gpu_blocks = kv_cache_config.num_blocks @@ -1364,12 +1363,12 @@ class Worker(WorkerBase): ) def init_weight_transfer_engine(self, init_info: dict) -> None: - """ - Initialize weight transfer mechanism. + """Initialize weight transfer mechanism. For NCCL backend, this creates a process group with the trainer. Args: init_info: Dictionary containing backend-specific initialization info + """ self._check_weight_transfer_engine() assert self.weight_transfer_engine is not None @@ -1378,8 +1377,7 @@ class Worker(WorkerBase): self.weight_transfer_engine.init_transfer_engine(typed_init_info) def start_weight_update(self) -> None: - """ - Start a new weight update session. + """Start a new weight update session. Delegates engine-specific preparation (e.g. layerwise reload setup) to the configured weight transfer engine. The worker only tracks that a @@ -1389,8 +1387,7 @@ class Worker(WorkerBase): self._start_weight_update() def start_draft_weight_update(self) -> None: - """ - Like start_weight_update, but retargets the engine at the speculative + """Like start_weight_update, but retargets the engine at the speculative draft model for this session. """ with set_current_vllm_config(self.vllm_config): @@ -1423,8 +1420,7 @@ class Worker(WorkerBase): self._weight_update_is_draft = is_draft def update_weights(self, update_info: dict | list[dict]) -> None: - """ - Receive one weight update chunk from the trainer. + """Receive one weight update chunk from the trainer. start_weight_update must be called before update_weights and finish_weight_update must be called after all chunks have been sent. @@ -1434,6 +1430,7 @@ class Worker(WorkerBase): Args: update_info: Backend-specific update info, or a list indexed by global worker rank across data parallel replicas. + """ self._check_weight_transfer_engine() assert self.weight_transfer_engine is not None diff --git a/vllm/v1/worker/kv_connector_model_runner_mixin.py b/vllm/v1/worker/kv_connector_model_runner_mixin.py index 6054bb129a..26ffc159f3 100644 --- a/vllm/v1/worker/kv_connector_model_runner_mixin.py +++ b/vllm/v1/worker/kv_connector_model_runner_mixin.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Define KV connector functionality mixin for model runners. -""" +"""Define KV connector functionality mixin for model runners.""" from collections.abc import Generator from contextlib import AbstractContextManager, contextmanager, nullcontext diff --git a/vllm/v1/worker/lora_model_runner_mixin.py b/vllm/v1/worker/lora_model_runner_mixin.py index 5e7f1f70c5..9d94e62edb 100644 --- a/vllm/v1/worker/lora_model_runner_mixin.py +++ b/vllm/v1/worker/lora_model_runner_mixin.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -Define LoRA functionality mixin for model runners. -""" +"""Define LoRA functionality mixin for model runners.""" from collections.abc import Callable from contextlib import contextmanager @@ -150,16 +148,17 @@ class LoRAModelRunnerMixin: num_sampled_tokens: np.ndarray | None = None, num_active_loras: int = 0, ): - """ - Context manager to select dummy LoRAs for capture/warmup. + """Context manager to select dummy LoRAs for capture/warmup. Args: lora_config: LoRA configuration, or None if LoRA is disabled. num_scheduled_tokens: Array of scheduled token counts per request. + mapping_type: Which LoRA mapping to build (language or encoder). num_sampled_tokens: Array of sampled token counts per request. num_active_loras: Number of distinct active LoRAs to use. - 0: No LoRA active (set up zero mappings). - >0: Use exactly this many distinct LoRAs. + """ if num_sampled_tokens is None: num_sampled_tokens = np.ones_like(num_scheduled_tokens, dtype=np.int32) @@ -255,8 +254,7 @@ class LoRAModelRunnerMixin: num_active_loras: int = 0, mapping_type: LoRAMappingType = LoRAMappingType.LANGUAGE, ): - """ - Context manager for dummy runs with LoRA. + """Context manager for dummy runs with LoRA. Args: lora_config: LoRA configuration. @@ -265,6 +263,8 @@ class LoRAModelRunnerMixin: remove_lora: Whether to remove LoRAs after the context exits. num_active_loras: Number of distinct active LoRAs to use. LoRA is activated when num_active_loras > 0. + mapping_type: Which LoRA mapping to build (language or encoder). + """ with ( self.maybe_setup_dummy_loras(lora_config, remove_lora), diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index 1deb89bda6..c2e0120e46 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -415,8 +415,7 @@ def postprocess_mamba_fused_kernel( # the existing 2D-grid contract. TEMPORAL_TILES: tl.constexpr = 1, ): - """ - Fused GPU kernel for postprocess_mamba that computes decisions AND performs + """Fused GPU kernel for postprocess_mamba that computes decisions AND performs mamba state copies without any CPU-GPU synchronization. Grid: (num_reqs, num_states [, TEMPORAL_TILES]) @@ -769,8 +768,7 @@ class MambaCopyBuffers: @dataclasses.dataclass class MambaSpecDecodeGPUContext: - """ - Context for GPU-side Mamba state copy operations during the + """Context for GPU-side Mamba state copy operations during the fused postprocess path. Only used when speculative decoding is enabled on a hybrid model @@ -927,8 +925,7 @@ class MambaSpecDecodeGPUContext: mamba_state_copy_funcs: MambaStateCopyFuncsByType, block_tables: list[torch.Tensor], ) -> None: - """ - Extract and cache memory layout metadata from Mamba state tensors. + """Extract and cache memory layout metadata from Mamba state tensors. This method populates the pre-allocated metadata tensors with information needed by `postprocess_mamba_fused_kernel` to perform state copies entirely @@ -960,6 +957,7 @@ class MambaSpecDecodeGPUContext: block_tables: per-mamba-group persistent block-table tensors, in the same order as `mamba_group_ids`. Their `data_ptr()` / `stride(0)` are captured once for the kernel to index into. + """ if self.is_initialized: return @@ -1096,7 +1094,7 @@ class MambaSpecDecodeGPUContext: seq_lens: torch.Tensor, num_reqs: int, ) -> torch.Tensor: - """compute every Mamba group's aligned physical state IDs in one launch.""" + """Compute every Mamba group's aligned physical state IDs in one launch.""" assert self.is_initialized assert seq_lens.is_cuda assert 0 <= num_reqs <= seq_lens.shape[0] @@ -1137,8 +1135,7 @@ class MambaSpecDecodeGPUContext: num_computed_tokens_gpu: torch.Tensor, num_draft_tokens_gpu: torch.Tensor, ) -> None: - """ - Run the fused postprocess_mamba kernel on GPU. + """Run the fused postprocess_mamba kernel on GPU. This computes decisions and performs mamba state copies entirely on GPU, eliminating the CPU-GPU sync that was previously needed. @@ -1150,6 +1147,7 @@ class MambaSpecDecodeGPUContext: num_scheduled_tokens_gpu: [num_reqs] scheduled token counts num_computed_tokens_gpu: [num_reqs] computed token counts num_draft_tokens_gpu: [num_reqs] draft token counts + """ if num_reqs == 0 or not self.is_initialized: return @@ -1205,6 +1203,7 @@ class MambaSpecDecodeGPUContext: token_bias_gpu: [max_reqs] accepted-token bias (num_accepted - 1). idx_mapping: optional [num_reqs] batch_idx -> req_state_idx. None means V1 batch order already equals request state order. + """ if num_reqs == 0 or not self.is_initialized: return @@ -1442,8 +1441,7 @@ def preprocess_mamba( copy_bufs: MambaCopyBuffers, align_ctx: MambaSpecDecodeGPUContext | None = None, ): - """ - Copy the mamba state of previous step to the last + """Copy the mamba state of previous step to the last (1 + num_speculative_blocks) block. """ fused = _resolve_fused_precopy(align_ctx) diff --git a/vllm/v1/worker/ubatch_utils.py b/vllm/v1/worker/ubatch_utils.py index 9743322c41..329c261a0a 100644 --- a/vllm/v1/worker/ubatch_utils.py +++ b/vllm/v1/worker/ubatch_utils.py @@ -43,8 +43,7 @@ class SMControlContextManager: set_comm_sms: Callable[[int], None], set_compute_sms: Callable[[int], None], ): - """ - Context manager for controlling SM (Streaming Multiprocessor) + """Context manager for controlling SM (Streaming Multiprocessor) allocation. Upon entering the context, it sets the number of SMs allocated for communication and computation to comm_sms and total_sms - comm_sms respectively. Upon exiting, it restores the @@ -57,8 +56,8 @@ class SMControlContextManager: A function that sets the number of SMs for communication. set_compute_sms (Callable[[int], None]): A function that sets the number of SMs for computation. - """ + """ assert current_platform.is_cuda() or current_platform.is_rocm(), ( "SM/CU control is supported on CUDA and ROCm platforms" ) @@ -221,8 +220,7 @@ def slice_query_start_locs( query_start_loc: torch.Tensor, request_slice: slice, ) -> torch.Tensor: - """ - Creates a new query_start_loc that corresponds to the requests in + """Creates a new query_start_loc that corresponds to the requests in request_slice. Note: This function creates a new tensor to hold the new query_start_locs. @@ -237,11 +235,9 @@ def slice_query_start_locs( def _make_metadata_with_slice( ubatch_slice: UBatchSlice, attn_metadata: CommonAttentionMetadata ) -> CommonAttentionMetadata: - """ - This function creates a new CommonAttentionMetadata that corresponds to + """This function creates a new CommonAttentionMetadata that corresponds to the requests included in ubatch_slice """ - assert not ubatch_slice.is_empty(), f"Ubatch slice {ubatch_slice} is empty" request_slice = ubatch_slice.request_slice @@ -338,8 +334,7 @@ def split_attn_metadata( ubatch_slices: list[UBatchSlice], common_attn_metadata: CommonAttentionMetadata, ) -> list[CommonAttentionMetadata]: - """ - Creates a new CommonAttentionMetadata instance that corresponds to the + """Creates a new CommonAttentionMetadata instance that corresponds to the requests for each UBatchSlice in ubatch_slices. Note: This function does not modify common_attn_metadata diff --git a/vllm/v1/worker/ubatching.py b/vllm/v1/worker/ubatching.py index 6d06ba2135..a773ee715d 100644 --- a/vllm/v1/worker/ubatching.py +++ b/vllm/v1/worker/ubatching.py @@ -18,9 +18,7 @@ _CURRENT_CONTEXTS: list["UBatchContext | None"] = [] class UBatchContext: - """ - Context manager for micro-batching synchronization using threading events. - """ + """Context manager for micro-batching synchronization using threading events.""" def __init__( self, diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index df2ebc7d36..6c2a05a626 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -331,8 +331,7 @@ def select_common_block_size( kv_manager_block_size: int, backends: list[type[AttentionBackend]], ) -> int: - """ - Select a block size that is supported by all backends and is a factor of + """Select a block size that is supported by all backends and is a factor of kv_manager_block_size. If kv_manager_block_size is supported by all backends, return it directly. @@ -347,6 +346,7 @@ def select_common_block_size( Raises: ValueError: If no valid block size found. + """ def block_size_is_supported( @@ -458,8 +458,7 @@ def allocate_kv_cache( def prepare_kernel_block_sizes( kv_cache_config: KVCacheConfig, attn_groups: list[list[AttentionGroup]] ) -> list[int]: - """ - Generate kernel_block_sizes that matches each block_size. + """Generate kernel_block_sizes that matches each block_size. For attention backends that support virtual block splitting, use the supported block sizes from the backend. @@ -471,6 +470,7 @@ def prepare_kernel_block_sizes( Returns: List of kernel block sizes for each cache group. + """ kernel_block_sizes = [] for kv_cache_gid, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups): @@ -505,8 +505,7 @@ def sanity_check_mm_encoder_outputs( mm_embeddings: MultiModalEmbeddings, expected_num_items: int, ) -> None: - """ - Perform sanity checks for the result of + """Perform sanity checks for the result of [`vllm.model_executor.models.SupportsMultiModal.embed_multimodal`][]. """ assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), ( @@ -532,8 +531,7 @@ def sanity_check_mm_encoder_outputs( def request_memory(init_snapshot: MemorySnapshot, cache_config: CacheConfig) -> int: - """ - Calculate the amount of memory required by vLLM, then validate + """Calculate the amount of memory required by vLLM, then validate that the current amount of free memory is sufficient for that. """ requested_memory = math.ceil( @@ -559,8 +557,7 @@ def add_kv_sharing_layers_to_kv_cache_groups( kv_cache_groups: list[KVCacheGroupSpec], runner_only_attn_layers: set[str] | None = None, ) -> None: - """ - Sets up KV cache sharing by reusing the allocated KV caches in `kv_caches` + """Sets up KV cache sharing by reusing the allocated KV caches in `kv_caches` for layers that do not allocate its own KV cache, based on the mapping in `shared_kv_cache_layers`. Adds these layers to the corresponding KV cache group, which is needed to ensure that attention metadata is assigned later. @@ -571,6 +568,9 @@ def add_kv_sharing_layers_to_kv_cache_groups( means this layer will perform attention using the keys and values from the KV cache of `shared_kv_cache_layers[layer_name]`. kv_cache_groups: The KV cache groups of the model. + runner_only_attn_layers: Attention layers handled by the runner only, + which are excluded from the KV cache groups. + """ if not shared_kv_cache_layers: return @@ -595,8 +595,7 @@ def bind_kv_cache( num_attn_module: int = 1, kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None, ) -> None: - """ - Bind the allocated KV cache to both ModelRunner and forward context so + """Bind the allocated KV cache to both ModelRunner and forward context so that the KV cache can be used in the forward pass. This function: @@ -607,9 +606,13 @@ def bind_kv_cache( Args: kv_caches: The allocated kv_caches with layer names as keys. + num_attn_module: Number of attention modules per layer entry. forward_context: The global forward context containing all Attention layers with layer names as keys. runner_kv_caches: The kv_cache declared by ModelRunner. + kv_cache_groups: The KV cache groups of the model, used to resolve + layers that share a KV cache. + """ # Bind kv_caches to ModelRunner assert len(runner_kv_caches) == 0 diff --git a/vllm/v1/worker/worker_base.py b/vllm/v1/worker/worker_base.py index cf5b0e7ce3..cf869edb26 100644 --- a/vllm/v1/worker/worker_base.py +++ b/vllm/v1/worker/worker_base.py @@ -55,8 +55,7 @@ class WorkerBase: distributed_init_method: str, is_driver_worker: bool = False, ) -> None: - """ - Initialize common worker components. + """Initialize common worker components. Args: vllm_config: Complete vLLM configuration @@ -65,6 +64,7 @@ class WorkerBase: distributed_init_method: Distributed initialization method is_driver_worker: Whether this worker handles driver responsibilities + """ self.vllm_config = vllm_config self.model_config = vllm_config.model_config @@ -118,6 +118,7 @@ class WorkerBase: Returns: Compilation times (language_model, encoder) in seconds. + """ raise NotImplementedError @@ -209,8 +210,7 @@ class WorkerBase: class WorkerWrapperBase: - """ - This class represents one process in an executor/engine. It is responsible + """This class represents one process in an executor/engine. It is responsible for lazily initializing the worker and handling the worker's lifecycle. We first instantiate the WorkerWrapper, which remembers the worker module and class name. Then, when we call `update_environment_variables`, and the @@ -222,8 +222,7 @@ class WorkerWrapperBase: rpc_rank: int = 0, global_rank: int | None = None, ) -> None: - """ - Initialize the worker wrapper with the given vllm_config and rpc_rank. + """Initialize the worker wrapper with the given vllm_config and rpc_rank. Note: rpc_rank is the rank of the worker in the executor. In most cases, it is also the rank of the worker in the distributed group. However, when multiple executors work together, they can be different. @@ -252,8 +251,7 @@ class WorkerWrapperBase: @instrument(span_name="Worker init") def init_worker(self, all_kwargs: list[dict[str, Any]]) -> None: - """ - Here we inject some common logic before initializing the worker. + """Here we inject some common logic before initializing the worker. Arguments are passed to the worker class constructor. """ kwargs = all_kwargs[self.rpc_rank] diff --git a/vllm/v1/worker/workspace.py b/vllm/v1/worker/workspace.py index 39b3b10349..cddb760acd 100644 --- a/vllm/v1/worker/workspace.py +++ b/vllm/v1/worker/workspace.py @@ -123,6 +123,7 @@ class WorkspaceManager: Returns: List of tensor views into the workspace buffer, one per shape/dtype pair. + """ actual_bytes = [_compute_bytes(s, d) for s, d in shapes_and_dtypes] aligned_bytes = [round_up(actual, 256) for actual in actual_bytes] @@ -148,6 +149,7 @@ class WorkspaceManager: Returns: The current workspace tensor. + """ ubatch_id = dbo_current_ubatch_id() lane = _workspace_lane.get() @@ -228,6 +230,7 @@ def is_workspace_manager_initialized() -> bool: Returns: True if workspace manager is initialized, False otherwise. + """ return _manager is not None @@ -237,6 +240,7 @@ def current_workspace_manager() -> "WorkspaceManager": Raises: AssertionError: If workspace manager has not been initialized. + """ assert _manager is not None, ( "WorkspaceManager not initialized. Call init_workspace_manager() " @@ -259,6 +263,7 @@ def init_workspace_manager( device: The device to allocate workspace on. num_ubatches: Number of workspace ubatch slots. Defaults to 1. num_lanes: Number of independent execution lanes per ubatch. Defaults to 1. + """ global _manager if _manager is not None: @@ -289,6 +294,7 @@ def lock_workspace() -> None: lock_workspace() # Now all get_workspace calls must fit in pre-allocated size + """ current_workspace_manager().lock() diff --git a/vllm/vllm_flash_attn/flash_attn_interface.py b/vllm/vllm_flash_attn/flash_attn_interface.py index 4ba4692f67..c10b381e40 100644 --- a/vllm/vllm_flash_attn/flash_attn_interface.py +++ b/vllm/vllm_flash_attn/flash_attn_interface.py @@ -261,11 +261,41 @@ def flash_attn_varlen_func( return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). + seqused_k: (batch_size,), dtype torch.int32. Actual number of key tokens per + sequence. Mutually exclusive with cu_seqlens_k, and required when + block_table is given. + q_v: optional value tensor fused with q. FA3 only. + block_table: (batch_size, max_blocks_per_seq), dtype torch.int32. Paged KV + cache block table. Requires seqused_k. + return_softmax_lse: bool. Whether to also return softmax_lse. + out: optional tensor to write the output into. + scheduler_metadata: precomputed scheduling metadata. Not supported by FA2. + q_descale: descale factor for FP8 q. Not supported by FA2, and ignored by + FA4 when the inputs are not FP8. + k_descale: descale factor for FP8 k. Same constraints as q_descale. + v_descale: descale factor for FP8 v. Same constraints as q_descale. + num_splits: int. Number of splits for the split-KV kernel; 0 lets the kernel + choose. FA2 only supports 0 or 1. + output_scale: scale for fused FP8 output quantization. FA4 only. + fa_version: int. FlashAttention major version to dispatch to. + s_aux: learnable per-head attention sink logits. Not supported by FA2. + cp_world_size: int. Context parallel world size. + cp_rank: int. Rank of this shard within the context parallel group. + cp_tot_seqused_k: (batch_size,), dtype torch.int32. Key lengths across the + whole context parallel group. + mask_mod: optional callable applying a custom mask. Not supported by FA2. + block_sparse_tensors: block-sparse index tensors. FA4 only, and must + materialize full_block_cnt and full_block_idx. + aux_tensors: auxiliary tensors consumed by mask_mod. FA4 only. + aux_tensor_leading_dims: leading dimensions of each entry in aux_tensors. + dynamic_causal: optional per-sequence causal offsets. FA4 only. + Return: out: (total, nheads, headdim). softmax_lse [optional, if return_softmax_lse=True]: (nheads, total_q_seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). + """ assert cu_seqlens_k is not None or seqused_k is not None, ( "cu_seqlens_k or seqused_k must be provided" @@ -577,11 +607,16 @@ def sparse_attn_func( return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). + softcap: float. Anything > 0 activates softcapping attention. + return_softmax_lse: bool. Whether to also return softmax_lse. + out: optional tensor to write the output into. + Return: out: (batch_size, seqlen, nheads, headdim). softmax_lse [optional, if return_softmax_lse=True]: (batch_size, nheads, seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). + """ if softmax_scale is None: softmax_scale = q.shape[-1] ** (-0.5) @@ -663,11 +698,15 @@ def sparse_attn_varlen_func( return_attn_probs: bool. Whether to return the attention probabilities. This option is for testing only. The returned probabilities are not guaranteed to be correct (they might not have the right scaling). + return_softmax_lse: bool. Whether to also return softmax_lse. + out: optional tensor to write the output into. + Return: out: (total, nheads, headdim). softmax_lse [optional, if return_softmax_lse=True]: (nheads, total_q_seqlen). The logsumexp of each row of the matrix QK^T * scaling (e.g., log of the softmax normalization factor). + """ if softmax_scale is None: softmax_scale = q.shape[-1] ** (-0.5)