diff --git a/tests/models/multimodal/generation/test_memory_leak.py b/tests/models/multimodal/generation/test_memory_leak.py index 45eac5b80a..f58b7308b5 100644 --- a/tests/models/multimodal/generation/test_memory_leak.py +++ b/tests/models/multimodal/generation/test_memory_leak.py @@ -15,7 +15,8 @@ from vllm import LLM, SamplingParams from vllm.distributed import cleanup_dist_env_and_memory from vllm.entrypoints.chat_utils import ChatCompletionMessageParam from vllm.platforms import current_platform -from vllm.utils.mem_utils import KiB_bytes, MiB_bytes, format_mib +from vllm.utils.mem_constants import KiB_bytes, MiB_bytes +from vllm.utils.mem_utils import format_mib MODEL_NAME = "Qwen/Qwen3-VL-4B-Instruct" RANDOM_PREFIX_LEN = 100 diff --git a/vllm/compilation/passes/fx_utils.py b/vllm/compilation/passes/fx_utils.py index 5b949d6140..a1f0b5f84b 100644 --- a/vllm/compilation/passes/fx_utils.py +++ b/vllm/compilation/passes/fx_utils.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import operator from collections.abc import Iterable, Iterator from torch import fx @@ -29,15 +28,6 @@ def find_auto_fn(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node: return node -# Returns the getitem node that extracts the idx-th element from node -# (if it exists) -def find_getitem_maybe(node: fx.Node, idx: int) -> fx.Node | None: - for user in node.users: - if is_func(user, operator.getitem) and user.args[1] == idx: - return user - return None - - # An auto-functionalization-aware utility for finding nodes with a specific op # Also handles op overload packets and finds all overloads def find_op_nodes( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py index 1383fc09eb..64e4893dcb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/lmcache_integration/utils.py @@ -23,11 +23,6 @@ _config_instance: V1Config | None = None _config_lock = threading.Lock() -def is_false(value: str) -> bool: - """Check if the given string value is equivalent to 'false'.""" - return value.lower() in ("false", "0", "no", "n", "off") - - def lmcache_get_or_create_config() -> V1Config: """Get the LMCache configuration from the environment variable `LMCACHE_CONFIG_FILE`. If the environment variable is not set, this @@ -97,75 +92,6 @@ def mla_enabled(model_config: "ModelConfig") -> bool: ) -def create_lmcache_metadata( - vllm_config=None, model_config=None, parallel_config=None, cache_config=None -): - """ - Create LMCacheEngineMetadata from vLLM configuration. - - This function extracts common metadata creation logic that was duplicated - across multiple files. - - Args: - vllm_config (VllmConfig): vLLM configuration object containing model, - parallel, and cache configs (alternative to - individual config parameters) - model_config (ModelConfig): Model configuration (alternative to - vllm_config) - parallel_config (ParallelConfig): Parallel configuration (alternative - to vllm_config) - cache_config (CacheConfig): Cache configuration (alternative to - vllm_config) - """ - # Third Party - # First Party - from lmcache.config import LMCacheEngineMetadata - - from vllm.utils.torch_utils import get_kv_cache_torch_dtype - - config = lmcache_get_or_create_config() - # Support both vllm_config object and individual config parameters - if vllm_config is not None: - model_cfg = vllm_config.model_config - parallel_cfg = vllm_config.parallel_config - cache_cfg = vllm_config.cache_config - else: - if model_config is None or parallel_config is None or cache_config is None: - raise ValueError( - "Either vllm_config must be provided, or all of " - "model_config, parallel_config, and cache_config must be provided." - ) - model_cfg = model_config - parallel_cfg = parallel_config - cache_cfg = cache_config - - # Get KV cache dtype - kv_dtype = get_kv_cache_torch_dtype(cache_cfg.cache_dtype, model_cfg.dtype) - - # Check if MLA is enabled - use_mla = mla_enabled(model_cfg) - - # Construct KV shape (for memory pool) - num_layer = model_cfg.get_num_layers(parallel_cfg) - chunk_size = config.chunk_size - num_kv_head = model_cfg.get_num_kv_heads(parallel_cfg) - head_size = model_cfg.get_head_size() - kv_shape = (num_layer, 1 if use_mla else 2, chunk_size, num_kv_head, head_size) - - # Create metadata - metadata = LMCacheEngineMetadata( - model_cfg.model, - parallel_cfg.world_size, - parallel_cfg.rank, - "vllm", - kv_dtype, - kv_shape, - use_mla, - ) - - return metadata, config - - def extract_mm_features( request: Union["Request", "NewRequestData"], modify: bool = False ) -> tuple[list[str], list["PlaceholderRange"]]: diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index caa52f2851..3e19c2b389 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -22,14 +22,6 @@ from vllm.utils.torch_utils import direct_register_custom_op logger = init_logger(__name__) -MOE_LAYER_ROUTER_GATE_SUFFIXES = { - "gate", - "router", - "router_gate", - "shared_expert_gate", - "expert_gate", -} - def get_token_bin_counts_and_mask( tokens: torch.Tensor, diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 93afe87036..f6a97961f4 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -1241,21 +1241,6 @@ def multi_thread_pt_weights_iterator( del state -def convert_pyslice_to_tensor(x: Any) -> torch.Tensor: - """convert PySafeSlice object from safetensors to torch.Tensor - - PySafeSlice object supports indexing, which is done before loading the - actual tensor and can reduce the amount of memory being read into the - memory. However, it does not support more advanced functionalities - like `.view()` or `.t()`. Therefore, if we need to modify the loaded - tensor with these more complicated operators, we need to convert to - tensor first. - """ - if not isinstance(x, torch.Tensor): - x = x[:] - return x - - def default_weight_loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None: """Default weight loader.""" try: diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index a3c24a3fb9..7c76dd7ba2 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -995,44 +995,6 @@ if has_flashinfer(): return torch.empty(A.shape[0], B.shape[1], dtype=out_dtype, device=A.device) -def flashinfer_mm_mxfp8( - a: torch.Tensor, - b: torch.Tensor, - block_scale_a: torch.Tensor, - block_scale_b: torch.Tensor, - out_dtype: torch.dtype, - backend: str = "cutlass", -) -> torch.Tensor: - """MXFP8 MM helper - mirrors flashinfer_scaled_fp4_mm API. - - Takes non-transposed weights and handles transpose internally. - - CRITICAL: mm_mxfp8 CUTLASS kernel requires SWIZZLED 1D scales for optimal - performance and accuracy. Both input and weight scales should be in - swizzled format from FlashInfer's mxfp8_quantize(is_sf_swizzled_layout=True). - """ - # a shape [M, K] - # b shape [K, N] - assert a.ndim == 2 and b.ndim == 2 - assert a.shape[1] == b.shape[1] # K dimension must match - - if block_scale_b.ndim != 1: - raise ValueError( - "mm_mxfp8 expects 1D swizzled weight scales for CUTLASS; " - f"got shape={tuple(block_scale_b.shape)}" - ) - - # Output tensor [M, N] - return mm_mxfp8( - a, - b.t(), # Transpose weight: [N, K] -> [K, N] - block_scale_a, - block_scale_b, - out_dtype, - backend=backend, - ) - - def flashinfer_scaled_fp4_mm( a: torch.Tensor, b: torch.Tensor, diff --git a/vllm/utils/hpc.py b/vllm/utils/hpc.py index abe546c916..289e1830c2 100644 --- a/vllm/utils/hpc.py +++ b/vllm/utils/hpc.py @@ -76,27 +76,6 @@ def fuse_moe_impl( ) -# @torch.library.register_fake( -# "vllm::fuse_moe_impl", -# ) -def fuse_moe_impl_fake( - x: torch.Tensor, - gate_up_weight: torch.Tensor, - down_weight: torch.Tensor, - gate_up_scale: torch.Tensor, - down_scale: torch.Tensor, - act_and_mul_scale: torch.Tensor, - topk_ids: torch.Tensor, - topk_scale: torch.Tensor, - rank_ep: int, - num_expert_total: int, - use_bf16_mul: bool = True, - shared_output: torch.Tensor = None, - output: torch.Tensor = None, -) -> torch.Tensor: - return torch.empty_like(x) - - def hpc_fuse_moe( x: torch.Tensor, gate_up_weight: torch.Tensor, @@ -166,26 +145,6 @@ def fuse_moe_blockwise_impl( ) -# @torch.library.register_fake( -# "vllm::fuse_moe_blockwise_impl", -# ) -def fuse_moe_blockwise_impl_fake( - x: torch.Tensor, - x_scale: torch.Tensor, - gate_up_weight: torch.Tensor, - gate_up_weight_scale: torch.Tensor, - down_weight: torch.Tensor, - down_weight_scale: torch.Tensor, - topk_ids: torch.Tensor, - topk_scale: torch.Tensor, - rank_ep: int, - num_expert_total: int, - shared_output: torch.Tensor = None, - output: torch.Tensor = None, -) -> torch.Tensor: - return torch.empty_like(x) - - def hpc_fuse_moe_blockwise( x: torch.Tensor, x_scale: torch.Tensor, diff --git a/vllm/utils/mem_utils.py b/vllm/utils/mem_utils.py index 6a522a150d..c092dff410 100644 --- a/vllm/utils/mem_utils.py +++ b/vllm/utils/mem_utils.py @@ -43,11 +43,6 @@ def get_max_shared_memory_bytes(gpu: int = 0) -> int: return int(max_shared_mem) -def get_cpu_memory() -> int: - """Returns the total CPU memory of the node in bytes.""" - return psutil.virtual_memory().total - - _UMA_PRESSURE_THRESHOLD = 0.8 _UMA_MIN_RELEASE_BYTES = 512 * MiB_bytes diff --git a/vllm/utils/torch_utils.py b/vllm/utils/torch_utils.py index 4f6be20fb4..557b3b6490 100644 --- a/vllm/utils/torch_utils.py +++ b/vllm/utils/torch_utils.py @@ -497,15 +497,6 @@ def get_kv_cache_quant_algo_string(quant_cfg: dict[str, Any]) -> str | None: return None -def get_kv_cache_quant_algo_dtype(quant_cfg: dict[str, Any]) -> torch.dtype | None: - """Get the KV cache quantization algorithm dtype from the quantization config.""" - kv_algo_str = get_kv_cache_quant_algo_string(quant_cfg) - if kv_algo_str is not None and kv_algo_str != "auto": - # Only convert if we have a valid dtype string (not "auto" fallback) - return STR_DTYPE_TO_TORCH_DTYPE[kv_algo_str] - return None - - def resolve_kv_cache_dtype_string( kv_cache_dtype: str, model_config: ModelConfig ) -> str: diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index c9596e1c9c..716f368959 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -3,7 +3,7 @@ import math from collections import defaultdict from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass, field, fields, make_dataclass +from dataclasses import dataclass, field, fields from typing import ( TYPE_CHECKING, Any, @@ -975,19 +975,6 @@ def reshape_attn_output_for_spec_decode(attn_output: torch.Tensor) -> torch.Tens return attn_output.view(total_tokens, attn_output.shape[2], attn_output.shape[3]) -def subclass_attention_metadata( - name_prefix: str, - metadata_cls: Any, - fields: list[tuple[str, Any, Any]], -) -> Any: - """ - Return a new subclass of `metadata_cls` with additional fields - """ - name: str = name_prefix + metadata_cls.__name__ # type: ignore - Wrapped = make_dataclass(name, fields, bases=(metadata_cls,)) - return Wrapped - - @runtime_checkable class KVSharingFastPrefillMetadata(Protocol): logits_indices_padded: torch.Tensor | None = None diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index a86f694797..0755a69299 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -86,8 +86,6 @@ class KVCacheCoordinator(ABC): num_prefill_lookahead: int = 0, ): self.kv_cache_config = kv_cache_config - self.max_model_len = max_model_len - self.enable_caching = enable_caching # The scheduling granularity (LCM of all group block sizes), must be a multiple # of the hash_block_size and the block size of each group. assert scheduler_block_size % hash_block_size == 0 and all( diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index fdbc6f07c5..a3cc1088e2 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2148,8 +2148,6 @@ class SinkFullAttentionManager(FullAttentionManager): ) sink_len = kv_cache_spec.sink_len assert sink_len is not None and sink_len > 0 and sink_len % self.block_size == 0 - num_sink_block = sink_len // self.block_size - self.sink_blocks = self.block_pool.free_block_queue.popleft_n(num_sink_block) def get_manager_for_kv_cache_spec( diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 0d40c186e0..ffc9a5de7e 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -85,7 +85,7 @@ from vllm.v1.fault_tolerance.engine_core_sentinel import ( EngineCoreSentinel, fault_tolerant_wrapper, ) -from vllm.v1.kv_cache_interface import KVCacheConfig, get_kv_cache_spec_kind +from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.metrics.stats import SchedulerIterationDetails, SchedulerStats from vllm.v1.outputs import ModelRunnerOutput from vllm.v1.request import Request, RequestStatus @@ -441,25 +441,6 @@ class EngineCore: supported_pooling_tasks, ) - def get_kv_cache_group_metadata(self) -> list[dict[str, int | str | None]]: - """Return msgspec-serializable metadata for scheduler KV cache groups.""" - kv_cache_config = getattr(self.scheduler, "kv_cache_config", None) - if kv_cache_config is None: - return [] - - metadata: list[dict[str, int | str | None]] = [] - for group_idx, group in enumerate(kv_cache_config.kv_cache_groups): - spec = group.kv_cache_spec - metadata.append( - { - "group_idx": group_idx, - "kind": get_kv_cache_spec_kind(spec).value, - "block_size": spec.block_size, - "sliding_window": getattr(spec, "sliding_window", None), - } - ) - return metadata - def add_request(self, request: Request, request_wave: int = 0): """Add request to the scheduler. diff --git a/vllm/v1/executor/ray_utils.py b/vllm/v1/executor/ray_utils.py index 305d97368a..71d4cf7471 100644 --- a/vllm/v1/executor/ray_utils.py +++ b/vllm/v1/executor/ray_utils.py @@ -176,9 +176,6 @@ try: output = output.get_output() return output - def override_env_vars(self, vars: dict[str, str]): - os.environ.update(vars) - def _is_intermediate_tensors(self, output) -> bool: return isinstance(output, IntermediateTensors) @@ -663,29 +660,3 @@ def initialize_ray_cluster( ) # Set the placement group in the parallel config parallel_config.placement_group = current_placement_group - - -def get_num_tpu_nodes() -> int: - from ray._private.accelerators import TPUAcceleratorManager - - cluster_resources = ray.cluster_resources() - total_tpus = int(cluster_resources["TPU"]) - tpus_per_node = TPUAcceleratorManager.get_current_node_num_accelerators() - assert total_tpus % tpus_per_node == 0 - return total_tpus // tpus_per_node - - -def get_num_nodes_in_placement_group() -> int: - pg_table = ray.util.placement_group_table() - current_pg = ray.util.get_current_placement_group() - num_nodes = 0 - - if current_pg: - nodes_in_pg = set() - for pg_key, pg in pg_table.items(): - if pg_key == current_pg.id.hex(): - for _, node in pg["bundles_to_node_id"].items(): - nodes_in_pg.add(node) - num_nodes = len(nodes_in_pg) - - return num_nodes diff --git a/vllm/v1/worker/cp_utils.py b/vllm/v1/worker/cp_utils.py index 92d8383c1f..7e208da393 100644 --- a/vllm/v1/worker/cp_utils.py +++ b/vllm/v1/worker/cp_utils.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, cast import torch from vllm.config import VllmConfig, get_layers_from_vllm_config -from vllm.distributed import get_dcp_group from vllm.logger import init_logger from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.utils import split_decodes_prefills_and_extends @@ -52,15 +51,6 @@ def check_attention_cp_compatibility(vllm_config: VllmConfig) -> None: ) -def get_kv_cache_shard_count() -> int: - try: - dcp_world_size = get_dcp_group().world_size - except AssertionError: - # DCP might not be initialized in testing - dcp_world_size = 1 - return dcp_world_size - - def get_dcp_dummy_context_len( dcp_world_size: int, cp_kv_cache_interleave_size: int,