[Misc] Replace Optional[X] with X | None syntax (#33332)
Signed-off-by: carlory <baofa.fan@daocloud.io> Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.5
parent
c87eac18f7
commit
1a7894dbdf
+2
-2
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from vllm.logprobs import Logprob
|
from vllm.logprobs import Logprob
|
||||||
from vllm.lora.request import LoRARequest
|
from vllm.lora.request import LoRARequest
|
||||||
@@ -27,7 +27,7 @@ class BeamSearchSequence:
|
|||||||
text: str | None = None
|
text: str | None = None
|
||||||
finish_reason: str | None = None
|
finish_reason: str | None = None
|
||||||
stop_reason: int | str | None = None
|
stop_reason: int | str | None = None
|
||||||
multi_modal_data: Optional["MultiModalDataDict"] = None
|
multi_modal_data: "MultiModalDataDict | None" = None
|
||||||
mm_processor_kwargs: dict[str, Any] | None = None
|
mm_processor_kwargs: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import TYPE_CHECKING, Optional, cast
|
from typing import TYPE_CHECKING, cast
|
||||||
|
|
||||||
from vllm.distributed.kv_transfer.kv_connector.base import (
|
from vllm.distributed.kv_transfer.kv_connector.base import (
|
||||||
KVConnectorBase,
|
KVConnectorBase,
|
||||||
@@ -44,7 +44,7 @@ class KVConnectorFactory:
|
|||||||
cls,
|
cls,
|
||||||
config: "VllmConfig",
|
config: "VllmConfig",
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
) -> KVConnectorBase:
|
) -> KVConnectorBase:
|
||||||
kv_transfer_config = config.kv_transfer_config
|
kv_transfer_config = config.kv_transfer_config
|
||||||
if kv_transfer_config is None:
|
if kv_transfer_config is None:
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ The class provides the following primitives:
|
|||||||
import enum
|
import enum
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
from typing import TYPE_CHECKING, Any, Literal, Optional
|
from typing import TYPE_CHECKING, Any, Literal
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -161,7 +161,7 @@ class KVConnectorBase_V1(ABC):
|
|||||||
self,
|
self,
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Initializing KVConnectorBase_V1. This API is experimental and "
|
"Initializing KVConnectorBase_V1. This API is experimental and "
|
||||||
@@ -383,13 +383,13 @@ class KVConnectorBase_V1(ABC):
|
|||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:
|
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
|
||||||
"""
|
"""
|
||||||
Get the KV connector stats collected during the last interval.
|
Get the KV connector stats collected during the last interval.
|
||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_kv_connector_kv_cache_events(self) -> Optional["KVConnectorKVEvents"]:
|
def get_kv_connector_kv_cache_events(self) -> "KVConnectorKVEvents | None":
|
||||||
"""
|
"""
|
||||||
Get the KV connector kv cache events collected during the last interval.
|
Get the KV connector kv cache events collected during the last interval.
|
||||||
This function should be called by the model runner every time after the
|
This function should be called by the model runner every time after the
|
||||||
@@ -558,7 +558,7 @@ class KVConnectorBase_V1(ABC):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def build_kv_connector_stats(
|
def build_kv_connector_stats(
|
||||||
cls, data: dict[str, Any] | None = None
|
cls, data: dict[str, Any] | None = None
|
||||||
) -> Optional["KVConnectorStats"]:
|
) -> "KVConnectorStats | None":
|
||||||
"""
|
"""
|
||||||
KVConnectorStats resolution method. This method allows dynamically
|
KVConnectorStats resolution method. This method allows dynamically
|
||||||
registered connectors to return their own KVConnectorStats object,
|
registered connectors to return their own KVConnectorStats object,
|
||||||
@@ -584,7 +584,7 @@ class KVConnectorBase_V1(ABC):
|
|||||||
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
|
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
|
||||||
labelnames: list[str],
|
labelnames: list[str],
|
||||||
per_engine_labelvalues: dict[int, list[object]],
|
per_engine_labelvalues: dict[int, list[object]],
|
||||||
) -> Optional["KVConnectorPromMetrics"]:
|
) -> "KVConnectorPromMetrics | None":
|
||||||
"""
|
"""
|
||||||
Create a KVConnectorPromMetrics subclass which should register
|
Create a KVConnectorPromMetrics subclass which should register
|
||||||
per-connector Prometheus metrics and implement observe() to
|
per-connector Prometheus metrics and implement observe() to
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ Usage:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -84,7 +84,7 @@ class DecodeBenchConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(vllm_config, role, kv_cache_config)
|
super().__init__(vllm_config, role, kv_cache_config)
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import os
|
import os
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import safetensors
|
import safetensors
|
||||||
import torch
|
import torch
|
||||||
@@ -91,7 +91,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import os
|
|||||||
import uuid
|
import uuid
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from lmcache import utils
|
from lmcache import utils
|
||||||
@@ -274,7 +274,7 @@ class ReqMeta:
|
|||||||
load_spec: LoadSpec | None = None,
|
load_spec: LoadSpec | None = None,
|
||||||
discard_partial_chunks: bool = True,
|
discard_partial_chunks: bool = True,
|
||||||
save_decode_cache: bool = False,
|
save_decode_cache: bool = False,
|
||||||
) -> Optional["ReqMeta"]:
|
) -> "ReqMeta | None":
|
||||||
"""Create the request metadata from a request tracker.
|
"""Create the request metadata from a request tracker.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
import enum
|
import enum
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any, Literal, Optional, cast
|
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import zmq
|
import zmq
|
||||||
@@ -385,7 +385,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(vllm_config, role, kv_cache_config)
|
super().__init__(vllm_config, role, kv_cache_config)
|
||||||
|
|
||||||
@@ -595,7 +595,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):
|
|||||||
self.worker_adapter.shutdown()
|
self.worker_adapter.shutdown()
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get_kv_connector_stats(self) -> Optional["KVConnectorStats"]:
|
def get_kv_connector_stats(self) -> "KVConnectorStats | None":
|
||||||
"""
|
"""
|
||||||
Get the KV connector stats collected during the last interval.
|
Get the KV connector stats collected during the last interval.
|
||||||
"""
|
"""
|
||||||
@@ -810,7 +810,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def build_kv_connector_stats(
|
def build_kv_connector_stats(
|
||||||
cls, data: dict[str, Any] | None = None
|
cls, data: dict[str, Any] | None = None
|
||||||
) -> Optional["KVConnectorStats"]:
|
) -> "KVConnectorStats | None":
|
||||||
"""
|
"""
|
||||||
KVConnectorStats resolution method. This method allows dynamically
|
KVConnectorStats resolution method. This method allows dynamically
|
||||||
registered connectors to return their own KVConnectorStats object,
|
registered connectors to return their own KVConnectorStats object,
|
||||||
@@ -825,7 +825,7 @@ class LMCacheMPConnector(KVConnectorBase_V1):
|
|||||||
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
|
metric_types: dict[type["PromMetric"], type["PromMetricT"]],
|
||||||
labelnames: list[str],
|
labelnames: list[str],
|
||||||
per_engine_labelvalues: dict[int, list[object]],
|
per_engine_labelvalues: dict[int, list[object]],
|
||||||
) -> Optional["KVConnectorPromMetrics"]:
|
) -> "KVConnectorPromMetrics | None":
|
||||||
"""
|
"""
|
||||||
Create a KVConnectorPromMetrics subclass which should register
|
Create a KVConnectorPromMetrics subclass which should register
|
||||||
per-connector Prometheus metrics and implement observe() to
|
per-connector Prometheus metrics and implement observe() to
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import time
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -115,7 +115,7 @@ class MooncakeConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(vllm_config, role, kv_cache_config)
|
super().__init__(vllm_config, role, kv_cache_config)
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import torch
|
import torch
|
||||||
@@ -101,7 +101,7 @@ class MoRIIOAgentMetadata(
|
|||||||
class RoleManager:
|
class RoleManager:
|
||||||
"""Manages role state across the connector."""
|
"""Manages role state across the connector."""
|
||||||
|
|
||||||
_instance: Optional["RoleManager"] = None
|
_instance: "RoleManager | None" = None
|
||||||
_lock = threading.Lock()
|
_lock = threading.Lock()
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from concurrent.futures import Future, ThreadPoolExecutor
|
from concurrent.futures import Future, ThreadPoolExecutor
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import msgpack
|
import msgpack
|
||||||
import msgspec
|
import msgspec
|
||||||
@@ -90,7 +90,7 @@ class MoRIIOConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(vllm_config, role)
|
super().__init__(vllm_config, role)
|
||||||
assert vllm_config.kv_transfer_config is not None, (
|
assert vllm_config.kv_transfer_config is not None, (
|
||||||
@@ -333,7 +333,7 @@ class MoRIIOConnectorScheduler:
|
|||||||
request: "Request",
|
request: "Request",
|
||||||
blocks: "KVCacheBlocks",
|
blocks: "KVCacheBlocks",
|
||||||
num_external_tokens: int,
|
num_external_tokens: int,
|
||||||
connector_worker: Optional["MoRIIOConnectorWorker"] = None,
|
connector_worker: "MoRIIOConnectorWorker | None" = None,
|
||||||
):
|
):
|
||||||
params = request.kv_transfer_params
|
params = request.kv_transfer_params
|
||||||
if not params:
|
if not params:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import threading
|
import threading
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
from weakref import ref as weakref_ref
|
from weakref import ref as weakref_ref
|
||||||
|
|
||||||
import msgpack
|
import msgpack
|
||||||
@@ -340,7 +340,7 @@ class MoRIIOWrapper:
|
|||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
moriio_engine: Optional["IOEngine"] = None,
|
moriio_engine: "IOEngine | None" = None,
|
||||||
tp_rank: int = 0,
|
tp_rank: int = 0,
|
||||||
dp_rank: int = 0,
|
dp_rank: int = 0,
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from collections import defaultdict
|
|||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from concurrent.futures import Future, ThreadPoolExecutor
|
from concurrent.futures import Future, ThreadPoolExecutor
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -302,7 +302,7 @@ class NixlConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(vllm_config, role, kv_cache_config)
|
super().__init__(vllm_config, role, kv_cache_config)
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -76,7 +76,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
|
|||||||
self,
|
self,
|
||||||
vllm_config: "VllmConfig",
|
vllm_config: "VllmConfig",
|
||||||
role: KVConnectorRole,
|
role: KVConnectorRole,
|
||||||
kv_cache_config: Optional["KVCacheConfig"] = None,
|
kv_cache_config: "KVCacheConfig | None" = None,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
vllm_config=vllm_config,
|
vllm_config=vllm_config,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType
|
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType
|
||||||
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||||
@@ -49,7 +49,7 @@ def is_v1_kv_transfer_group(connector: KVConnectorBaseType | None = None) -> boo
|
|||||||
|
|
||||||
|
|
||||||
def ensure_kv_transfer_initialized(
|
def ensure_kv_transfer_initialized(
|
||||||
vllm_config: "VllmConfig", kv_cache_config: Optional["KVCacheConfig"] = None
|
vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig | None" = None
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""
|
||||||
Initialize KV cache transfer parallel group.
|
Initialize KV cache transfer parallel group.
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ from contextlib import contextmanager, nullcontext
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from multiprocessing import shared_memory
|
from multiprocessing import shared_memory
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
@@ -106,7 +106,7 @@ def _get_unique_name(name: str) -> str:
|
|||||||
return newname
|
return newname
|
||||||
|
|
||||||
|
|
||||||
_groups: dict[str, Callable[[], Optional["GroupCoordinator"]]] = {}
|
_groups: dict[str, Callable[[], "GroupCoordinator | None"]] = {}
|
||||||
|
|
||||||
|
|
||||||
def _register_group(group: "GroupCoordinator") -> None:
|
def _register_group(group: "GroupCoordinator") -> None:
|
||||||
@@ -784,7 +784,7 @@ class GroupCoordinator:
|
|||||||
self,
|
self,
|
||||||
tensor_dict: dict[str, torch.Tensor | Any],
|
tensor_dict: dict[str, torch.Tensor | Any],
|
||||||
dst: int | None = None,
|
dst: int | None = None,
|
||||||
all_gather_group: Optional["GroupCoordinator"] = None,
|
all_gather_group: "GroupCoordinator | None" = None,
|
||||||
all_gather_tensors: dict[str, bool] | None = None,
|
all_gather_tensors: dict[str, bool] | None = None,
|
||||||
) -> dict[str, torch.Tensor | Any] | None:
|
) -> dict[str, torch.Tensor | Any] | None:
|
||||||
"""Send the input tensor dictionary.
|
"""Send the input tensor dictionary.
|
||||||
@@ -871,7 +871,7 @@ class GroupCoordinator:
|
|||||||
def recv_tensor_dict(
|
def recv_tensor_dict(
|
||||||
self,
|
self,
|
||||||
src: int | None = None,
|
src: int | None = None,
|
||||||
all_gather_group: Optional["GroupCoordinator"] = None,
|
all_gather_group: "GroupCoordinator | None" = None,
|
||||||
all_gather_tensors: dict[str, bool] | None = None,
|
all_gather_tensors: dict[str, bool] | None = None,
|
||||||
) -> dict[str, torch.Tensor | Any] | None:
|
) -> dict[str, torch.Tensor | Any] | None:
|
||||||
"""Recv the input tensor dictionary.
|
"""Recv the input tensor dictionary.
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
"""Pydantic models for Anthropic API protocol"""
|
"""Pydantic models for Anthropic API protocol"""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
from typing import Any, Literal, Optional
|
from typing import Any, Literal
|
||||||
|
|
||||||
from pydantic import BaseModel, field_validator
|
from pydantic import BaseModel, field_validator
|
||||||
|
|
||||||
@@ -135,7 +135,7 @@ class AnthropicStreamEvent(BaseModel):
|
|||||||
"ping",
|
"ping",
|
||||||
"error",
|
"error",
|
||||||
]
|
]
|
||||||
message: Optional["AnthropicMessagesResponse"] = None
|
message: "AnthropicMessagesResponse | None" = None
|
||||||
delta: AnthropicDelta | None = None
|
delta: AnthropicDelta | None = None
|
||||||
content_block: AnthropicContentBlock | None = None
|
content_block: AnthropicContentBlock | None = None
|
||||||
index: int | None = None
|
index: int | None = None
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from collections.abc import Sequence as GenericSequence
|
from collections.abc import Sequence as GenericSequence
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
import torch.types
|
import torch.types
|
||||||
@@ -126,7 +125,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def pack(
|
def pack(
|
||||||
cls, loras: GenericSequence[Optional["LoRALayerWeights"]]
|
cls, loras: GenericSequence["LoRALayerWeights | None"]
|
||||||
) -> "PackedLoRALayerWeights":
|
) -> "PackedLoRALayerWeights":
|
||||||
"""Pack a list of LoRAs into a single LoRA.
|
"""Pack a list of LoRAs into a single LoRA.
|
||||||
|
|
||||||
@@ -155,7 +154,7 @@ class PackedLoRALayerWeights(LoRALayerWeights):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def pack_moe(
|
def pack_moe(
|
||||||
cls,
|
cls,
|
||||||
loras: GenericSequence[Optional["LoRALayerWeights"]],
|
loras: GenericSequence["LoRALayerWeights | None"],
|
||||||
module_name: str,
|
module_name: str,
|
||||||
is_non_gated_moe: bool = False,
|
is_non_gated_moe: bool = False,
|
||||||
) -> "PackedLoRALayerWeights":
|
) -> "PackedLoRALayerWeights":
|
||||||
|
|||||||
+2
-2
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import huggingface_hub
|
import huggingface_hub
|
||||||
from huggingface_hub.utils import HfHubHTTPError, HFValidationError
|
from huggingface_hub.utils import HfHubHTTPError, HFValidationError
|
||||||
@@ -131,7 +131,7 @@ def replace_submodule(
|
|||||||
|
|
||||||
|
|
||||||
def parse_fine_tuned_lora_name(
|
def parse_fine_tuned_lora_name(
|
||||||
name: str, weights_mapper: Optional["WeightsMapper"] = None
|
name: str, weights_mapper: "WeightsMapper | None" = None
|
||||||
) -> tuple[str, bool]:
|
) -> tuple[str, bool]:
|
||||||
"""Parse the name of lora weights.
|
"""Parse the name of lora weights.
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
from typing import Optional, Union
|
from typing import Union
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -284,7 +284,7 @@ class FusedMoEQuantConfig:
|
|||||||
return self._w1.bias
|
return self._w1.bias
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def w1_precision(self) -> Optional["PrecisionConfig"]:
|
def w1_precision(self) -> "PrecisionConfig | None":
|
||||||
assert self._w1.scale is None or isinstance(self._w1.scale, PrecisionConfig)
|
assert self._w1.scale is None or isinstance(self._w1.scale, PrecisionConfig)
|
||||||
return self._w1.scale
|
return self._w1.scale
|
||||||
|
|
||||||
@@ -306,7 +306,7 @@ class FusedMoEQuantConfig:
|
|||||||
return self._w2.bias
|
return self._w2.bias
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def w2_precision(self) -> Optional["PrecisionConfig"]:
|
def w2_precision(self) -> "PrecisionConfig | None":
|
||||||
assert self._w2.scale is None or isinstance(self._w2.scale, PrecisionConfig)
|
assert self._w2.scale is None or isinstance(self._w2.scale, PrecisionConfig)
|
||||||
return self._w2.scale
|
return self._w2.scale
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||||
@@ -148,7 +148,7 @@ class AWQMarlinConfig(QuantizationConfig):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def override_quantization_method(
|
def override_quantization_method(
|
||||||
cls, hf_quant_cfg, user_quant
|
cls, hf_quant_cfg, user_quant
|
||||||
) -> Optional["QuantizationMethods"]:
|
) -> "QuantizationMethods | None":
|
||||||
can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
|
can_convert = cls.is_awq_marlin_compatible(hf_quant_cfg)
|
||||||
is_valid_user_quant = (
|
is_valid_user_quant = (
|
||||||
user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"
|
user_quant is None or user_quant == "marlin" or user_quant == "awq_marlin"
|
||||||
@@ -173,7 +173,7 @@ class AWQMarlinConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase) or (
|
if isinstance(layer, LinearBase) or (
|
||||||
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
|
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import TYPE_CHECKING, Any, Literal, Optional, cast
|
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from compressed_tensors.config import (
|
from compressed_tensors.config import (
|
||||||
@@ -160,7 +160,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
|||||||
self,
|
self,
|
||||||
layer: torch.nn.Module,
|
layer: torch.nn.Module,
|
||||||
prefix: str,
|
prefix: str,
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
# collect schemes
|
# collect schemes
|
||||||
quant_scheme = self.get_scheme(layer=layer, layer_name=prefix)
|
quant_scheme = self.get_scheme(layer=layer, layer_name=prefix)
|
||||||
@@ -691,7 +691,7 @@ class CompressedTensorsConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_scheme(
|
def get_scheme(
|
||||||
self, layer: torch.nn.Module, layer_name: str | None = None
|
self, layer: torch.nn.Module, layer_name: str | None = None
|
||||||
) -> Optional["CompressedTensorsScheme"]:
|
) -> "CompressedTensorsScheme | None":
|
||||||
"""
|
"""
|
||||||
compressed-tensors supports non uniform in the following way:
|
compressed-tensors supports non uniform in the following way:
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||||
@@ -105,7 +105,7 @@ class CPUAWQConfig(QuantizationConfig):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def override_quantization_method(
|
def override_quantization_method(
|
||||||
cls, hf_quant_cfg, user_quant
|
cls, hf_quant_cfg, user_quant
|
||||||
) -> Optional["QuantizationMethods"]:
|
) -> "QuantizationMethods | None":
|
||||||
quant_method = hf_quant_cfg.get("quant_method", "").lower()
|
quant_method = hf_quant_cfg.get("quant_method", "").lower()
|
||||||
if current_platform.is_cpu() and (quant_method == "awq"):
|
if current_platform.is_cpu() and (quant_method == "awq"):
|
||||||
return cls.get_name()
|
return cls.get_name()
|
||||||
@@ -113,7 +113,7 @@ class CPUAWQConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase) or (
|
if isinstance(layer, LinearBase) or (
|
||||||
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
|
isinstance(layer, ParallelLMHead) and self.lm_head_quantized
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -52,7 +52,7 @@ class ExpertsInt8Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
return UnquantizedLinearMethod()
|
return UnquantizedLinearMethod()
|
||||||
elif isinstance(layer, FusedMoE):
|
elif isinstance(layer, FusedMoE):
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch.nn import Module
|
from torch.nn import Module
|
||||||
@@ -78,7 +78,7 @@ class FBGEMMFp8Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
if is_layer_skipped(
|
if is_layer_skipped(
|
||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch.nn import Module
|
from torch.nn import Module
|
||||||
@@ -182,7 +182,7 @@ class Fp8Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_xpu_quant_method(
|
def get_xpu_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
from vllm.model_executor.layers.quantization.ipex_quant import (
|
from vllm.model_executor.layers.quantization.ipex_quant import (
|
||||||
XPUFp8LinearMethod,
|
XPUFp8LinearMethod,
|
||||||
XPUFp8MoEMethod,
|
XPUFp8MoEMethod,
|
||||||
@@ -218,7 +218,7 @@ class Fp8Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if current_platform.is_xpu():
|
if current_platform.is_xpu():
|
||||||
return self.get_xpu_quant_method(layer, prefix)
|
return self.get_xpu_quant_method(layer, prefix)
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from types import MappingProxyType
|
from types import MappingProxyType
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import gguf
|
import gguf
|
||||||
import torch
|
import torch
|
||||||
@@ -77,7 +77,7 @@ class GGUFConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
if is_layer_skipped_gguf(
|
if is_layer_skipped_gguf(
|
||||||
prefix, self.unquantized_modules, self.packed_modules_mapping
|
prefix, self.unquantized_modules, self.packed_modules_mapping
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
|
||||||
@@ -240,7 +240,7 @@ class GPTQMarlinConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, FusedMoE):
|
if isinstance(layer, FusedMoE):
|
||||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from fractions import Fraction
|
from fractions import Fraction
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -456,7 +456,7 @@ class INCConfig(QuantizationConfig):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def override_quantization_method(
|
def override_quantization_method(
|
||||||
cls, hf_quant_cfg, user_quant
|
cls, hf_quant_cfg, user_quant
|
||||||
) -> Optional["QuantizationMethods"]:
|
) -> "QuantizationMethods | None":
|
||||||
"""Override the `auto-round` method to `inc`."""
|
"""Override the `auto-round` method to `inc`."""
|
||||||
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
|
is_auto_round_format = hf_quant_cfg.get("quant_method", None) == "auto-round"
|
||||||
if is_auto_round_format:
|
if is_auto_round_format:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from packaging import version
|
from packaging import version
|
||||||
@@ -144,7 +144,7 @@ class IPEXConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["LinearMethodBase"]:
|
) -> "LinearMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
if self.method == "awq":
|
if self.method == "awq":
|
||||||
if is_layer_skipped(
|
if is_layer_skipped(
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from fnmatch import fnmatch
|
from fnmatch import fnmatch
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch.nn import Module
|
from torch.nn import Module
|
||||||
@@ -181,7 +181,7 @@ class ModelOptQuantConfigBase(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
# handle kv-cache first so we can focus only on weight quantization thereafter
|
# handle kv-cache first so we can focus only on weight quantization thereafter
|
||||||
if isinstance(layer, Attention):
|
if isinstance(layer, Attention):
|
||||||
return self.KVCacheMethodCls(self)
|
return self.KVCacheMethodCls(self)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -163,7 +163,7 @@ class MoeWNA16Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
|
if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
|
||||||
if isinstance(layer, FusedMoE):
|
if isinstance(layer, FusedMoE):
|
||||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch.nn.parameter import Parameter
|
from torch.nn.parameter import Parameter
|
||||||
@@ -197,7 +196,7 @@ class Mxfp4Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
if self.ignored_layers and is_layer_skipped(
|
if self.ignored_layers and is_layer_skipped(
|
||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/modelopt.py
|
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/modelopt.py
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -159,7 +159,7 @@ class PetitNvFp4Config(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
exclude = self.require_exclude_modules()
|
exclude = self.require_exclude_modules()
|
||||||
|
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch.nn.parameter import Parameter
|
from torch.nn.parameter import Parameter
|
||||||
@@ -67,7 +67,7 @@ class PTPCFp8Config(Fp8Config):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if isinstance(layer, LinearBase):
|
if isinstance(layer, LinearBase):
|
||||||
if is_layer_skipped(prefix, self.ignored_layers):
|
if is_layer_skipped(prefix, self.ignored_layers):
|
||||||
return UnquantizedLinearMethod()
|
return UnquantizedLinearMethod()
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import fnmatch
|
import fnmatch
|
||||||
from typing import TYPE_CHECKING, Any, Optional, cast
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -102,7 +102,7 @@ class QuarkConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
# Check if the layer is skipped for quantization.
|
# Check if the layer is skipped for quantization.
|
||||||
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
|
exclude_layers = cast(list[str], self.quant_config.get("exclude"))
|
||||||
if should_ignore_layer(
|
if should_ignore_layer(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import importlib
|
|||||||
import json
|
import json
|
||||||
import types
|
import types
|
||||||
from importlib.util import find_spec
|
from importlib.util import find_spec
|
||||||
from typing import Any, Optional
|
from typing import Any
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -209,7 +209,7 @@ class TorchAOConfig(QuantizationConfig):
|
|||||||
|
|
||||||
def get_quant_method(
|
def get_quant_method(
|
||||||
self, layer: torch.nn.Module, prefix: str
|
self, layer: torch.nn.Module, prefix: str
|
||||||
) -> Optional["QuantizeMethodBase"]:
|
) -> "QuantizeMethodBase | None":
|
||||||
if not isinstance(layer, LinearBase):
|
if not isinstance(layer, LinearBase):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -9,7 +9,7 @@ if TYPE_CHECKING:
|
|||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
|
|
||||||
# 1. Create a global variable as a placeholder for the module
|
# 1. Create a global variable as a placeholder for the module
|
||||||
_petit_kernel: Optional["ModuleType"] = None
|
_petit_kernel: "ModuleType | None" = None
|
||||||
|
|
||||||
_PETIT_INSTALL_MSG = (
|
_PETIT_INSTALL_MSG = (
|
||||||
"Petit is not installed. Please install it with `pip install petit-kernel`."
|
"Petit is not installed. Please install it with `pip install petit-kernel`."
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from collections.abc import Generator, MutableMapping
|
from collections.abc import Generator, MutableMapping
|
||||||
from dataclasses import asdict, dataclass, field, fields
|
from dataclasses import asdict, dataclass, field, fields
|
||||||
from typing import TYPE_CHECKING, Any, ClassVar, Optional
|
from typing import TYPE_CHECKING, Any, ClassVar
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
import torch
|
import torch
|
||||||
@@ -323,7 +323,7 @@ class TensorizerConfig(MutableMapping):
|
|||||||
" is unstable and may lead to errors."
|
" is unstable and may lead to errors."
|
||||||
)
|
)
|
||||||
|
|
||||||
def open_stream(self, tensorizer_args: Optional["TensorizerArgs"] = None):
|
def open_stream(self, tensorizer_args: "TensorizerArgs | None" = None):
|
||||||
if tensorizer_args is None:
|
if tensorizer_args is None:
|
||||||
tensorizer_args = self._construct_tensorizer_args()
|
tensorizer_args = self._construct_tensorizer_args()
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from typing import (
|
|||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Literal,
|
Literal,
|
||||||
Optional,
|
|
||||||
TypeAlias,
|
TypeAlias,
|
||||||
TypedDict,
|
TypedDict,
|
||||||
Union,
|
Union,
|
||||||
@@ -186,7 +185,7 @@ class PlaceholderRange:
|
|||||||
length: int
|
length: int
|
||||||
"""The length of the placeholder."""
|
"""The length of the placeholder."""
|
||||||
|
|
||||||
is_embed: Optional["torch.Tensor"] = None
|
is_embed: "torch.Tensor | None" = None
|
||||||
"""
|
"""
|
||||||
A boolean mask of shape `(length,)` indicating which positions
|
A boolean mask of shape `(length,)` indicating which positions
|
||||||
between `offset` and `offset + length` to assign embeddings to.
|
between `offset` and `offset + length` to assign embeddings to.
|
||||||
@@ -341,7 +340,7 @@ class MultiModalFeatureSpec:
|
|||||||
`MultiModalFeatureSpec` per item.
|
`MultiModalFeatureSpec` per item.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
data: Optional["MultiModalKwargsItem"]
|
data: "MultiModalKwargsItem | None"
|
||||||
"""
|
"""
|
||||||
Represents multimodal data for this feature.
|
Represents multimodal data for this feature.
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ pynvml. However, it should not initialize cuda context.
|
|||||||
import os
|
import os
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from functools import cache, wraps
|
from functools import cache, wraps
|
||||||
from typing import TYPE_CHECKING, Optional, TypeVar
|
from typing import TYPE_CHECKING, TypeVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from typing_extensions import ParamSpec
|
from typing_extensions import ParamSpec
|
||||||
@@ -382,7 +382,7 @@ class CudaPlatformBase(Platform):
|
|||||||
cls,
|
cls,
|
||||||
head_size: int,
|
head_size: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
backend: Optional["AttentionBackendEnum"] = None,
|
backend: "AttentionBackendEnum | None" = None,
|
||||||
) -> "AttentionBackendEnum":
|
) -> "AttentionBackendEnum":
|
||||||
if backend is not None:
|
if backend is not None:
|
||||||
assert backend in cls.get_supported_vit_attn_backends(), (
|
assert backend in cls.get_supported_vit_attn_backends(), (
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import platform
|
|||||||
import random
|
import random
|
||||||
import sys
|
import sys
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
|
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -243,7 +243,7 @@ class Platform:
|
|||||||
cls,
|
cls,
|
||||||
head_size: int,
|
head_size: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
backend: Optional["AttentionBackendEnum"] = None,
|
backend: "AttentionBackendEnum | None" = None,
|
||||||
) -> "AttentionBackendEnum":
|
) -> "AttentionBackendEnum":
|
||||||
"""
|
"""
|
||||||
Get the vision attention backend class of a device.
|
Get the vision attention backend class of a device.
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
from functools import cache, lru_cache, wraps
|
from functools import cache, lru_cache, wraps
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -356,7 +356,7 @@ class RocmPlatform(Platform):
|
|||||||
cls,
|
cls,
|
||||||
head_size: int,
|
head_size: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
backend: Optional["AttentionBackendEnum"] = None,
|
backend: "AttentionBackendEnum | None" = None,
|
||||||
) -> "AttentionBackendEnum":
|
) -> "AttentionBackendEnum":
|
||||||
if backend is not None:
|
if backend is not None:
|
||||||
assert backend in cls.get_supported_vit_attn_backends(), (
|
assert backend in cls.get_supported_vit_attn_backends(), (
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
|
|
||||||
import contextlib
|
import contextlib
|
||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -88,7 +88,7 @@ class XPUPlatform(Platform):
|
|||||||
cls,
|
cls,
|
||||||
head_size: int,
|
head_size: int,
|
||||||
dtype: torch.dtype,
|
dtype: torch.dtype,
|
||||||
backend: Optional["AttentionBackendEnum"] = None,
|
backend: "AttentionBackendEnum | None" = None,
|
||||||
) -> "AttentionBackendEnum":
|
) -> "AttentionBackendEnum":
|
||||||
if backend is not None:
|
if backend is not None:
|
||||||
assert backend in cls.get_supported_vit_attn_backends(), (
|
assert backend in cls.get_supported_vit_attn_backends(), (
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from typing import Annotated, Any, Optional
|
from typing import Annotated, Any
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
|
|
||||||
@@ -80,7 +80,7 @@ class PoolingParams(
|
|||||||
return deepcopy(self)
|
return deepcopy(self)
|
||||||
|
|
||||||
def verify(
|
def verify(
|
||||||
self, task: PoolingTask, model_config: Optional["ModelConfig"] = None
|
self, task: PoolingTask, model_config: "ModelConfig | None" = None
|
||||||
) -> None:
|
) -> None:
|
||||||
if self.task is None:
|
if self.task is None:
|
||||||
self.task = task
|
self.task = task
|
||||||
@@ -106,7 +106,7 @@ class PoolingParams(
|
|||||||
self._verify_valid_parameters()
|
self._verify_valid_parameters()
|
||||||
|
|
||||||
def _merge_default_parameters(
|
def _merge_default_parameters(
|
||||||
self, model_config: Optional["ModelConfig"] = None
|
self, model_config: "ModelConfig | None" = None
|
||||||
) -> None:
|
) -> None:
|
||||||
if model_config is None:
|
if model_config is None:
|
||||||
return
|
return
|
||||||
@@ -160,7 +160,7 @@ class PoolingParams(
|
|||||||
if getattr(self, k, None) is None:
|
if getattr(self, k, None) is None:
|
||||||
setattr(self, k, getattr(pooler_config, k))
|
setattr(self, k, getattr(pooler_config, k))
|
||||||
|
|
||||||
def _set_default_parameters(self, model_config: Optional["ModelConfig"]):
|
def _set_default_parameters(self, model_config: "ModelConfig | None"):
|
||||||
if self.task in ["embed", "token_embed"]:
|
if self.task in ["embed", "token_embed"]:
|
||||||
if self.use_activation is None:
|
if self.use_activation is None:
|
||||||
self.use_activation = True
|
self.use_activation = True
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import copy
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass, field
|
||||||
from typing import Any, Optional, TypeAlias
|
from typing import Any, TypeAlias
|
||||||
|
|
||||||
from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
|
from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
|
||||||
from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent
|
from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent
|
||||||
@@ -31,7 +31,7 @@ except ImportError:
|
|||||||
@dataclass
|
@dataclass
|
||||||
class _ModuleTreeNode:
|
class _ModuleTreeNode:
|
||||||
event: _ProfilerEvent
|
event: _ProfilerEvent
|
||||||
parent: Optional["_ModuleTreeNode"] = None
|
parent: "_ModuleTreeNode | None" = None
|
||||||
children: list["_ModuleTreeNode"] = field(default_factory=list)
|
children: list["_ModuleTreeNode"] = field(default_factory=list)
|
||||||
trace: str = ""
|
trace: str = ""
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import fnmatch
|
import fnmatch
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.utils.import_utils import PlaceholderModule
|
from vllm.utils.import_utils import PlaceholderModule
|
||||||
|
|
||||||
@@ -32,7 +32,7 @@ def _filter_ignore(paths: list[str], patterns: list[str]) -> list[str]:
|
|||||||
|
|
||||||
|
|
||||||
def glob(
|
def glob(
|
||||||
s3: Optional["BaseClient"] = None,
|
s3: "BaseClient | None" = None,
|
||||||
path: str = "",
|
path: str = "",
|
||||||
allow_pattern: list[str] | None = None,
|
allow_pattern: list[str] | None = None,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, ClassVar, Optional
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -707,7 +707,7 @@ class FlashMLASparseImpl(MLACommonBaseImpl[FlashMLASparseMetadata]):
|
|||||||
kv_sharing_target_layer_name: str | None,
|
kv_sharing_target_layer_name: str | None,
|
||||||
# MLA Specific Arguments
|
# MLA Specific Arguments
|
||||||
topk_indice_buffer: torch.Tensor | None = None,
|
topk_indice_buffer: torch.Tensor | None = None,
|
||||||
indexer: Optional["Indexer"] = None,
|
indexer: "Indexer | None" = None,
|
||||||
**mla_args,
|
**mla_args,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(
|
super().__init__(
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import TYPE_CHECKING, ClassVar, Optional
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
@@ -284,7 +284,7 @@ class ROCMAiterMLASparseImpl(MLACommonBaseImpl[ROCMAiterMLASparseMetadata]):
|
|||||||
kv_sharing_target_layer_name: str | None,
|
kv_sharing_target_layer_name: str | None,
|
||||||
# MLA Specific Arguments
|
# MLA Specific Arguments
|
||||||
topk_indice_buffer: torch.Tensor | None = None,
|
topk_indice_buffer: torch.Tensor | None = None,
|
||||||
indexer: Optional["Indexer"] = None,
|
indexer: "Indexer | None" = None,
|
||||||
**mla_args,
|
**mla_args,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(
|
super().__init__(
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
import ast
|
import ast
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import ClassVar, Optional
|
from typing import ClassVar
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -87,11 +87,11 @@ class TreeAttentionMetadata:
|
|||||||
tree_attn_bias: torch.Tensor | None = None
|
tree_attn_bias: torch.Tensor | None = None
|
||||||
|
|
||||||
# Cached Prefill/decode metadata.
|
# Cached Prefill/decode metadata.
|
||||||
_cached_prefill_metadata: Optional["TreeAttentionMetadata"] = None
|
_cached_prefill_metadata: "TreeAttentionMetadata | None" = None
|
||||||
_cached_decode_metadata: Optional["TreeAttentionMetadata"] = None
|
_cached_decode_metadata: "TreeAttentionMetadata | None" = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def prefill_metadata(self) -> Optional["TreeAttentionMetadata"]:
|
def prefill_metadata(self) -> "TreeAttentionMetadata | None":
|
||||||
if self.num_prefills == 0:
|
if self.num_prefills == 0:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ class TreeAttentionMetadata:
|
|||||||
return self._cached_prefill_metadata
|
return self._cached_prefill_metadata
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def decode_metadata(self) -> Optional["TreeAttentionMetadata"]:
|
def decode_metadata(self) -> "TreeAttentionMetadata | None":
|
||||||
if self.num_decode_tokens == 0:
|
if self.num_decode_tokens == 0:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
|
||||||
|
|
||||||
@@ -189,7 +189,7 @@ class SchedulerInterface(ABC):
|
|||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def make_stats(self) -> Optional["SchedulerStats"]:
|
def make_stats(self) -> "SchedulerStats | None":
|
||||||
"""Make a SchedulerStats object for logging.
|
"""Make a SchedulerStats object for logging.
|
||||||
|
|
||||||
The SchedulerStats object is created for every scheduling step.
|
The SchedulerStats object is created for every scheduling step.
|
||||||
@@ -201,5 +201,5 @@ class SchedulerInterface(ABC):
|
|||||||
"""Shutdown the scheduler."""
|
"""Shutdown the scheduler."""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def get_kv_connector(self) -> Optional["KVConnectorBase_V1"]:
|
def get_kv_connector(self) -> "KVConnectorBase_V1 | None":
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from copy import copy
|
from copy import copy
|
||||||
from typing import Optional, cast
|
from typing import cast
|
||||||
|
|
||||||
from vllm.outputs import CompletionOutput
|
from vllm.outputs import CompletionOutput
|
||||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||||
@@ -133,7 +133,7 @@ class ParentRequest:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def observe_finished_request(
|
def observe_finished_request(
|
||||||
parent_req: Optional["ParentRequest"],
|
parent_req: "ParentRequest | None",
|
||||||
iteration_stats: IterationStats,
|
iteration_stats: IterationStats,
|
||||||
num_generation_tokens: int,
|
num_generation_tokens: int,
|
||||||
):
|
):
|
||||||
|
|||||||
+2
-2
@@ -7,7 +7,7 @@ from collections import deque
|
|||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import TYPE_CHECKING, Any, Optional
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -68,7 +68,7 @@ class Request:
|
|||||||
arrival_time: float | None = None,
|
arrival_time: float | None = None,
|
||||||
prompt_embeds: torch.Tensor | None = None,
|
prompt_embeds: torch.Tensor | None = None,
|
||||||
mm_features: list[MultiModalFeatureSpec] | None = None,
|
mm_features: list[MultiModalFeatureSpec] | None = None,
|
||||||
lora_request: Optional["LoRARequest"] = None,
|
lora_request: "LoRARequest | None" = None,
|
||||||
cache_salt: str | None = None,
|
cache_salt: str | None = None,
|
||||||
priority: int = 0,
|
priority: int = 0,
|
||||||
trace_headers: Mapping[str, str] | None = None,
|
trace_headers: Mapping[str, str] | None = None,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum, auto
|
from enum import Enum, auto
|
||||||
from typing import TYPE_CHECKING, Optional
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -94,7 +94,7 @@ class LogitsProcessor(ABC):
|
|||||||
@abstractmethod
|
@abstractmethod
|
||||||
def update_state(
|
def update_state(
|
||||||
self,
|
self,
|
||||||
batch_update: Optional["BatchUpdate"],
|
batch_update: "BatchUpdate | None",
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Called when there are new output tokens, prior
|
"""Called when there are new output tokens, prior
|
||||||
to each forward pass.
|
to each forward pass.
|
||||||
|
|||||||
+1
-2
@@ -14,7 +14,6 @@ from typing import (
|
|||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Generic,
|
Generic,
|
||||||
Optional,
|
|
||||||
TypeVar,
|
TypeVar,
|
||||||
Union,
|
Union,
|
||||||
overload,
|
overload,
|
||||||
@@ -229,7 +228,7 @@ def wait_for_completion_or_failure(
|
|||||||
api_server_manager: APIServerProcessManager,
|
api_server_manager: APIServerProcessManager,
|
||||||
engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]
|
engine_manager: Union["CoreEngineProcManager", "CoreEngineActorManager"]
|
||||||
| None = None,
|
| None = None,
|
||||||
coordinator: Optional["DPCoordinator"] = None,
|
coordinator: "DPCoordinator | None" = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Wait for all processes to complete or detect if any fail.
|
"""Wait for all processes to complete or detect if any fail.
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
import threading
|
import threading
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -15,7 +14,7 @@ logger = init_logger(__name__)
|
|||||||
_THREAD_ID_TO_CONTEXT: dict = {}
|
_THREAD_ID_TO_CONTEXT: dict = {}
|
||||||
# Here we hardcode the number of microbatches to 2 for default.
|
# Here we hardcode the number of microbatches to 2 for default.
|
||||||
_NUM_UBATCHES: int = 2
|
_NUM_UBATCHES: int = 2
|
||||||
_CURRENT_CONTEXTS: list[Optional["UBatchContext"]] = []
|
_CURRENT_CONTEXTS: list["UBatchContext | None"] = []
|
||||||
|
|
||||||
|
|
||||||
class UBatchContext:
|
class UBatchContext:
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import inspect
|
|||||||
import os
|
import os
|
||||||
from itertools import accumulate
|
from itertools import accumulate
|
||||||
from math import prod
|
from math import prod
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -26,7 +25,7 @@ _MB = 1024**2
|
|||||||
_GiB = 1024**3
|
_GiB = 1024**3
|
||||||
|
|
||||||
# Global workspace manager instance
|
# Global workspace manager instance
|
||||||
_manager: Optional["WorkspaceManager"] = None
|
_manager: "WorkspaceManager | None" = None
|
||||||
|
|
||||||
|
|
||||||
class WorkspaceManager:
|
class WorkspaceManager:
|
||||||
|
|||||||
Reference in New Issue
Block a user