Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -34,6 +34,7 @@ def human_readable_int(value: str) -> int:
|
||||
- '1k' -> 1,000
|
||||
- '1K' -> 1,024
|
||||
- '25.6k' -> 25,600
|
||||
|
||||
"""
|
||||
value = value.strip()
|
||||
|
||||
@@ -82,6 +83,7 @@ def human_readable_int_or_auto(value: str) -> int:
|
||||
- '1K' -> 1,024
|
||||
- '25.6k' -> 25,600
|
||||
- '-1' or 'auto' -> -1 (special value for auto-detection)
|
||||
|
||||
"""
|
||||
value = value.strip()
|
||||
|
||||
@@ -95,8 +97,7 @@ class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpForma
|
||||
"""SortedHelpFormatter that sorts arguments by their option strings."""
|
||||
|
||||
def _split_lines(self, text, width):
|
||||
"""
|
||||
1. Sentences split across lines have their single newlines removed.
|
||||
"""1. Sentences split across lines have their single newlines removed.
|
||||
2. Paragraphs and lists are split into separate lines.
|
||||
3. Each line is wrapped to the specified width (width of terminal).
|
||||
"""
|
||||
@@ -466,7 +467,7 @@ class FlexibleArgumentParser(ArgumentParser):
|
||||
The arguments in config file will be inserted between
|
||||
the argument list.
|
||||
|
||||
example:
|
||||
Example:
|
||||
```yaml
|
||||
port: 12323
|
||||
tensor-parallel-size: 4
|
||||
@@ -492,6 +493,7 @@ class FlexibleArgumentParser(ArgumentParser):
|
||||
Please note how the config args are inserted after the sub command.
|
||||
this way the order of priorities is maintained when these are args
|
||||
parsed by super().
|
||||
|
||||
"""
|
||||
assert args.count("--config") <= 1, "More than one config file specified!"
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Contains helpers related to asynchronous code.
|
||||
"""Contains helpers related to asynchronous code.
|
||||
|
||||
This is similar in concept to the `asyncio` module.
|
||||
"""
|
||||
@@ -29,8 +28,7 @@ def make_async(
|
||||
func: Callable[P, T],
|
||||
executor: Executor | None = None,
|
||||
) -> Callable[P, Awaitable[T]]:
|
||||
"""
|
||||
Take a blocking function, and run it on in an executor thread.
|
||||
"""Take a blocking function, and run it on in an executor thread.
|
||||
|
||||
This function prevents the blocking function from blocking the
|
||||
asyncio event loop.
|
||||
@@ -49,8 +47,7 @@ def make_async_with_semaphore(
|
||||
func: Callable[P, T],
|
||||
executor: ThreadPoolExecutor,
|
||||
) -> Callable[P, Awaitable[T]]:
|
||||
"""
|
||||
Take a blocking function, and run it on in an executor thread.
|
||||
"""Take a blocking function, and run it on in an executor thread.
|
||||
|
||||
This function prevents the blocking function from blocking the
|
||||
asyncio event loop.
|
||||
@@ -59,7 +56,6 @@ def make_async_with_semaphore(
|
||||
The function is wrapped in a semaphore to limit the number of
|
||||
concurrent executions making it easier to cancel tasks before they start.
|
||||
"""
|
||||
|
||||
semaphore = asyncio.Semaphore(executor._max_workers)
|
||||
|
||||
async def _async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
|
||||
|
||||
+4
-6
@@ -102,8 +102,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
|
||||
return self.currsize / self.maxsize
|
||||
|
||||
def stat(self, *, delta: bool = False) -> CacheInfo:
|
||||
"""
|
||||
Gets the cumulative number of hits and queries against this cache.
|
||||
"""Gets the cumulative number of hits and queries against this cache.
|
||||
|
||||
If `delta=True`, instead gets these statistics
|
||||
since the last call that also passed `delta=True`.
|
||||
@@ -165,6 +164,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
|
||||
|
||||
Returns:
|
||||
`True` if the item was cached, otherwise `False`.
|
||||
|
||||
"""
|
||||
try:
|
||||
self[key] = value
|
||||
@@ -175,8 +175,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
|
||||
return True
|
||||
|
||||
def pin(self, key: _K) -> None:
|
||||
"""
|
||||
Pins a key in the cache preventing it from being
|
||||
"""Pins a key in the cache preventing it from being
|
||||
evicted in the LRU order.
|
||||
"""
|
||||
if key not in self:
|
||||
@@ -184,8 +183,7 @@ class LRUCache(cachetools.LRUCache[_K, _V]):
|
||||
self.pinned_items.add(key)
|
||||
|
||||
def _unpin(self, key: _K) -> None:
|
||||
"""
|
||||
Unpins a key in the cache allowing it to be
|
||||
"""Unpins a key in the cache allowing it to be
|
||||
evicted in the LRU order.
|
||||
"""
|
||||
self.pinned_items.remove(key)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Contains helpers that are applied to collections.
|
||||
"""Contains helpers that are applied to collections.
|
||||
|
||||
This is similar in concept to the `collections` module.
|
||||
"""
|
||||
@@ -20,8 +19,7 @@ _V = TypeVar("_V")
|
||||
|
||||
|
||||
class LazyDict(Mapping[str, _V], Generic[_V]):
|
||||
"""
|
||||
Evaluates dictionary items only when they are accessed.
|
||||
"""Evaluates dictionary items only when they are accessed.
|
||||
|
||||
Adapted from: https://stackoverflow.com/a/47212782/5082708
|
||||
"""
|
||||
@@ -116,8 +114,7 @@ def flatten_2d_lists(lists: Iterable[Iterable[T]]) -> list[T]:
|
||||
|
||||
|
||||
def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
|
||||
"""
|
||||
Unlike [`itertools.groupby`][], groups are not broken by
|
||||
"""Unlike [`itertools.groupby`][], groups are not broken by
|
||||
non-contiguous data.
|
||||
"""
|
||||
groups = defaultdict[_K, list[_V]](list)
|
||||
|
||||
@@ -19,10 +19,10 @@ class Counter:
|
||||
|
||||
|
||||
class AtomicCounter:
|
||||
"""An atomic, thread-safe counter"""
|
||||
"""An atomic, thread-safe counter."""
|
||||
|
||||
def __init__(self, initial: int = 0) -> None:
|
||||
"""Initialize a new atomic counter to given initial value"""
|
||||
"""Initialize a new atomic counter to given initial value."""
|
||||
super().__init__()
|
||||
|
||||
self._value = initial
|
||||
@@ -33,13 +33,13 @@ class AtomicCounter:
|
||||
return self._value
|
||||
|
||||
def inc(self, num: int = 1) -> int:
|
||||
"""Atomically increment the counter by num and return the new value"""
|
||||
"""Atomically increment the counter by num and return the new value."""
|
||||
with self._lock:
|
||||
self._value += num
|
||||
return self._value
|
||||
|
||||
def dec(self, num: int = 1) -> int:
|
||||
"""Atomically decrement the counter by num and return the new value"""
|
||||
"""Atomically decrement the counter by num and return the new value."""
|
||||
with self._lock:
|
||||
self._value -= num
|
||||
return self._value
|
||||
|
||||
@@ -128,6 +128,7 @@ def check_cgroup_memory_available(
|
||||
Low headroom logs a warning, but does not reject the allocation because
|
||||
cgroup usage can include reclaimable memory. If the cgroup limit or usage
|
||||
cannot be read, the check is skipped.
|
||||
|
||||
"""
|
||||
cgroup_limit = get_cgroup_memory_limit()
|
||||
cgroup_usage = get_cgroup_memory_usage()
|
||||
@@ -171,7 +172,7 @@ def get_memory_affinity(pid: int = 0) -> list[int]:
|
||||
|
||||
|
||||
def parse_id_list(raw_str: str) -> list[int]:
|
||||
"""Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8]"""
|
||||
"""Parses strings like '0-2,4,7-8' into [0, 1, 2, 4, 7, 8]."""
|
||||
result: list[int] = []
|
||||
if not raw_str:
|
||||
return result
|
||||
@@ -273,7 +274,8 @@ def get_visible_memory_node() -> list[int]:
|
||||
def _synthesize_cpu_list() -> list[LogicalCPUInfo]:
|
||||
"""Synthesize a flat CPU list: each logical CPU is its own core on
|
||||
NUMA node 0. Used when lscpu output is unavailable or unparsable
|
||||
(e.g. macOS, RISC-V)."""
|
||||
(e.g. macOS, RISC-V).
|
||||
"""
|
||||
cpu_count = os.cpu_count()
|
||||
assert cpu_count
|
||||
return [LogicalCPUInfo(i, i, 0) for i in range(cpu_count)]
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Contains replacement functions to fallback Triton usages in CPU backend
|
||||
"""
|
||||
"""Contains replacement functions to fallback Triton usages in CPU backend."""
|
||||
|
||||
import ctypes
|
||||
from collections.abc import Callable
|
||||
|
||||
+14
-6
@@ -64,7 +64,7 @@ class DeepGemmQuantScaleFMT(Enum):
|
||||
|
||||
@classmethod
|
||||
def init_oracle_cache(cls) -> None:
|
||||
"""Initialize the oracle decision and store it in the class cache"""
|
||||
"""Initialize the oracle decision and store it in the class cache."""
|
||||
cached = getattr(cls, "_oracle_cache", None)
|
||||
if cached is not None:
|
||||
return
|
||||
@@ -426,7 +426,7 @@ def mk_alignment_scope(value: int):
|
||||
|
||||
|
||||
def get_col_major_tma_aligned_tensor(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor"""
|
||||
"""Wrapper for DeepGEMM's get_mn_major_tma_aligned_tensor."""
|
||||
_lazy_init()
|
||||
if _get_mn_major_tma_aligned_tensor_impl is None:
|
||||
return _missing()
|
||||
@@ -568,6 +568,7 @@ def fp8_fp4_mqa_logits(
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [M, N], dtype `torch.float32`.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _fp8_fp4_mqa_logits_impl is None:
|
||||
@@ -623,6 +624,7 @@ def get_paged_mqa_logits_metadata(
|
||||
Returns:
|
||||
Tensor of shape [slots + 1, 2] consumed by `fp8_fp4_paged_mqa_logits`
|
||||
to schedule work across SMs.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_paged_mqa_logits_metadata_impl is None:
|
||||
@@ -674,6 +676,7 @@ def fp8_fp4_paged_mqa_logits(
|
||||
Returns:
|
||||
Logits tensor of shape [B * next_n, max_model_len], dtype
|
||||
`torch.float32`.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _fp8_fp4_paged_mqa_logits_impl is None:
|
||||
@@ -701,7 +704,8 @@ def fp8_fp4_paged_mqa_logits(
|
||||
|
||||
def has_deep_gemm_sparse_mqa() -> bool:
|
||||
"""Whether the installed DeepGEMM provides the sparse-indexer kernels
|
||||
(``fp8_fp4_(paged_)sparse_mqa_logits``, added in DeepGEMM 2.8, SM100-only)."""
|
||||
(``fp8_fp4_(paged_)sparse_mqa_logits``, added in DeepGEMM 2.8, SM100-only).
|
||||
"""
|
||||
_lazy_init()
|
||||
return (
|
||||
_fp8_fp4_sparse_mqa_logits_impl is not None
|
||||
@@ -740,6 +744,7 @@ def get_sparse_mqa_logits_metadata(
|
||||
qk_dtype: dtype of the packed Q values (``torch.int8`` for MXFP4,
|
||||
``torch.float8_e4m3fn`` for FP8).
|
||||
sparse_block_kv: Tokens per sparse block, 8 or 16.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_sparse_mqa_logits_metadata_impl is None:
|
||||
@@ -777,6 +782,7 @@ def get_paged_sparse_mqa_logits_metadata(
|
||||
block ids, [num_q_tokens, num_max_sparse_blocks], int32.
|
||||
qk_dtype: dtype of the packed Q values.
|
||||
sparse_block_kv: Tokens per sparse block, 8 or 16.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_paged_sparse_mqa_logits_metadata_impl is None:
|
||||
@@ -817,6 +823,7 @@ def fp8_fp4_sparse_mqa_logits(
|
||||
bf16 logits of shape [M, num_max_sparse_blocks * sparse_block_kv];
|
||||
column ``j * sparse_block_kv + o`` scores the token at
|
||||
``sparse_kv_block_indices[row, j] * sparse_block_kv + ks % sbk + o``.
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _fp8_fp4_sparse_mqa_logits_impl is None:
|
||||
@@ -848,10 +855,13 @@ def fp8_fp4_paged_sparse_mqa_logits(
|
||||
page stride 512B-aligned.
|
||||
weights: [num_q_tokens, H] ``torch.bfloat16``.
|
||||
metadata: From `get_paged_sparse_mqa_logits_metadata`.
|
||||
num_max_sparse_blocks: Candidate blocks per row.
|
||||
sparse_block_kv: Tokens per sparse block, 8 or 16.
|
||||
|
||||
Returns:
|
||||
bf16 logits of shape
|
||||
[num_q_tokens, num_max_sparse_blocks * sparse_block_kv].
|
||||
|
||||
"""
|
||||
_lazy_init()
|
||||
if _fp8_fp4_paged_sparse_mqa_logits_impl is None:
|
||||
@@ -873,8 +883,7 @@ def tf32_hc_prenorm_gemm(
|
||||
sqrsum: torch.Tensor,
|
||||
num_split: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Perform the following computation:
|
||||
"""Perform the following computation:
|
||||
out = x.float() @ fn.T
|
||||
sqrsum = x.float().square().sum(-1)
|
||||
|
||||
@@ -950,7 +959,6 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor):
|
||||
and report `1 - sim`. Once kernel accuracy improves this helper can be
|
||||
removed.
|
||||
"""
|
||||
|
||||
x, y = x.double(), y.double()
|
||||
denominator = (x * x + y * y).sum()
|
||||
sim = 2 * (x * y).sum() / denominator
|
||||
|
||||
@@ -589,8 +589,7 @@ def supports_trtllm_attention(is_prefill: bool = False) -> bool:
|
||||
|
||||
|
||||
def force_use_trtllm_attention() -> bool | None:
|
||||
"""
|
||||
This function should only be called during initialization stage when vllm config
|
||||
"""This function should only be called during initialization stage when vllm config
|
||||
is set.
|
||||
Return `None` if --attention-config.use_trtllm_attention is not set,
|
||||
return `True` if TRTLLM attention is forced to be used,
|
||||
@@ -628,7 +627,6 @@ def use_trtllm_attention(
|
||||
has_spec: bool = False,
|
||||
) -> bool:
|
||||
"""Return `True` if TRTLLM attention is used."""
|
||||
|
||||
# CLI argument is set to 0 - respect it
|
||||
if force_use_trtllm is not None and not force_use_trtllm:
|
||||
return False
|
||||
@@ -736,6 +734,7 @@ if has_flashinfer():
|
||||
k_nope: The nope part of k, shape [num_tokens, num_heads, nope_dim].
|
||||
k_pe: The rope part of k (shared), shape [num_tokens, 1, rope_dim].
|
||||
This is broadcast to all heads.
|
||||
|
||||
"""
|
||||
from flashinfer.concat_ops import concat_mla_k
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Contains helpers that are applied to functions.
|
||||
"""Contains helpers that are applied to functions.
|
||||
|
||||
This is similar in concept to the `functools` module.
|
||||
"""
|
||||
@@ -127,8 +126,7 @@ def get_allowed_kwarg_only_overrides(
|
||||
requires_kw_only: bool = True,
|
||||
allow_var_kwargs: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Given a callable which has one or more keyword only params and a dict
|
||||
"""Given a callable which has one or more keyword only params and a dict
|
||||
mapping param names to values, drop values that can be not be kwarg
|
||||
expanded to overwrite one or more keyword-only args. This is used in a
|
||||
few places to handle custom processor overrides for multimodal models,
|
||||
@@ -139,12 +137,14 @@ def get_allowed_kwarg_only_overrides(
|
||||
callable: Callable which takes 0 or more keyword only arguments.
|
||||
If None is provided, all overrides names are allowed.
|
||||
overrides: Potential overrides to be used when invoking the callable.
|
||||
requires_kw_only: If True, only keyword-only params may be overridden.
|
||||
allow_var_kwargs: Allows overrides that are expandable for var kwargs.
|
||||
|
||||
Returns:
|
||||
Dictionary containing the kwargs to be leveraged which may be used
|
||||
to overwrite one or more keyword only arguments when invoking the
|
||||
callable.
|
||||
|
||||
"""
|
||||
if not overrides:
|
||||
return {}
|
||||
|
||||
+7
-17
@@ -14,8 +14,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GCDebugConfig:
|
||||
"""
|
||||
Config for GC Debugger.
|
||||
"""Config for GC Debugger.
|
||||
- 0: disable GC debugger
|
||||
- 1: enable GC debugger with gc.collect elapsed times
|
||||
- '{"top_objects":5}': enable GC debugger with top 5 collected objects
|
||||
@@ -44,8 +43,7 @@ class GCDebugConfig:
|
||||
|
||||
|
||||
class GCDebugger:
|
||||
"""
|
||||
Debugger for GC which logs helpful information for GC understanding.
|
||||
"""Debugger for GC which logs helpful information for GC understanding.
|
||||
To enable, you should call maybe_attach_gc_debug_callback in the process.
|
||||
"""
|
||||
|
||||
@@ -59,9 +57,7 @@ class GCDebugger:
|
||||
self.gc_top_collected_objects: str = ""
|
||||
|
||||
def handle(self, phase: str, info: dict[str, int]) -> None:
|
||||
"""
|
||||
Handles a GC event (e.g. GC start or GC finish)
|
||||
"""
|
||||
"""Handles a GC event (e.g. GC start or GC finish)."""
|
||||
generation = info.get("generation")
|
||||
if generation is None:
|
||||
return
|
||||
@@ -122,8 +118,7 @@ def freeze_gc_for_cudagraph_capture():
|
||||
|
||||
|
||||
def freeze_gc_heap() -> None:
|
||||
"""
|
||||
Freeze all objects tracked by the garbage collector. It should be invoked
|
||||
"""Freeze all objects tracked by the garbage collector. It should be invoked
|
||||
after server init / warmup, to reduce GC overhead from static objects
|
||||
during serving time.
|
||||
"""
|
||||
@@ -137,9 +132,7 @@ def freeze_gc_heap() -> None:
|
||||
|
||||
|
||||
def maybe_attach_gc_debug_callback() -> None:
|
||||
"""
|
||||
Attached a callback for GC debug when VLLM_GC_DEBUG is enabled.
|
||||
"""
|
||||
"""Attached a callback for GC debug when VLLM_GC_DEBUG is enabled."""
|
||||
config = GCDebugConfig(envs.VLLM_GC_DEBUG)
|
||||
if config.enabled:
|
||||
debugger: GCDebugger = GCDebugger(config)
|
||||
@@ -151,8 +144,7 @@ def maybe_attach_gc_debug_callback() -> None:
|
||||
|
||||
|
||||
def _compute_detailed_type(o: Any) -> str:
|
||||
"""
|
||||
Detailed object type.
|
||||
"""Detailed object type.
|
||||
|
||||
TODO(Jialin): Further enhance the detailed type with element types for
|
||||
easier debugging. We tried but occasionally it would run into signals
|
||||
@@ -167,9 +159,7 @@ def _compute_detailed_type(o: Any) -> str:
|
||||
|
||||
|
||||
def _compute_top_gc_collected_objects(objects: list[Any], top: int) -> str:
|
||||
"""
|
||||
Group collected objects by types.
|
||||
"""
|
||||
"""Group collected objects by types."""
|
||||
if top <= 0:
|
||||
return ""
|
||||
object_types = [_compute_detailed_type(o) for o in objects]
|
||||
|
||||
@@ -72,7 +72,8 @@ _sync_filter_head: tuple | None = None
|
||||
|
||||
def _sync_warning_hook(message, category, filename, lineno, file=None, line=None):
|
||||
"""Turn torch's sync warning into an error, but only on a thread that is
|
||||
being checked and is outside any allow region."""
|
||||
being checked and is outside any allow region.
|
||||
"""
|
||||
if _TORCH_SYNC_WARNING in str(message):
|
||||
mode = _checking.get()
|
||||
if mode is None or _allow_depth.get():
|
||||
@@ -251,7 +252,8 @@ def _is_dense(t: torch.Tensor) -> bool:
|
||||
"""Whether `t`'s elements fill a contiguous storage range, possibly
|
||||
permuted (e.g. a transposed matrix). Such layouts can be copied with
|
||||
pitched cudaMemcpy2D/3DAsync directly from/to pinned memory, whereas
|
||||
gapped layouts (e.g. strided slices) stage through a pageable temp."""
|
||||
gapped layouts (e.g. strided slices) stage through a pageable temp.
|
||||
"""
|
||||
if t.is_contiguous():
|
||||
# The common case, answered by a memoized TensorImpl flag.
|
||||
return True
|
||||
|
||||
@@ -35,6 +35,7 @@ def sha256(input: Any) -> bytes:
|
||||
|
||||
Returns:
|
||||
Bytes representing the SHA-256 hash of the serialized input.
|
||||
|
||||
"""
|
||||
input_bytes = pickle.dumps(input, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
return hashlib.sha256(input_bytes).digest()
|
||||
@@ -53,6 +54,7 @@ def sha256_cbor(input: Any) -> bytes:
|
||||
|
||||
Returns:
|
||||
Bytes representing the SHA-256 hash of the CBOR serialized input.
|
||||
|
||||
"""
|
||||
input_bytes = cbor2.dumps(input, canonical=True)
|
||||
return hashlib.sha256(input_bytes).digest()
|
||||
@@ -87,6 +89,7 @@ def get_hash_fn_by_name(hash_fn_name: str) -> Callable[[Any], bytes]:
|
||||
|
||||
Returns:
|
||||
A hash function.
|
||||
|
||||
"""
|
||||
if hash_fn_name == "sha256":
|
||||
return sha256
|
||||
@@ -110,6 +113,7 @@ def safe_hash(data: bytes, usedforsecurity: bool = True) -> HASH:
|
||||
|
||||
Returns:
|
||||
Hash object
|
||||
|
||||
"""
|
||||
try:
|
||||
return hashlib.md5(data, usedforsecurity=usedforsecurity)
|
||||
|
||||
+10
-22
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Contains helpers related to importing modules.
|
||||
"""Contains helpers related to importing modules.
|
||||
|
||||
This is similar in concept to the `importlib` module.
|
||||
"""
|
||||
@@ -24,8 +23,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def import_pynvml():
|
||||
"""
|
||||
Historical comments:
|
||||
"""Historical comments:
|
||||
|
||||
libnvml.so is the library behind nvidia-smi, and
|
||||
pynvml is a Python wrapper around it. We use it to get GPU
|
||||
@@ -57,8 +55,7 @@ def import_pynvml():
|
||||
|
||||
@cache
|
||||
def import_triton_kernels():
|
||||
"""
|
||||
For convenience, prioritize triton_kernels that is available in
|
||||
"""For convenience, prioritize triton_kernels that is available in
|
||||
`site-packages`. Use `vllm.third_party.triton_kernels` as a fall-back.
|
||||
"""
|
||||
if _has_module("triton_kernels"):
|
||||
@@ -83,8 +80,7 @@ def import_triton_kernels():
|
||||
|
||||
|
||||
def import_plugin(plugin_path: str) -> ModuleType | None:
|
||||
"""
|
||||
Import a user-defined plugin.
|
||||
"""Import a user-defined plugin.
|
||||
|
||||
Plugin can be either:
|
||||
* a module in site-packages
|
||||
@@ -104,8 +100,7 @@ def import_plugin(plugin_path: str) -> ModuleType | None:
|
||||
|
||||
|
||||
def import_from_path(module_name: str, file_path: str | os.PathLike) -> ModuleType:
|
||||
"""
|
||||
Import a Python file according to its file path.
|
||||
"""Import a Python file according to its file path.
|
||||
|
||||
Based on the official recipe:
|
||||
https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
|
||||
@@ -123,9 +118,7 @@ def import_from_path(module_name: str, file_path: str | os.PathLike) -> ModuleTy
|
||||
|
||||
|
||||
def resolve_obj_by_qualname(qualname: str) -> Any:
|
||||
"""
|
||||
Resolve an object by its fully-qualified class name.
|
||||
"""
|
||||
"""Resolve an object by its fully-qualified class name."""
|
||||
module_name, obj_name = qualname.rsplit(".", 1)
|
||||
module = importlib.import_module(module_name)
|
||||
return getattr(module, obj_name)
|
||||
@@ -148,8 +141,7 @@ def get_vllm_optional_dependencies():
|
||||
|
||||
|
||||
class _PlaceholderBase:
|
||||
"""
|
||||
Disallows downstream usage of placeholder modules.
|
||||
"""Disallows downstream usage of placeholder modules.
|
||||
|
||||
We need to explicitly override each dunder method because
|
||||
[`__getattr__`][vllm.utils.import_utils._PlaceholderBase.__getattr__]
|
||||
@@ -160,8 +152,7 @@ class _PlaceholderBase:
|
||||
"""
|
||||
|
||||
def __getattr__(self, key: str) -> Never:
|
||||
"""
|
||||
The main class should implement this to throw an error
|
||||
"""The main class should implement this to throw an error
|
||||
for attribute accesses representing downstream usage.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -304,8 +295,7 @@ class _PlaceholderBase:
|
||||
|
||||
|
||||
class PlaceholderModule(_PlaceholderBase):
|
||||
"""
|
||||
A placeholder object to use when a module does not exist.
|
||||
"""A placeholder object to use when a module does not exist.
|
||||
|
||||
This enables more informative errors when trying to access attributes
|
||||
of a module that does not exist.
|
||||
@@ -360,8 +350,7 @@ class _PlaceholderModuleAttr(_PlaceholderBase):
|
||||
|
||||
|
||||
class LazyLoader(ModuleType):
|
||||
"""
|
||||
`LazyLoader` module borrowed from [Tensorflow]
|
||||
"""`LazyLoader` module borrowed from [Tensorflow]
|
||||
(https://github.com/tensorflow/tensorflow/blob/main/tensorflow/python/util/lazy_loader.py)
|
||||
with an addition of "module caching".
|
||||
|
||||
@@ -562,7 +551,6 @@ def has_tilelang() -> bool:
|
||||
|
||||
def has_arctic_inference() -> bool:
|
||||
"""Whether the optional `arctic_inference` package is available."""
|
||||
|
||||
return _has_module("arctic_inference")
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Monitor unexpected kernel JIT compilation during inference.
|
||||
"""Monitor unexpected kernel JIT compilation during inference.
|
||||
|
||||
After server warmup completes, any kernel JIT compilation or autotuning event
|
||||
indicates a cache miss or unexpected input shape that causes a latency spike.
|
||||
|
||||
@@ -142,8 +142,7 @@ def json_reduce_leaves(
|
||||
initial: _U = ..., # type: ignore[assignment]
|
||||
/,
|
||||
) -> _T | _U:
|
||||
"""
|
||||
Apply a function of two arguments cumulatively to each leaf in a
|
||||
"""Apply a function of two arguments cumulatively to each leaf in a
|
||||
nested JSON structure, from left to right, so as to reduce the
|
||||
sequence to a single value.
|
||||
"""
|
||||
|
||||
@@ -13,7 +13,7 @@ def cdiv(a: int, b: int) -> int:
|
||||
|
||||
|
||||
def next_power_of_2(n: int) -> int:
|
||||
"""The next power of 2 (inclusive)"""
|
||||
"""The next power of 2 (inclusive)."""
|
||||
return 1 if n < 1 else 1 << (n - 1).bit_length()
|
||||
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ def release_device_memory_under_pressure(device: torch.device) -> bool:
|
||||
|
||||
Returns:
|
||||
True if memory was released.
|
||||
|
||||
"""
|
||||
if device.type != "cuda" or not current_platform.is_integrated_gpu(device.index):
|
||||
return False
|
||||
@@ -230,8 +231,7 @@ def memory_profiling(
|
||||
baseline_snapshot: MemorySnapshot,
|
||||
weights_memory: int = 0,
|
||||
) -> Generator[MemoryProfilingResult, None, None]:
|
||||
"""
|
||||
Memory profiling context manager.
|
||||
"""Memory profiling context manager.
|
||||
|
||||
baseline_snapshot: the memory snapshot before the current vLLM instance.
|
||||
weights_memory: memory used by PyTorch when loading the model weights.
|
||||
|
||||
@@ -41,6 +41,7 @@ def maybe_execute_in_parallel(
|
||||
|
||||
Returns:
|
||||
Tuple of (fn0_result, fn1_result).
|
||||
|
||||
"""
|
||||
if aux_stream is not None:
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
@@ -100,6 +101,7 @@ def execute_in_parallel(
|
||||
Returns:
|
||||
Tuple of (default_result, aux_results) where aux_results[i] is the
|
||||
result of aux_fns[i] (or None when skipped).
|
||||
|
||||
"""
|
||||
aux_results: list[Any]
|
||||
if aux_streams is None or not enable:
|
||||
|
||||
@@ -172,8 +172,7 @@ def _get_reserved_port_range() -> range:
|
||||
|
||||
|
||||
def get_open_port() -> int:
|
||||
"""
|
||||
Get an open port for the vLLM process to listen on.
|
||||
"""Get an open port for the vLLM process to listen on.
|
||||
An edge case to handle, is when we run data parallel,
|
||||
we need to avoid ports that are potentially used by
|
||||
the data parallel master process.
|
||||
@@ -298,6 +297,7 @@ def make_zmq_path(scheme: str, host: str, port: int | None = None) -> str:
|
||||
|
||||
Returns:
|
||||
A properly formatted ZMQ path string.
|
||||
|
||||
"""
|
||||
if port is None:
|
||||
return f"{scheme}://{host}"
|
||||
@@ -317,7 +317,6 @@ def make_zmq_socket(
|
||||
router_handover: bool = False,
|
||||
) -> zmq.Socket | zmq.asyncio.Socket: # type: ignore[name-defined]
|
||||
"""Make a ZMQ socket with the proper bind/connect semantics."""
|
||||
|
||||
mem = psutil.virtual_memory()
|
||||
socket = ctx.socket(socket_type)
|
||||
|
||||
@@ -377,8 +376,7 @@ def zmq_socket_ctx(
|
||||
identity: bytes | None = None,
|
||||
router_handover: bool = False,
|
||||
) -> Iterator[zmq.Socket]:
|
||||
"""Context manager for a ZMQ socket"""
|
||||
|
||||
"""Context manager for a ZMQ socket."""
|
||||
ctx = zmq.Context() # type: ignore[attr-defined]
|
||||
try:
|
||||
yield make_zmq_socket(
|
||||
|
||||
@@ -25,7 +25,7 @@ def print_tensor(tensor_obj, prefix, tensor_list=None):
|
||||
|
||||
|
||||
def process_layer_params(module_obj):
|
||||
"""Extract the static parameters from LLM and VLM relevant layer types"""
|
||||
"""Extract the static parameters from LLM and VLM relevant layer types."""
|
||||
param_info = {}
|
||||
# Extract parameters for layers commonly used in LLMs and VLMs
|
||||
if isinstance(module_obj, (torch.nn.Conv1d, torch.nn.Conv2d, torch.nn.Conv3d)):
|
||||
@@ -186,6 +186,7 @@ def layerwise_nvtx_marker_context(module_name, module_obj, in_tensor=None, kwarg
|
||||
kwargs=kwargs) as ctx:
|
||||
ctx.result = module(*args, **kwargs)
|
||||
return ctx.result
|
||||
|
||||
"""
|
||||
holder = ResultHolder()
|
||||
|
||||
@@ -221,9 +222,9 @@ class PytHooks:
|
||||
a PytHook object. Then call the register_hooks method.
|
||||
|
||||
Example:
|
||||
|
||||
my_hook = PytHook()
|
||||
my_hook.register_hooks(my_network_model)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
|
||||
@@ -187,18 +187,19 @@ class OMPProcessManager:
|
||||
def _get_autobind_cpu_ids(
|
||||
self, cpu_selector: Callable[[list[LogicalCPUInfo]], list[LogicalCPUInfo]]
|
||||
) -> tuple[list[list[LogicalCPUInfo]], list[LogicalCPUInfo]]:
|
||||
"""
|
||||
Return CPU ids to bind based on NUMA nodes, and CPU ids reserved for
|
||||
"""Return CPU ids to bind based on NUMA nodes, and CPU ids reserved for
|
||||
other processes.
|
||||
Currently for rank N, only CPU ids on the N-th node in available NUMA
|
||||
node list will be selected.
|
||||
|
||||
Args:
|
||||
cpu_selector: a callable object to select CPUs from a CPU list
|
||||
of a physical core. The input is a LogicalCPUInfo list contains
|
||||
logical CPUs of a physical CPU, sorted by the LogicalCPUInfo.id.
|
||||
A selected LogicalCPUInfo list should be returned.
|
||||
"""
|
||||
of a physical core. The input is a LogicalCPUInfo list contains
|
||||
logical CPUs of a physical CPU, sorted by the
|
||||
LogicalCPUInfo.id. A selected LogicalCPUInfo list should be
|
||||
returned.
|
||||
|
||||
"""
|
||||
# this memory node list has been sliced for DP offset
|
||||
allowed_numa_nodes = cr_utils.get_visible_memory_node()
|
||||
logical_cpu_list = cr_utils.get_allowed_cpu_list()
|
||||
|
||||
@@ -29,7 +29,8 @@ def cuda_get_device_properties(
|
||||
device, names: Sequence[str], init_cuda=False
|
||||
) -> tuple[Any, ...]:
|
||||
"""Get specified CUDA device property values without initializing CUDA in
|
||||
the current process."""
|
||||
the current process.
|
||||
"""
|
||||
if init_cuda or cuda_is_initialized():
|
||||
props = torch.cuda.get_device_properties(device)
|
||||
return tuple(getattr(props, name) for name in names)
|
||||
|
||||
+4
-11
@@ -6,8 +6,7 @@ _T = TypeVar("_T", bound=type)
|
||||
|
||||
|
||||
class ExtensionManager:
|
||||
"""
|
||||
A registry for managing pluggable extension classes.
|
||||
"""A registry for managing pluggable extension classes.
|
||||
|
||||
This class provides a simple mechanism to register and instantiate
|
||||
extension classes by name. It is commonly used to implement plugin
|
||||
@@ -26,15 +25,11 @@ class ExtensionManager:
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""
|
||||
Initialize an empty extension registry.
|
||||
"""
|
||||
"""Initialize an empty extension registry."""
|
||||
self.name2class: dict[str, type] = {}
|
||||
|
||||
def register(self, name: str):
|
||||
"""
|
||||
Decorator to register a class with the given name.
|
||||
"""
|
||||
"""Decorator to register a class with the given name."""
|
||||
|
||||
def wrap(cls_to_register: _T) -> _T:
|
||||
self.name2class[name] = cls_to_register
|
||||
@@ -43,9 +38,7 @@ class ExtensionManager:
|
||||
return wrap
|
||||
|
||||
def load(self, cls_name: str, *args, **kwargs) -> Any:
|
||||
"""
|
||||
Instantiate and return a registered extension class by name.
|
||||
"""
|
||||
"""Instantiate and return a registered extension class by name."""
|
||||
cls = self.name2class.get(cls_name)
|
||||
assert cls is not None, f"Extension class {cls_name} not found"
|
||||
return cls(*args, **kwargs)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Safe deserialization helpers for client-supplied embedding tensors.
|
||||
"""Safe deserialization helpers for client-supplied embedding tensors.
|
||||
|
||||
`check_sparse_tensor_invariants_threadsafe()` is a thread-safe wrapper for
|
||||
sparse tensor invariant validation. `safe_to_dense()` bounds the memory a
|
||||
|
||||
@@ -60,8 +60,7 @@ def set_env_var(key: str, value: str) -> Iterator[None]:
|
||||
|
||||
@contextlib.contextmanager
|
||||
def suppress_stdout():
|
||||
"""
|
||||
Suppress stdout from C libraries at the file descriptor level.
|
||||
"""Suppress stdout from C libraries at the file descriptor level.
|
||||
|
||||
Only suppresses stdout, not stderr, to preserve error messages.
|
||||
Suppression is disabled when VLLM_LOGGING_LEVEL is set to DEBUG.
|
||||
@@ -70,6 +69,7 @@ def suppress_stdout():
|
||||
with suppress_stdout():
|
||||
# C library calls that would normally print to stdout
|
||||
torch.distributed.new_group(ranks, backend="gloo")
|
||||
|
||||
"""
|
||||
# Don't suppress if logging level is DEBUG
|
||||
if envs.VLLM_LOGGING_LEVEL == "DEBUG":
|
||||
@@ -114,7 +114,6 @@ def unique_filepath(fn: Callable[[int], Path]) -> Path:
|
||||
|
||||
def _sync_visible_devices_env_vars():
|
||||
"""Sync HIP/CUDA visibility env vars before spawning (ROCm only)."""
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return
|
||||
|
||||
@@ -254,11 +253,11 @@ def decorate_logs(
|
||||
|
||||
|
||||
def kill_process_tree(pid: int):
|
||||
"""
|
||||
Kills all descendant processes of the given pid by sending SIGKILL.
|
||||
"""Kills all descendant processes of the given pid by sending SIGKILL.
|
||||
|
||||
Args:
|
||||
pid (int): Process ID of the parent process
|
||||
|
||||
"""
|
||||
try:
|
||||
parent = psutil.Process(pid)
|
||||
|
||||
@@ -161,7 +161,6 @@ class TensorSchema:
|
||||
dynamic_dims: set[str],
|
||||
) -> None:
|
||||
"""Validate that the actual tensor shape matches the expected shape."""
|
||||
|
||||
if len(actual_shape) != len(expected_shape):
|
||||
raise ValueError(
|
||||
f"{field_name} has rank {len(actual_shape)} "
|
||||
|
||||
+17
-27
@@ -96,8 +96,7 @@ def is_meta_module(module: torch.nn.Module) -> bool:
|
||||
|
||||
|
||||
def is_strictly_contiguous(t: torch.Tensor) -> bool:
|
||||
"""
|
||||
Check if tensor is contiguous AND has no degenerate strides.
|
||||
"""Check if tensor is contiguous AND has no degenerate strides.
|
||||
|
||||
A degenerate stride occurs when a dimension has size 1 but the stride
|
||||
doesn't match the canonical contiguous layout. This can cause issues
|
||||
@@ -231,7 +230,8 @@ def _cgroup_cpu_limit() -> float | None:
|
||||
|
||||
def available_cpu_count() -> int:
|
||||
"""CPUs actually usable by this process: scheduling affinity capped by
|
||||
the cgroup CPU quota (unlike `os.cpu_count()`, which is quota-blind)."""
|
||||
the cgroup CPU quota (unlike `os.cpu_count()`, which is quota-blind).
|
||||
"""
|
||||
if sys.platform != "linux":
|
||||
return os.cpu_count() or 1
|
||||
count = len(os.sched_getaffinity(0))
|
||||
@@ -294,8 +294,7 @@ def set_torch_threads_for_runtime() -> None:
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_default_torch_num_threads(num_threads: int | None = None):
|
||||
"""
|
||||
Sets the default number of threads for PyTorch to the given value.
|
||||
"""Sets the default number of threads for PyTorch to the given value.
|
||||
|
||||
`None` means using the value of the environment variable `OMP_NUM_THREADS`
|
||||
(or `1` if that is not available).
|
||||
@@ -363,8 +362,7 @@ def _get_precision_level(dtype: torch.dtype) -> int:
|
||||
|
||||
|
||||
def is_lossless_cast(src_dtype: torch.dtype, tgt_dtype: torch.dtype):
|
||||
"""
|
||||
Test whether it is lossless to cast a tensor from
|
||||
"""Test whether it is lossless to cast a tensor from
|
||||
`src_dtype` to `tgt_dtype`.
|
||||
"""
|
||||
if src_dtype == tgt_dtype:
|
||||
@@ -395,8 +393,7 @@ def is_lossless_cast(src_dtype: torch.dtype, tgt_dtype: torch.dtype):
|
||||
|
||||
|
||||
def common_broadcastable_dtype(dtypes: Collection[torch.dtype]):
|
||||
"""
|
||||
Get the common `dtype` where all of the other `dtypes` can be
|
||||
"""Get the common `dtype` where all of the other `dtypes` can be
|
||||
cast to it without losing any information.
|
||||
"""
|
||||
return max(
|
||||
@@ -567,6 +564,7 @@ def nvfp4_split_data_scale(
|
||||
Returns:
|
||||
``(data, scale)`` where *data* is uint8 and *scale* is
|
||||
float8_e4m3fn, both views of the same storage.
|
||||
|
||||
"""
|
||||
num_pages = kv_side.shape[0]
|
||||
dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2]
|
||||
@@ -744,8 +742,7 @@ def make_ndarray_with_pad(
|
||||
*,
|
||||
max_len: int | None = None,
|
||||
) -> npt.NDArray:
|
||||
"""
|
||||
Make a padded array from 2D inputs.
|
||||
"""Make a padded array from 2D inputs.
|
||||
|
||||
The padding is applied to the end of each inner list until it reaches
|
||||
`max_len`.
|
||||
@@ -771,8 +768,7 @@ def make_tensor_with_pad(
|
||||
device: str | torch.device | None = None,
|
||||
pin_memory: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Make a padded tensor from 2D inputs.
|
||||
"""Make a padded tensor from 2D inputs.
|
||||
|
||||
The padding is applied to the end of each inner list until it reaches
|
||||
`max_len`.
|
||||
@@ -806,8 +802,7 @@ class _StreamPlaceholder:
|
||||
|
||||
|
||||
def current_stream() -> torch.cuda.Stream:
|
||||
"""
|
||||
replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`.
|
||||
"""Replace `torch.cuda.current_stream()` with `vllm.utils.current_stream()`.
|
||||
it turns out that `torch.cuda.current_stream()` is quite expensive,
|
||||
as it will construct a new stream object at each call.
|
||||
here we patch `torch.cuda.set_stream` to keep track of the current stream
|
||||
@@ -858,9 +853,7 @@ _aux_stream: torch.cuda.Stream | None = None
|
||||
|
||||
|
||||
def aux_stream() -> torch.cuda.Stream | None:
|
||||
"""
|
||||
Ensures aux_stream is initialized only once
|
||||
"""
|
||||
"""Ensures aux_stream is initialized only once."""
|
||||
global _aux_stream
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
@@ -872,8 +865,7 @@ def aux_stream() -> torch.cuda.Stream | None:
|
||||
|
||||
|
||||
def weak_ref_tensor(tensor: Any) -> Any:
|
||||
"""
|
||||
Create a weak reference to a tensor.
|
||||
"""Create a weak reference to a tensor.
|
||||
The new tensor will share the same data as the original tensor,
|
||||
but will not keep the original tensor alive.
|
||||
This ignores 0-size tensors as those don't allocate any memory.
|
||||
@@ -890,8 +882,7 @@ def weak_ref_tensors(
|
||||
| tuple[torch.Tensor]
|
||||
| IntermediateTensors,
|
||||
) -> torch.Tensor | list[Any] | tuple[Any] | Any:
|
||||
"""
|
||||
Convenience function to create weak references to tensors,
|
||||
"""Convenience function to create weak references to tensors,
|
||||
for single tensor, list of tensors or tuple of tensors.
|
||||
"""
|
||||
if isinstance(tensors, torch.Tensor):
|
||||
@@ -913,9 +904,7 @@ def weak_ref_tensors(
|
||||
|
||||
|
||||
def get_accelerator_view_from_cpu_tensor(cpu_tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Get an accelerator view of a CPU tensor using Unified Virtual Addressing (UVA).
|
||||
"""
|
||||
"""Get an accelerator view of a CPU tensor via Unified Virtual Addressing."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_xpu():
|
||||
@@ -951,6 +940,7 @@ def is_torch_equal_or_newer(target: str) -> bool:
|
||||
|
||||
Returns:
|
||||
Whether the condition meets.
|
||||
|
||||
"""
|
||||
try:
|
||||
return _is_torch_equal_or_newer(str(torch.__version__), target)
|
||||
@@ -979,6 +969,7 @@ def is_torch_equal(target: str) -> bool:
|
||||
|
||||
Returns:
|
||||
Whether the condition meets.
|
||||
|
||||
"""
|
||||
try:
|
||||
return _is_torch_equal(target)
|
||||
@@ -1064,8 +1055,7 @@ def direct_register_custom_op(
|
||||
dispatch_key: str | None = None,
|
||||
tags: tuple[torch.Tag, ...] = (),
|
||||
):
|
||||
"""
|
||||
`torch.library.custom_op` can have significant overhead because it
|
||||
"""`torch.library.custom_op` can have significant overhead because it
|
||||
needs to consider complicated dispatching logic. This function
|
||||
directly registers a custom op and dispatches it to the CUDA backend.
|
||||
See https://gist.github.com/youkaichao/ecbea9ec9fc79a45d2adce1784d7a9a5
|
||||
|
||||
Reference in New Issue
Block a user