53 lines
1.7 KiB
Python
53 lines
1.7 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from typing import Any, Protocol, cast
|
|
|
|
|
|
class AdmissionCounterBuffer(Protocol):
|
|
def __getitem__(self, index: int) -> int: ...
|
|
|
|
def __setitem__(self, index: int, value: int) -> None: ...
|
|
|
|
|
|
class SharedAdmissionStats:
|
|
"""Lock-free admission stats shared by API server processes.
|
|
|
|
Each process writes exclusively to its own cache-line-sized slot. Readers
|
|
aggregate the slots, avoiding both inter-process locks and cache-line
|
|
contention on the request path.
|
|
"""
|
|
|
|
# Space per-process counters 64 bytes apart to avoid false sharing on CPUs
|
|
# with 64-byte cache lines: one process's writes would otherwise invalidate
|
|
# the cache line containing another process's counter.
|
|
CACHE_LINE_BYTES = 64
|
|
COUNTER_BYTES = 8
|
|
COUNTERS_PER_SLOT = CACHE_LINE_BYTES // COUNTER_BYTES
|
|
|
|
def __init__(
|
|
self,
|
|
client_addresses: dict[str, Any],
|
|
client_count: int = 1,
|
|
client_index: int = 0,
|
|
):
|
|
self._counters = cast(
|
|
AdmissionCounterBuffer,
|
|
client_addresses.pop("mp_admission_counters"),
|
|
)
|
|
self._client_count = client_count
|
|
self._slot_offset = client_index * self.COUNTERS_PER_SLOT
|
|
|
|
@classmethod
|
|
def num_counters(cls, client_count: int) -> int:
|
|
return client_count * cls.COUNTERS_PER_SLOT
|
|
|
|
def set_num_requests(self, count: int) -> None:
|
|
self._counters[self._slot_offset] = count
|
|
|
|
def get_num_requests(self) -> int:
|
|
return sum(
|
|
self._counters[client_index * self.COUNTERS_PER_SLOT]
|
|
for client_index in range(self._client_count)
|
|
)
|