[Realtime API] Adds minimal realtime API based on websockets (#33187)

Signed-off-by: Patrick von Platen <patrick.v.platen@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Patrick von Platen
2026-01-30 18:41:29 +08:00
committed by GitHub
co-authored by Nick Hill
parent 1a7894dbdf
commit 10152d2194
21 changed files with 1316 additions and 48 deletions
+33 -1
View File
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterable, Mapping, MutableSequence
import asyncio
from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, MutableSequence
from contextlib import ExitStack, contextmanager, nullcontext
from typing import (
TYPE_CHECKING,
@@ -1015,6 +1016,37 @@ class SupportsQuant:
return None
@runtime_checkable
class SupportsRealtime(Protocol):
"""The interface required for all models that support transcription."""
supports_realtime: ClassVar[Literal[True]] = True
@classmethod
async def buffer_realtime_audio(
cls,
audio_stream: AsyncGenerator[np.ndarray, None],
input_stream: asyncio.Queue[list[int]],
model_config: ModelConfig,
) -> AsyncGenerator[PromptType, None]: ...
@overload
def supports_realtime(
model: type[object],
) -> TypeIs[type[SupportsRealtime]]: ...
@overload
def supports_realtime(model: object) -> TypeIs[SupportsRealtime]: ...
def supports_realtime(
model: type[object] | object,
) -> TypeIs[type[SupportsRealtime]] | TypeIs[SupportsRealtime]:
return getattr(model, "supports_realtime", False)
@runtime_checkable
class SupportsTranscription(Protocol):
"""The interface required for all models that support transcription."""