Hidden states extraction improvements (#43805)
Signed-off-by: Fynn Schmitt-Ulms <fschmitt@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ab3a1fd2e6
commit
ebc6ef971a
@@ -19,10 +19,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorRole,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_rank
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -76,43 +76,20 @@ def cleanup_hidden_states(path: str, keep_hidden_states: bool = False) -> None:
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReqMeta:
|
||||
# Request ID
|
||||
class PendingSave:
|
||||
req_id: str
|
||||
# Request filename
|
||||
filename: str
|
||||
# Request tokens
|
||||
token_ids: torch.Tensor
|
||||
# Whether this request is a new request or partially computed already
|
||||
new_req: bool
|
||||
|
||||
@staticmethod
|
||||
def make_meta(
|
||||
req_id: str,
|
||||
filename: str,
|
||||
token_ids: list[int],
|
||||
new_req: bool,
|
||||
) -> "ReqMeta":
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
filename=filename,
|
||||
token_ids=torch.tensor(token_ids),
|
||||
new_req=new_req,
|
||||
)
|
||||
block_ids: list[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExampleHiddenStatesConnectorMetadata(KVConnectorMetadata):
|
||||
requests: list[ReqMeta] = field(default_factory=list)
|
||||
|
||||
def add_request(
|
||||
self,
|
||||
req_id: str,
|
||||
filename: str,
|
||||
token_ids: list[int],
|
||||
new_req: bool = True,
|
||||
) -> None:
|
||||
self.requests.append(ReqMeta.make_meta(req_id, filename, token_ids, new_req))
|
||||
pending_saves: list[PendingSave] = field(default_factory=list)
|
||||
# req_id → filename for newly scheduled requests — the worker pre-creates
|
||||
# lock files for these so the lock exists before the client receives the
|
||||
# output path.
|
||||
new_req_filenames: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
@@ -167,9 +144,16 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
getattr(spec_config, "eagle_aux_hidden_state_layer_ids", [])
|
||||
)
|
||||
|
||||
# Scheduler-side state
|
||||
self._pending_saves: dict[str, PendingSave] = {}
|
||||
self._request_filenames: dict[str, str] = {}
|
||||
self._active_requests: dict[str, NewRequestData] = {}
|
||||
self._req_blocks: dict[str, list[int]] = {}
|
||||
|
||||
# Worker-side state (set by register_kv_caches).
|
||||
self._kv_cache: torch.Tensor | None = None
|
||||
self._hs_group_idx: int = 0
|
||||
# Only TP rank 0 writes hidden states to disk; other TP ranks no-op.
|
||||
# Set in register_kv_caches (after distributed init).
|
||||
self._is_tp_rank_zero: bool = True
|
||||
|
||||
# Async write infrastructure (worker-side).
|
||||
# Dedicated CUDA stream for DtoH copies so they don't block
|
||||
@@ -184,14 +168,23 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Whether to use a filesystem lock when writing files to shared storage.
|
||||
# This is necessary for online transfer clients to avoid incomplete reads,
|
||||
# but can be disabled for offline tasks that run tasks in batches to completion
|
||||
self.allow_custom_save_path = self._kv_transfer_config.get_from_extra_config(
|
||||
"allow_custom_save_path", False
|
||||
)
|
||||
if self.allow_custom_save_path:
|
||||
logger.warning(
|
||||
"allow_custom_save_path is enabled. API clients can write "
|
||||
"hidden states to arbitrary paths on the server filesystem. "
|
||||
"Only enable this with trusted clients."
|
||||
)
|
||||
self.use_lock = self._kv_transfer_config.get_from_extra_config(
|
||||
"use_synchronization_lock", True
|
||||
)
|
||||
# (tensors_dict, copy_done_event, filename, req_id) queued by
|
||||
# save_kv_layer, submitted to thread pool by wait_for_save.
|
||||
self._pending_copies: list[
|
||||
tuple[dict[str, torch.Tensor], torch.cuda.Event, str, str]
|
||||
] = []
|
||||
# req_id → open fd on the .lock file with LOCK_EX held.
|
||||
# Pre-created in wait_for_save when a request first arrives,
|
||||
# consumed by _submit_async_write which passes the fd to the
|
||||
# thread pool worker for release after writing.
|
||||
self._lock_fds: dict[str, int] = {}
|
||||
# req_id → in-flight disk-write Future for that req_id.
|
||||
self._req_futures: dict[str, Future] = {}
|
||||
# req_id → CUDA event marking completion of the DtoH copy. Once
|
||||
@@ -218,42 +211,28 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
pass # Store-only connector — nothing to load
|
||||
|
||||
def wait_for_save(self):
|
||||
"""Submit pending async copies to the thread pool for disk write.
|
||||
def wait_for_save(self) -> None:
|
||||
"""Pre-create lock files for newly arrived requests.
|
||||
|
||||
For each pending write we acquire an exclusive flock on a
|
||||
companion ``.lock`` file **before** submitting to the thread pool.
|
||||
The thread worker releases the lock after the data file is fully
|
||||
written. Clients call :func:`load_hidden_states` which takes a
|
||||
shared flock — the kernel sleeps the client until the writer is
|
||||
done. Because ``wait_for_save`` runs before the worker returns
|
||||
output to the scheduler, the lock file is guaranteed to exist
|
||||
(and be held) by the time the client receives the path.
|
||||
|
||||
The lock can be disabled via the "use_synchronization_lock" extra config.
|
||||
This runs on the worker BEFORE the scheduler returns the output
|
||||
path to the client, guaranteeing that the lock file exists (and
|
||||
LOCK_EX is held) by the time the client tries to open it.
|
||||
"""
|
||||
for tensors, event, filename, req_id in self._pending_copies:
|
||||
prior = self._req_futures.get(req_id)
|
||||
assert prior is None, "Found another KV transfer request with same req_id!"
|
||||
|
||||
lock_fd = None
|
||||
if self.use_lock:
|
||||
# Create/open the lock file and acquire an exclusive lock.
|
||||
# The lock is held by this fd; the thread worker will close
|
||||
# the fd after writing, which releases the lock.
|
||||
lock_path = filename + ".lock"
|
||||
lock_fd = os.open(
|
||||
lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644
|
||||
)
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
|
||||
future = self._executor.submit(
|
||||
self._write_tensors, tensors, event, filename, lock_fd
|
||||
)
|
||||
self._req_copy_events[req_id] = event
|
||||
self._req_futures[req_id] = future
|
||||
future.add_done_callback(partial(self._on_write_done, req_id))
|
||||
self._pending_copies.clear()
|
||||
if not self._is_tp_rank_zero:
|
||||
return
|
||||
if not self.use_lock or not self.has_connector_metadata():
|
||||
return
|
||||
metadata = self._get_connector_metadata()
|
||||
if not isinstance(metadata, ExampleHiddenStatesConnectorMetadata):
|
||||
return
|
||||
for req_id, filename in metadata.new_req_filenames.items():
|
||||
if req_id in self._lock_fds:
|
||||
continue
|
||||
lock_path = filename + ".lock"
|
||||
os.makedirs(os.path.dirname(lock_path), exist_ok=True)
|
||||
lock_fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
self._lock_fds[req_id] = lock_fd
|
||||
|
||||
def _on_write_done(self, req_id: str, future: Future) -> None:
|
||||
"""Surface any exception from the disk-write thread and drop the
|
||||
@@ -264,6 +243,9 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
logger.error("Hidden-states write failed for req_id=%s: %r", req_id, exc)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
# Delay tp rank0 initialization until after distributed init
|
||||
self._is_tp_rank_zero = get_tensor_model_parallel_rank() == 0
|
||||
|
||||
from vllm.model_executor.models.extract_hidden_states import (
|
||||
CacheOnlyAttentionLayer,
|
||||
)
|
||||
@@ -276,6 +258,14 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert len(self.cache_layers) == 1, (
|
||||
f"Expected 1 CacheOnlyAttentionLayer, got {len(self.cache_layers)}"
|
||||
)
|
||||
self._kv_cache = kv_caches[self.cache_layers[0]]
|
||||
|
||||
# Find the KV cache group index for hidden states
|
||||
if self._kv_cache_config is not None:
|
||||
for i, group in enumerate(self._kv_cache_config.kv_cache_groups):
|
||||
if self.cache_layers[0] in group.layer_names:
|
||||
self._hs_group_idx = i
|
||||
break
|
||||
|
||||
@staticmethod
|
||||
def _write_tensors(
|
||||
@@ -304,35 +294,33 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""Start saving the KV cache of the layer from vLLM's paged buffer
|
||||
to the connector.
|
||||
# Hidden states are already cached by CacheOnlyAttentionLayer during
|
||||
# forward. Extraction happens in get_finished once all tokens are done.
|
||||
pass
|
||||
|
||||
Launches an async DtoH copy on a dedicated CUDA stream. The
|
||||
actual disk write is deferred to wait_for_save() which submits
|
||||
it to a thread pool.
|
||||
def _submit_async_write(
|
||||
self,
|
||||
pending: PendingSave,
|
||||
) -> None:
|
||||
"""Extract hidden states from KV cache and submit async DtoH + disk write.
|
||||
|
||||
Args:
|
||||
layer_name (str): the name of the layer.
|
||||
kv_layer (torch.Tensor): the paged KV buffer of the current
|
||||
layer in vLLM.
|
||||
attn_metadata (AttentionMetadata): the attention metadata.
|
||||
**kwargs: additional arguments for the save operation.
|
||||
Called from get_finished for each request that has finished generating.
|
||||
"""
|
||||
if layer_name not in self.cache_layers:
|
||||
if not self._is_tp_rank_zero:
|
||||
return
|
||||
assert self._kv_cache is not None
|
||||
|
||||
from vllm.model_executor.models.extract_hidden_states import (
|
||||
CacheOnlyAttentionMetadata,
|
||||
# Compute slot mapping from block_ids
|
||||
block_ids_t = torch.tensor(pending.block_ids, dtype=torch.long)
|
||||
num_blocks = block_ids_t.shape[0]
|
||||
block_offsets = torch.arange(0, self._block_size, dtype=torch.long)
|
||||
slot_mapping = (
|
||||
block_offsets.reshape((1, self._block_size))
|
||||
+ block_ids_t.reshape((num_blocks, 1)) * self._block_size
|
||||
)
|
||||
slot_mapping = slot_mapping.flatten()
|
||||
|
||||
assert isinstance(attn_metadata, CacheOnlyAttentionMetadata), (
|
||||
"ExampleHiddenStatesConnector only supports CacheOnlyAttentionBackend"
|
||||
)
|
||||
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, ExampleHiddenStatesConnectorMetadata)
|
||||
|
||||
os.makedirs(self._storage_path, exist_ok=True)
|
||||
num_tokens = pending.token_ids.shape[0]
|
||||
|
||||
copy_stream = self._get_copy_stream()
|
||||
|
||||
@@ -341,39 +329,56 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
ready_event.record()
|
||||
copy_stream.wait_event(ready_event)
|
||||
|
||||
slot_mapping = get_forward_context().slot_mapping[layer_name] # type: ignore
|
||||
offset = 0
|
||||
for request in connector_metadata.requests:
|
||||
num_tokens = request.token_ids.shape[0]
|
||||
with torch.cuda.stream(copy_stream):
|
||||
req_slot_mapping_gpu = slot_mapping[offset : offset + num_tokens]
|
||||
assert req_slot_mapping_gpu.device == kv_layer.device
|
||||
offset += num_tokens
|
||||
|
||||
hidden_states_gpu = extract_from_kv_cache(
|
||||
kv_layer, req_slot_mapping_gpu, num_tokens
|
||||
)
|
||||
# Async DtoH copy into pinned host memory.
|
||||
pinned_hs = torch.empty_like(
|
||||
hidden_states_gpu, device="cpu", pin_memory=True
|
||||
)
|
||||
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
|
||||
|
||||
# Record completion of this copy on the copy stream.
|
||||
copy_done = torch.cuda.Event()
|
||||
copy_done.record(copy_stream)
|
||||
|
||||
# token_ids is already on CPU (created in ReqMeta.make_meta).
|
||||
assert not request.token_ids.is_cuda, (
|
||||
"Expected token_ids on CPU, got CUDA tensor"
|
||||
with torch.cuda.stream(copy_stream):
|
||||
# Move the CPU slot_mapping to GPU on the copy stream so the
|
||||
# implicit H2D inside fancy indexing doesn't sync the default
|
||||
# stream.
|
||||
slot_mapping_gpu = slot_mapping.to(
|
||||
device=self._kv_cache.device, non_blocking=True
|
||||
)
|
||||
tensors = {
|
||||
"hidden_states": pinned_hs,
|
||||
"token_ids": request.token_ids.clone(),
|
||||
}
|
||||
self._pending_copies.append(
|
||||
(tensors, copy_done, request.filename, request.req_id)
|
||||
hidden_states_gpu = extract_from_kv_cache(
|
||||
self._kv_cache, slot_mapping_gpu, num_tokens
|
||||
)
|
||||
# Async DtoH copy into pinned host memory.
|
||||
pinned_hs = torch.empty_like(
|
||||
hidden_states_gpu, device="cpu", pin_memory=True
|
||||
)
|
||||
pinned_hs.copy_(hidden_states_gpu, non_blocking=True)
|
||||
|
||||
# Record completion of this copy on the copy stream.
|
||||
copy_done = torch.cuda.Event()
|
||||
copy_done.record(copy_stream)
|
||||
|
||||
# token_ids is already on CPU (created in request_finished).
|
||||
assert not pending.token_ids.is_cuda, (
|
||||
"Expected token_ids on CPU, got CUDA tensor"
|
||||
)
|
||||
tensors = {
|
||||
"hidden_states": pinned_hs,
|
||||
"token_ids": pending.token_ids.clone(),
|
||||
}
|
||||
|
||||
# Submit to thread pool for disk write.
|
||||
prior = self._req_futures.get(pending.req_id)
|
||||
assert prior is None, "Found another KV transfer request with same req_id!"
|
||||
|
||||
os.makedirs(os.path.dirname(pending.filename), exist_ok=True)
|
||||
|
||||
# Use the pre-created lock fd from wait_for_save (already holds
|
||||
# LOCK_EX). Falls back to creating one here if use_lock is True
|
||||
# but no pre-created fd exists (shouldn't happen in normal flow).
|
||||
lock_fd = self._lock_fds.pop(pending.req_id, None)
|
||||
if lock_fd is None and self.use_lock:
|
||||
lock_path = pending.filename + ".lock"
|
||||
lock_fd = os.open(lock_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644)
|
||||
fcntl.flock(lock_fd, fcntl.LOCK_EX)
|
||||
|
||||
future = self._executor.submit(
|
||||
self._write_tensors, tensors, copy_done, pending.filename, lock_fd
|
||||
)
|
||||
self._req_copy_events[pending.req_id] = copy_done
|
||||
self._req_futures[pending.req_id] = future
|
||||
future.add_done_callback(partial(self._on_write_done, pending.req_id))
|
||||
|
||||
# ==============================
|
||||
# Scheduler-side methods
|
||||
@@ -421,17 +426,34 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
"""
|
||||
meta = ExampleHiddenStatesConnectorMetadata()
|
||||
|
||||
# Transfer pending saves into metadata (scheduler → worker bridge)
|
||||
meta.pending_saves = list(self._pending_saves.values())
|
||||
self._pending_saves.clear()
|
||||
|
||||
# Resolve save paths for new requests and tell the worker so it can
|
||||
# pre-create lock files before the client receives the output path.
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
token_ids = new_req.prompt_token_ids or []
|
||||
filename = os.path.join(self._storage_path, f"{new_req.req_id}.safetensors")
|
||||
meta.add_request(
|
||||
new_req.req_id,
|
||||
filename=filename,
|
||||
token_ids=token_ids,
|
||||
default_path = os.path.join(
|
||||
self._storage_path, f"{new_req.req_id}.safetensors"
|
||||
)
|
||||
kv_params = (
|
||||
new_req.sampling_params.extra_args.get("kv_transfer_params")
|
||||
if new_req.sampling_params and new_req.sampling_params.extra_args
|
||||
else None
|
||||
) or {}
|
||||
custom_path = kv_params.get("hidden_states_path")
|
||||
if custom_path is not None and not self.allow_custom_save_path:
|
||||
logger.warning(
|
||||
"Request %s provided hidden_states_path but "
|
||||
"allow_custom_save_path is disabled. Ignoring "
|
||||
"custom path and using default.",
|
||||
new_req.req_id,
|
||||
)
|
||||
custom_path = None
|
||||
filename = custom_path or default_path
|
||||
self._request_filenames[new_req.req_id] = filename
|
||||
self._active_requests[new_req.req_id] = new_req
|
||||
self._req_blocks[new_req.req_id] = list(new_req.block_ids[0])
|
||||
meta.new_req_filenames[new_req.req_id] = filename
|
||||
|
||||
return meta
|
||||
|
||||
@@ -444,35 +466,54 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
Called exactly once when a request has finished, before its blocks are
|
||||
freed.
|
||||
|
||||
The connector may assumes responsibility for freeing the blocks
|
||||
asynchronously by returning True.
|
||||
|
||||
Returns:
|
||||
True if the request is being saved/sent asynchronously and blocks
|
||||
should not be freed until the request_id is returned from
|
||||
get_finished().
|
||||
Optional KVTransferParams to be included in the request outputs
|
||||
returned by the engine.
|
||||
Returns True to delay block freeing until get_finished extracts
|
||||
the hidden states from the KV cache.
|
||||
"""
|
||||
req_id = request.request_id
|
||||
req_filename = self._request_filenames.pop(req_id, None)
|
||||
_ = self._active_requests.pop(req_id, None)
|
||||
_ = self._req_blocks.pop(req_id, None)
|
||||
|
||||
return True, {"hidden_states_path": req_filename}
|
||||
filename = self._request_filenames.pop(req_id)
|
||||
kv_params = request.kv_transfer_params or {}
|
||||
if kv_params.get("include_output_tokens", False):
|
||||
# Exclude the final token — it was the model's output, never an
|
||||
# input to a forward pass, so its hidden state is not in the cache.
|
||||
token_ids = torch.tensor(list(request.all_token_ids)[:-1])
|
||||
elif request.prompt_token_ids is not None:
|
||||
token_ids = torch.tensor(request.prompt_token_ids)
|
||||
else:
|
||||
logger.warning(
|
||||
"Request %s has no prompt_token_ids (prompt_embeds only). "
|
||||
"Saved token_ids will be empty.",
|
||||
req_id,
|
||||
)
|
||||
token_ids = torch.tensor([], dtype=torch.long)
|
||||
self._pending_saves[req_id] = PendingSave(
|
||||
req_id=req_id,
|
||||
filename=filename,
|
||||
token_ids=token_ids,
|
||||
block_ids=list(block_ids),
|
||||
)
|
||||
return True, {"hidden_states_path": filename}
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
"""Poll DtoH-copy completion for requests that finished generating.
|
||||
"""Extract hidden states and poll DtoH-copy completion.
|
||||
|
||||
The scheduler passes finished_req_ids to tell the worker which
|
||||
requests are done generating. We accumulate these across calls
|
||||
and return a request as "finished sending" once its DtoH copy
|
||||
event is complete (or if it never had a pending copy). The
|
||||
subsequent disk write may still be in flight; clients block on
|
||||
the per-file flock to wait for it.
|
||||
On the worker side, connector metadata carries pending saves from the
|
||||
scheduler. For each one we extract from the KV cache and launch an
|
||||
async DtoH copy + thread-pool disk write.
|
||||
|
||||
We then poll accumulated finished req_ids: a request is "done sending"
|
||||
once its DtoH copy event is complete. The subsequent disk write may
|
||||
still be in flight; clients block on the per-file flock to wait for it.
|
||||
"""
|
||||
# Extract and submit async writes for newly finished requests
|
||||
if self.has_connector_metadata():
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
if isinstance(connector_metadata, ExampleHiddenStatesConnectorMetadata):
|
||||
for pending in connector_metadata.pending_saves:
|
||||
self._submit_async_write(pending)
|
||||
|
||||
# Poll for completed DtoH copies
|
||||
self._accumulated_finished_req_ids.update(finished_req_ids)
|
||||
|
||||
done_sending: set[str] = set()
|
||||
@@ -482,6 +523,11 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
self._req_copy_events.pop(req_id, None)
|
||||
done_sending.add(req_id)
|
||||
self._accumulated_finished_req_ids.discard(req_id)
|
||||
# Clean up any leftover lock fds (e.g. aborted requests
|
||||
# that never went through _submit_async_write).
|
||||
lock_fd = self._lock_fds.pop(req_id, None)
|
||||
if lock_fd is not None:
|
||||
os.close(lock_fd)
|
||||
|
||||
return done_sending or None, None
|
||||
|
||||
@@ -490,7 +536,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: "Request",
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
return self.request_finished(request, block_ids[0])
|
||||
return self.request_finished(request, block_ids[self._hs_group_idx])
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None:
|
||||
|
||||
Reference in New Issue
Block a user