Add full API docs and improve the UX of navigating them (#17485)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
# MLA Common Components
|
||||
|
||||
This file implements common components for MLA implementations.
|
||||
|
||||
First we define:
|
||||
|
||||
@@ -180,6 +180,7 @@ class KVCacheManager:
|
||||
as eagle.
|
||||
|
||||
Blocks layout:
|
||||
```
|
||||
-----------------------------------------------------------------------
|
||||
| < computed > | < new computed > | < new > | < pre-allocated > |
|
||||
-----------------------------------------------------------------------
|
||||
@@ -189,6 +190,7 @@ class KVCacheManager:
|
||||
------------------------------------------------
|
||||
| <new full> |
|
||||
--------------
|
||||
```
|
||||
The following *_blocks are illustrated in this layout.
|
||||
|
||||
Returns:
|
||||
|
||||
@@ -308,7 +308,7 @@ class OutputProcessor:
|
||||
* If there is no queue (for usage with LLMEngine),
|
||||
return a list of RequestOutput objects.
|
||||
|
||||
****************** NOTE FOR DEVELOPERS ******************
|
||||
NOTE FOR DEVELOPERS
|
||||
|
||||
vLLM V1 minimizes the number of python loops over the full
|
||||
batch to ensure system overheads are minimized. This is the
|
||||
@@ -316,8 +316,6 @@ class OutputProcessor:
|
||||
|
||||
If you need to touch every element of the batch, do it from
|
||||
within the loop below.
|
||||
|
||||
**********************************************************
|
||||
"""
|
||||
|
||||
request_outputs: list[RequestOutput] = []
|
||||
|
||||
@@ -75,7 +75,7 @@ class RejectionSampler(nn.Module):
|
||||
outside of the rejection sampler with the default sampling
|
||||
strategy. It allows for more flexibility in the sampling
|
||||
process such as top_p, top_k sampling.
|
||||
sampling_metadata (SamplingMetadata):
|
||||
sampling_metadata (vllm.v1.sample.metadata.SamplingMetadata):
|
||||
Additional metadata needed for sampling, such as temperature,
|
||||
top-k/top-p parameters, or other relevant information.
|
||||
Returns:
|
||||
|
||||
@@ -170,9 +170,10 @@ class Worker(WorkerBase):
|
||||
Then, it calculate the free memory that can be used for KV cache in
|
||||
bytes.
|
||||
|
||||
.. tip::
|
||||
You may limit the usage of GPU memory
|
||||
by adjusting the `gpu_memory_utilization` parameter.
|
||||
:::{tip}
|
||||
You may limit the usage of GPU memory
|
||||
by adjusting the `gpu_memory_utilization` parameter.
|
||||
:::
|
||||
"""
|
||||
torch.cuda.empty_cache()
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
|
||||
@@ -10,7 +10,7 @@ def sanity_check_mm_encoder_outputs(
|
||||
) -> None:
|
||||
"""
|
||||
Perform sanity checks for the result of
|
||||
:meth:`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`.
|
||||
{meth}`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`.
|
||||
"""
|
||||
assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), (
|
||||
"Expected multimodal embeddings to be a list/tuple of 2D tensors, "
|
||||
@@ -39,7 +39,7 @@ def scatter_mm_placeholders(
|
||||
Scatter the multimodal embeddings into a contiguous tensor that represents
|
||||
the placeholder tokens.
|
||||
|
||||
:class:`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
|
||||
{class}`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
|
||||
|
||||
Args:
|
||||
embeds: The multimodal embeddings.
|
||||
@@ -66,7 +66,7 @@ def gather_mm_placeholders(
|
||||
"""
|
||||
Reconstructs the embeddings from the placeholder tokens.
|
||||
|
||||
This is the operation of :func:`scatter_mm_placeholders`.
|
||||
This is the operation of {func}`scatter_mm_placeholders`.
|
||||
"""
|
||||
if is_embed is None:
|
||||
return placeholders
|
||||
|
||||
Reference in New Issue
Block a user