[Docs] Use gen-files for generated docs content (#49587)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -173,9 +173,6 @@ venv.bak/
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
docs/argparse
|
||||
docs/examples/*
|
||||
!docs/examples/README.md
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
@@ -3,6 +3,9 @@ MD007:
|
||||
MD013: false
|
||||
MD024:
|
||||
siblings_only: true
|
||||
MD025:
|
||||
# Allow front matter title to be different from the first heading in the document.
|
||||
front_matter_title: ""
|
||||
MD031:
|
||||
list_items: false
|
||||
MD033: false
|
||||
|
||||
@@ -260,10 +260,6 @@ repos:
|
||||
files: ^docker/(Dockerfile|versions\.json)$
|
||||
pass_filenames: false
|
||||
additional_dependencies: [dockerfile-parse]
|
||||
- id: attention-backend-docs
|
||||
name: Check attention backend documentation is up to date
|
||||
entry: python tools/pre_commit/generate_attention_backend_docs.py --check
|
||||
language: python
|
||||
- id: check-boolean-context-manager
|
||||
name: Check for boolean ops in with-statements
|
||||
entry: python tools/pre_commit/check_boolean_context_manager.py
|
||||
|
||||
+3
-1
@@ -56,7 +56,9 @@ nav:
|
||||
- API Reference:
|
||||
- api/README.md
|
||||
- api/vllm
|
||||
- CLI Reference: cli
|
||||
- CLI Reference:
|
||||
- cli/README.md
|
||||
- vllm: cli
|
||||
- Community:
|
||||
- community/*
|
||||
- Governance: governance
|
||||
|
||||
+7
-9
@@ -1,10 +1,8 @@
|
||||
nav:
|
||||
- README.md
|
||||
- serve.md
|
||||
- chat.md
|
||||
- complete.md
|
||||
- run-batch.md
|
||||
- vllm bench:
|
||||
- bench/**/*.md
|
||||
- vllm launch:
|
||||
- launch/**/*.md
|
||||
- "*.md"
|
||||
- bench:
|
||||
- bench/*.md
|
||||
- sweep:
|
||||
- bench/sweep/*.md
|
||||
- launch:
|
||||
- launch/*.md
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench latency
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_latency.inc.md"
|
||||
@@ -1,55 +0,0 @@
|
||||
# vllm bench mm-processor
|
||||
|
||||
## Overview
|
||||
|
||||
`vllm bench mm-processor` profiles the multimodal input processor pipeline of
|
||||
vision-language models. It measures per-stage latency from the HuggingFace
|
||||
processor through to the encoder forward pass, helping you identify
|
||||
preprocessing bottlenecks and understand how different image resolutions or
|
||||
item counts affect end-to-end request time.
|
||||
|
||||
The benchmark supports two data sources: synthetic random multimodal inputs
|
||||
(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before
|
||||
measurement to ensure stable results.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
vllm bench mm-processor \
|
||||
--model Qwen/Qwen2-VL-7B-Instruct \
|
||||
--dataset-name random-mm \
|
||||
--num-prompts 50 \
|
||||
--random-input-len 300 \
|
||||
--random-output-len 40 \
|
||||
--random-mm-base-items-per-request 2 \
|
||||
--random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \
|
||||
--random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'
|
||||
```
|
||||
|
||||
## Measured Stages
|
||||
|
||||
| Stage | Description |
|
||||
| ----- | ----------- |
|
||||
| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |
|
||||
| `get_cache_missing_items_secs` | Time spent looking up the processor cache |
|
||||
| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |
|
||||
| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |
|
||||
| `apply_prompt_updates_secs` | Time spent updating prompt tokens |
|
||||
| `preprocessor_total_secs` | Total preprocessing time |
|
||||
| `encoder_forward_secs` | Time spent in the encoder model forward pass |
|
||||
| `num_encoder_calls` | Number of encoder invocations per request |
|
||||
|
||||
The benchmark also reports end-to-end latency (TTFT + decode time) per
|
||||
request. Use `--metric-percentiles` to select which percentiles to report
|
||||
(default: p99) and `--output-json` to save results.
|
||||
|
||||
For more examples (HF datasets, warmup, JSON output), see
|
||||
[Benchmarking CLI — Multimodal Processor Benchmark](../../benchmarking/cli.md#multimodal-processor-benchmark).
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_mm_processor.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench serve
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_serve.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench sweep plot
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_sweep_plot.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench sweep plot_pareto
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_sweep_plot_pareto.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench sweep serve
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_sweep_serve.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench sweep serve_workload
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_sweep_serve_workload.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm bench throughput
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/bench_throughput.inc.md"
|
||||
@@ -1,5 +0,0 @@
|
||||
# vllm chat
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/chat.inc.md"
|
||||
@@ -1,5 +0,0 @@
|
||||
# vllm complete
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/complete.inc.md"
|
||||
@@ -1,10 +0,0 @@
|
||||
<!-- markdownlint-disable MD041 -->
|
||||
When passing JSON CLI arguments, the following sets of arguments are equivalent:
|
||||
|
||||
- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`
|
||||
- `--json-arg.key1 value1 --json-arg.key2.key3 value2`
|
||||
|
||||
Additionally, list elements can be passed individually using `+`:
|
||||
|
||||
- `--json-arg '{"key4": ["value3", "value4", "value5"]}'`
|
||||
- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`
|
||||
@@ -1,22 +0,0 @@
|
||||
# vllm launch render
|
||||
|
||||
## Overview
|
||||
|
||||
`vllm launch render` starts a GPU-less rendering server for preprocessing and
|
||||
postprocessing only.
|
||||
|
||||
```bash
|
||||
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
|
||||
```
|
||||
|
||||
This command reuses the standard serving parser, so model, frontend,
|
||||
networking, and related CLI options follow the same conventions as
|
||||
[`vllm serve`](../serve.md).
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/launch_render.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm run-batch
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/run-batch.inc.md"
|
||||
@@ -1,9 +0,0 @@
|
||||
# vllm serve
|
||||
|
||||
## JSON CLI Arguments
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## Arguments
|
||||
|
||||
--8<-- "docs/generated/argparse/serve.inc.md"
|
||||
@@ -11,12 +11,4 @@ Engine arguments control the behavior of the vLLM engine.
|
||||
|
||||
The engine argument classes, [EngineArgs][vllm.engine.arg_utils.EngineArgs] and [AsyncEngineArgs][vllm.engine.arg_utils.AsyncEngineArgs], are a combination of the configuration classes defined in [vllm.config][]. Therefore, if you are interested in developer documentation, we recommend looking at these configuration classes as they are the source of truth for types, defaults and docstrings.
|
||||
|
||||
--8<-- "docs/cli/json_tip.inc.md"
|
||||
|
||||
## `EngineArgs`
|
||||
|
||||
--8<-- "docs/generated/argparse/engine_args.inc.md"
|
||||
|
||||
## `AsyncEngineArgs`
|
||||
|
||||
--8<-- "docs/generated/argparse/async_engine_args.inc.md"
|
||||
--8<-- "gen:engine-args"
|
||||
|
||||
@@ -1,15 +1,9 @@
|
||||
# Attention Backend Feature Support
|
||||
|
||||
This document is auto-generated by `tools/pre_commit/generate_attention_backend_docs.py`.
|
||||
It shows the feature support for each registered attention backend
|
||||
based on the checks in `AttentionBackend.validate_configuration()`.
|
||||
|
||||
**Do not edit this file manually.** Run the following command to
|
||||
regenerate it:
|
||||
|
||||
```bash
|
||||
python tools/pre_commit/generate_attention_backend_docs.py
|
||||
```
|
||||
The priority and feature tables on this page are auto-generated from the
|
||||
attention backend registry by
|
||||
`docs/mkdocs/gen_files/generate_attention_backends.py`, based on the checks in
|
||||
`AttentionBackend.validate_configuration()`.
|
||||
|
||||
## Setting the Attention Backend
|
||||
|
||||
@@ -98,40 +92,11 @@ Priority is **1 = highest** (tried first).
|
||||
|
||||
### Standard Attention (MHA, MQA, GQA)
|
||||
|
||||
**Blackwell (SM 10.x):**
|
||||
|
||||
| Priority | Backend |
|
||||
| -------- | ------- |
|
||||
| 1 | `FLASHINFER` |
|
||||
| 2 | `FLASH_ATTN` |
|
||||
| 3 | `TRITON_ATTN` |
|
||||
| 4 | `FLEX_ATTENTION` |
|
||||
| 5 | `TURBOQUANT` |
|
||||
|
||||
**Ampere/Hopper (SM 8.x-9.x):**
|
||||
|
||||
| Priority | Backend |
|
||||
| -------- | ------- |
|
||||
| 1 | `FLASH_ATTN` |
|
||||
| 2 | `FLASHINFER` |
|
||||
| 3 | `TRITON_ATTN` |
|
||||
| 4 | `FLEX_ATTENTION` |
|
||||
| 5 | `TURBOQUANT` |
|
||||
--8<-- "gen:priority-standard"
|
||||
|
||||
### MLA Attention (DeepSeek-style)
|
||||
|
||||
**Blackwell (SM 10.x):**
|
||||
|
||||
| Priority | Backend |
|
||||
| -------- | ------- |
|
||||
| 1 | `FLASHINFER_MLA` |
|
||||
| 2 | `TOKENSPEED_MLA` |
|
||||
| 3 | `CUTLASS_MLA` |
|
||||
| 4 | `FLASH_ATTN_MLA` |
|
||||
| 5 | `FLASHMLA` |
|
||||
| 6 | `TRITON_MLA` |
|
||||
| 7 | `FLASHINFER_MLA_SPARSE`**\*** |
|
||||
| 8 | `FLASHMLA_SPARSE` |
|
||||
--8<-- "gen:priority-mla"
|
||||
|
||||
> **\*** For sparse MLA, FP8 KV cache always prefers `FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` is preferred for low query-head counts (<= 16), while `FLASHMLA_SPARSE` is preferred otherwise.
|
||||
>
|
||||
@@ -157,24 +122,7 @@ Priority is **1 = highest** (tried first).
|
||||
|
||||
## Standard Attention (MHA, MQA, GQA) Backends
|
||||
|
||||
| Backend | Version | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. |
|
||||
| ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
|
||||
| `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ✅ | ❌ | ❌ | All | N/A |
|
||||
| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 8.x-9.x |
|
||||
| `FLASHINFER` | XQA† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 9.0 |
|
||||
| `FLASHINFER` | trtllm-gen† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 |
|
||||
| `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x |
|
||||
| `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |
|
||||
| `FLASH_ATTN_DIFFKV` | | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ✅ | Decoder | Any |
|
||||
| `FLEX_ATTENTION` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder Only | Any |
|
||||
| `HPC_ATTN` | | fp16, bf16 | `auto`, `bfloat16`, `fp8_e4m3` | 64 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | ≥9.0 |
|
||||
| `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ✅ | ✅ | ❌ | ❌ | Decoder | N/A |
|
||||
| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |
|
||||
| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |
|
||||
| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int4_per_token_head`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ✅ | ✅ | ❌ | All | Any |
|
||||
| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
--8<-- "gen:table-standard"
|
||||
|
||||
> **†** FlashInfer Native is the regular FlashInfer path. XQA is the SM90 decode path exposed through FlashInfer's TRTLLM decode API. trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen via `--attention-config.use_trtllm_attention=0`.
|
||||
>
|
||||
@@ -188,9 +136,7 @@ automatic priority lists above. A lightning indexer scores KV blocks, the
|
||||
top-k blocks (plus fixed init/local blocks) are selected, and attention
|
||||
attends only to those blocks; index keys live in a separate side cache.
|
||||
|
||||
| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. |
|
||||
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
|
||||
| `MINIMAX_M3_SPARSE` | bf16, fp16 | `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 128 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
--8<-- "gen:table-minimax"
|
||||
|
||||
## MLA (Multi-head Latent Attention) Backends
|
||||
|
||||
@@ -203,12 +149,7 @@ To explicitly select a prefill backend, use
|
||||
Otherwise, the prefill backend is selected automatically at runtime based on
|
||||
hardware and configuration.
|
||||
|
||||
| Backend | Description | Dtypes | Compute Cap. | Notes |
|
||||
| ------- | ----------- | ------ | ------------ | ----- |
|
||||
| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=64, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) (FA2/FA3 only) |
|
||||
| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only |
|
||||
| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||
| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only |
|
||||
--8<-- "gen:table-mla-prefill"
|
||||
|
||||
> **‡** Automatic selection tries FlashAttention first. On Blackwell
|
||||
> (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then
|
||||
@@ -219,22 +160,7 @@ hardware and configuration.
|
||||
MLA decode backends are selected using the standard
|
||||
`-ac.backend=<BACKEND>` argument (e.g., `FLASHMLA`, `TRITON_MLA`).
|
||||
|
||||
| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. |
|
||||
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
|
||||
| `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x |
|
||||
| `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |
|
||||
| `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
|
||||
| `FLASH_ATTN_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x |
|
||||
| `FLASH_ATTN_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x |
|
||||
| `ROCM_AITER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %1 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
|
||||
| `ROCM_AITER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 1, 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | N/A |
|
||||
| `ROCM_AITER_TRITON_MLA` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
|
||||
| `TOKENSPEED_MLA` | fp16, bf16 | `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `TRITON_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | Any |
|
||||
| `XPU_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | Any | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | Any |
|
||||
--8<-- "gen:table-mla-decode"
|
||||
|
||||
### DeepSeek V4 Decode Backends
|
||||
|
||||
@@ -245,8 +171,4 @@ pipeline (compressor + SWA + indexer, 256-token blocks, head 512);
|
||||
default on NVIDIA is `FLASHINFER_MLA_SPARSE_DSV4` on SM12x and
|
||||
`FLASHMLA_SPARSE_DSV4` on other supported CUDA architectures.
|
||||
|
||||
| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. |
|
||||
| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
|
||||
| `FLASHINFER_MLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 256 | 512 | ✅ | ❌ | ✅ | ❌ | ❌ | Decoder | 10.x, 12.x |
|
||||
| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `fp8_ds_mla`, `fp8` | 256 | 512 | ✅ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |
|
||||
| `ROCM_FLASHMLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |
|
||||
--8<-- "gen:table-mla-v4-decode"
|
||||
|
||||
+183
-48
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
import inspect
|
||||
import logging
|
||||
import sys
|
||||
import textwrap
|
||||
@@ -10,17 +11,21 @@ from argparse import SUPPRESS, Action, HelpFormatter
|
||||
from collections.abc import Callable, Iterable
|
||||
from importlib.machinery import ModuleSpec
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import mkdocs_gen_files
|
||||
import regex as re
|
||||
from pydantic_core import core_schema
|
||||
|
||||
logger = logging.getLogger("mkdocs")
|
||||
|
||||
ROOT_DIR = Path(__file__).parent.parent.parent.parent
|
||||
ARGPARSE_DOC_DIR = ROOT_DIR / "docs/generated/argparse"
|
||||
|
||||
sys.path.insert(0, str(ROOT_DIR))
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from generated_content import fill_markers # noqa: E402
|
||||
|
||||
|
||||
def mock_if_no_torch(mock_module: str, mock: MagicMock):
|
||||
@@ -132,8 +137,8 @@ def auto_mock(module_name: str, attr: str, max_mocks: int = 100):
|
||||
|
||||
|
||||
bench_latency = auto_mock("vllm.benchmarks", "latency")
|
||||
bench_mm_processor = auto_mock("vllm.benchmarks", "mm_processor")
|
||||
bench_serve = auto_mock("vllm.benchmarks", "serve")
|
||||
bench_startup = auto_mock("vllm.benchmarks", "startup")
|
||||
bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs")
|
||||
bench_sweep_plot_pareto = auto_mock(
|
||||
"vllm.benchmarks.sweep.plot_pareto", "SweepPlotParetoArgs"
|
||||
@@ -142,12 +147,28 @@ bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs")
|
||||
bench_sweep_serve_workload = auto_mock(
|
||||
"vllm.benchmarks.sweep.serve_workload", "SweepServeWorkloadArgs"
|
||||
)
|
||||
bench_sweep_startup = auto_mock("vllm.benchmarks.sweep.startup", "SweepStartupArgs")
|
||||
bench_throughput = auto_mock("vllm.benchmarks", "throughput")
|
||||
AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs")
|
||||
EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs")
|
||||
ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand")
|
||||
CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand")
|
||||
BenchmarkSubcommand = auto_mock(
|
||||
"vllm.entrypoints.cli.benchmark.main", "BenchmarkSubcommand"
|
||||
)
|
||||
import_bench_subcommands = auto_mock(
|
||||
"vllm.entrypoints.cli.benchmark.main", "_import_bench_subcommand_modules"
|
||||
)
|
||||
BenchmarkSubcommandBase = auto_mock(
|
||||
"vllm.entrypoints.cli.benchmark.base", "BenchmarkSubcommandBase"
|
||||
)
|
||||
BenchmarkMMProcessorSubcommand = auto_mock(
|
||||
"vllm.entrypoints.cli.benchmark.mm_processor", "BenchmarkMMProcessorSubcommand"
|
||||
)
|
||||
LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase")
|
||||
launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION")
|
||||
RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand")
|
||||
sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS")
|
||||
openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args")
|
||||
openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch")
|
||||
|
||||
@@ -179,7 +200,7 @@ class MarkdownFormatter(HelpFormatter):
|
||||
|
||||
def add_text(self, text: str):
|
||||
if text:
|
||||
self._markdown_output.append(f"{text.strip()}\n\n")
|
||||
self._markdown_output.append(f"{inspect.cleandoc(text)}\n\n")
|
||||
|
||||
def add_usage(self, usage, actions, groups, prefix=None):
|
||||
pass
|
||||
@@ -241,49 +262,163 @@ def create_parser(add_cli_args, **kwargs) -> FlexibleArgumentParser:
|
||||
return _parser or parser
|
||||
|
||||
|
||||
def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
|
||||
logger.info("Generating argparse documentation")
|
||||
logger.debug("Root directory: %s", ROOT_DIR.resolve())
|
||||
logger.debug("Output directory: %s", ARGPARSE_DOC_DIR.resolve())
|
||||
|
||||
# Create the ARGPARSE_DOC_DIR if it doesn't exist
|
||||
if not ARGPARSE_DOC_DIR.exists():
|
||||
ARGPARSE_DOC_DIR.mkdir(parents=True)
|
||||
|
||||
# Create parsers to document
|
||||
parsers = {
|
||||
# Engine args
|
||||
"engine_args": create_parser(EngineArgs.add_cli_args),
|
||||
"async_engine_args": create_parser(
|
||||
AsyncEngineArgs.add_cli_args, async_args_only=True
|
||||
),
|
||||
# CLI
|
||||
"serve": create_parser(openai_cli_args.make_arg_parser),
|
||||
"chat": create_parser(ChatCommand.add_cli_args),
|
||||
"complete": create_parser(CompleteCommand.add_cli_args),
|
||||
"launch_render": create_parser(RenderSubcommand.add_cli_args),
|
||||
"run-batch": create_parser(openai_run_batch.make_arg_parser),
|
||||
# Benchmark CLI
|
||||
"bench_latency": create_parser(bench_latency.add_cli_args),
|
||||
"bench_mm_processor": create_parser(bench_mm_processor.add_cli_args),
|
||||
"bench_serve": create_parser(bench_serve.add_cli_args),
|
||||
"bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args),
|
||||
"bench_sweep_plot_pareto": create_parser(bench_sweep_plot_pareto.add_cli_args),
|
||||
"bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args),
|
||||
"bench_sweep_serve_workload": create_parser(
|
||||
bench_sweep_serve_workload.add_cli_args
|
||||
),
|
||||
"bench_throughput": create_parser(bench_throughput.add_cli_args),
|
||||
}
|
||||
|
||||
# Generate documentation for each parser
|
||||
for stem, parser in parsers.items():
|
||||
doc_path = ARGPARSE_DOC_DIR / f"{stem}.inc.md"
|
||||
# Specify encoding for building on Windows
|
||||
with open(doc_path, "w", encoding="utf-8") as f:
|
||||
f.write(super(type(parser), parser).format_help())
|
||||
logger.info("Argparse generated: %s", doc_path.relative_to(ROOT_DIR))
|
||||
def format_help(parser: FlexibleArgumentParser) -> str:
|
||||
"""Format a parser's help as markdown using `MarkdownFormatter`."""
|
||||
return super(type(parser), parser).format_help()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
on_startup("build", False)
|
||||
# Absolute docs URLs are kept in the help text because they are useful in the
|
||||
# terminal. Wrap them as markdown links so the `url_schemes` hook can rewrite
|
||||
# them into doc-relative links / cross-references at render time.
|
||||
_DOCS_URL = re.compile(r"https://docs\.vllm\.ai/en/[^/\s]+/[^\s)>]+")
|
||||
|
||||
|
||||
def linkify_docs_urls(text: str) -> str:
|
||||
"""Wrap bare docs.vllm.ai URLs in help text as markdown links."""
|
||||
return _DOCS_URL.sub(lambda m: f"[{m.group()}]({m.group()})", text)
|
||||
|
||||
|
||||
logger.info("Generating argparse documentation")
|
||||
logger.debug("Root directory: %s", ROOT_DIR.resolve())
|
||||
|
||||
# The JSON tip is always rendered immediately before generated argument content,
|
||||
# and the generator is its only consumer, so it lives here rather than in a
|
||||
# separate snippet file. (The runtime terminal equivalent is
|
||||
# `FlexibleArgumentParser._json_tip` in vllm/utils/argparse_utils.py.)
|
||||
JSON_TIP = """## JSON CLI Arguments
|
||||
|
||||
When passing JSON CLI arguments, the following sets of arguments are equivalent:
|
||||
|
||||
- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`
|
||||
- `--json-arg.key1 value1 --json-arg.key2.key3 value2`
|
||||
|
||||
Additionally, list elements can be passed individually using `+`:
|
||||
|
||||
- `--json-arg '{"key4": ["value3", "value4", "value5"]}'`
|
||||
- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`
|
||||
|
||||
"""
|
||||
|
||||
# Argument sections filled into `gen:` markers on handwritten pages
|
||||
engine_args = create_parser(EngineArgs.add_cli_args)
|
||||
async_engine_args = create_parser(AsyncEngineArgs.add_cli_args, async_args_only=True)
|
||||
fill_markers(
|
||||
"configuration/engine_args.md",
|
||||
{
|
||||
"engine-args": (
|
||||
f"{JSON_TIP}## `EngineArgs`\n\n"
|
||||
f"{linkify_docs_urls(format_help(engine_args))}"
|
||||
f"## `AsyncEngineArgs`\n\n"
|
||||
f"{linkify_docs_urls(format_help(async_engine_args))}"
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
# CLI reference pages generated entirely from their parser: page -> (parser, JSON tip)
|
||||
pages = {
|
||||
"cli/serve.md": (create_parser(openai_cli_args.make_arg_parser), True),
|
||||
"cli/chat.md": (create_parser(ChatCommand.add_cli_args), False),
|
||||
"cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False),
|
||||
"cli/run-batch.md": (create_parser(openai_run_batch.make_arg_parser), True),
|
||||
"cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True),
|
||||
"cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True),
|
||||
# URL kept as `mm_processor` for back-compat; command name is `mm-processor`
|
||||
"cli/bench/mm_processor.md": (
|
||||
create_parser(BenchmarkMMProcessorSubcommand.add_cli_args),
|
||||
True,
|
||||
),
|
||||
"cli/bench/serve.md": (create_parser(bench_serve.add_cli_args), True),
|
||||
"cli/bench/startup.md": (create_parser(bench_startup.add_cli_args), True),
|
||||
"cli/bench/throughput.md": (create_parser(bench_throughput.add_cli_args), True),
|
||||
"cli/bench/sweep/plot.md": (create_parser(bench_sweep_plot.add_cli_args), True),
|
||||
"cli/bench/sweep/plot_pareto.md": (
|
||||
create_parser(bench_sweep_plot_pareto.add_cli_args),
|
||||
True,
|
||||
),
|
||||
"cli/bench/sweep/serve.md": (create_parser(bench_sweep_serve.add_cli_args), True),
|
||||
"cli/bench/sweep/serve_workload.md": (
|
||||
create_parser(bench_sweep_serve_workload.add_cli_args),
|
||||
True,
|
||||
),
|
||||
"cli/bench/sweep/startup.md": (
|
||||
create_parser(bench_sweep_startup.add_cli_args),
|
||||
True,
|
||||
),
|
||||
}
|
||||
|
||||
# Command name for pages whose file stem differs (URL kept for back-compat).
|
||||
COMMAND_NAMES = {"cli/bench/mm_processor.md": "mm-processor"}
|
||||
|
||||
for doc_path, (parser, json_tip) in pages.items():
|
||||
segments = Path(doc_path).relative_to("cli").with_suffix("").parts
|
||||
label = COMMAND_NAMES.get(doc_path, segments[-1])
|
||||
command = " ".join([*segments[:-1], label])
|
||||
# `title` frontmatter keeps the nav label to just this command's segment,
|
||||
# while the H1 stays the full `vllm ...` command for the page heading.
|
||||
content = f"---\ntitle: {label}\n---\n\n"
|
||||
content += f"# vllm {command}\n\n"
|
||||
if parser.description:
|
||||
content += f"## Overview\n\n{parser.description}\n\n"
|
||||
# Rendered above instead of at the top of the Arguments section
|
||||
parser.description = None
|
||||
if json_tip:
|
||||
content += JSON_TIP
|
||||
content += f"## Arguments\n\n{linkify_docs_urls(format_help(parser))}"
|
||||
with mkdocs_gen_files.open(doc_path, "w") as f:
|
||||
f.write(content)
|
||||
logger.debug("CLI reference generated: %s", doc_path)
|
||||
|
||||
logger.info("Total argparse docs generated: %d", len(pages) + 2)
|
||||
|
||||
|
||||
# --- Bare subcommand (group) pages -------------------------------------------
|
||||
# Mirror `vllm <group> --help`: an overview plus a table of child subcommands,
|
||||
# each linked to its reference page. Children are read from the CLI registries
|
||||
# so the listing can never drift from the actual subcommands. Each page is the
|
||||
# `README.md` of its command directory so it becomes that section's index and is
|
||||
# picked up by the existing nav globs.
|
||||
import_bench_subcommands() # populate BenchmarkSubcommandBase.__subclasses__()
|
||||
bench_subcommands = BenchmarkSubcommandBase.__subclasses__()
|
||||
bench_children = [(cmd.name, cmd.help) for cmd in bench_subcommands]
|
||||
|
||||
groups = {
|
||||
"cli/bench/README.md": (BenchmarkSubcommand.help, bench_children),
|
||||
"cli/launch/README.md": (
|
||||
launch_description,
|
||||
[(cmd.name, cmd.help) for cmd in LaunchSubcommandBase.__subclasses__()],
|
||||
),
|
||||
"cli/bench/sweep/README.md": (
|
||||
dict(bench_children).get("sweep"),
|
||||
[(args.parser_name, args.parser_help) for args, _ in sweep_subcommands],
|
||||
),
|
||||
}
|
||||
|
||||
# Doc paths that exist, so we only link a child that has a reference page.
|
||||
existing_pages = set(pages) | set(groups)
|
||||
|
||||
|
||||
def child_link(group_doc: str, name: str) -> str | None:
|
||||
group_dir = Path(group_doc).parent # cli/bench/README.md -> cli/bench
|
||||
for stem in (name, name.replace("-", "_")):
|
||||
# A leaf page (bench/latency.md) or a nested group index (sweep/README.md)
|
||||
for candidate in (group_dir / f"{stem}.md", group_dir / stem / "README.md"):
|
||||
if candidate.as_posix() in existing_pages:
|
||||
return candidate.relative_to(group_dir).as_posix()
|
||||
return None
|
||||
|
||||
|
||||
for doc_path, (overview, children) in groups.items():
|
||||
title = "vllm " + Path(doc_path).parent.relative_to("cli").as_posix()
|
||||
lines = [f"# {title.replace('/', ' ')}", ""]
|
||||
if overview:
|
||||
lines += ["## Overview", "", overview.strip(), ""]
|
||||
lines += ["## Subcommands", "", "| Command | Description |", "| --- | --- |"]
|
||||
for name, summary in children:
|
||||
link = child_link(doc_path, name)
|
||||
command = f"[`{name}`]({link})" if link else f"`{name}`"
|
||||
lines.append(f"| {command} | {(summary or '').strip()} |")
|
||||
with mkdocs_gen_files.open(doc_path, "w") as f:
|
||||
f.write("\n".join(lines) + "\n")
|
||||
logger.debug("CLI group reference generated: %s", doc_path)
|
||||
|
||||
logger.info("CLI group reference pages generated: %d", len(groups))
|
||||
+61
-404
@@ -9,33 +9,28 @@ based on the checks in AttentionBackend.validate_configuration().
|
||||
|
||||
This approach avoids requiring CUDA/ROCm/GPU libraries to be installed.
|
||||
|
||||
When used as a pre-commit hook, this script receives filenames as arguments
|
||||
and only runs the check if any of the relevant files were modified.
|
||||
It runs as an mkdocs-gen-files script, so the page is generated at docs build
|
||||
time rather than being committed to the repository.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import fnmatch
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from generated_content import fill_markers # noqa: E402
|
||||
|
||||
logger = logging.getLogger("mkdocs")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants and file paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent.parent
|
||||
|
||||
RELEVANT_PATTERNS = [
|
||||
"vllm/v1/attention/backends/*.py",
|
||||
"vllm/v1/attention/backends/**/*.py",
|
||||
"vllm/models/minimax_m3/common/sparse_attention.py",
|
||||
"vllm/model_executor/layers/attention/mla_attention.py",
|
||||
"vllm/platforms/cuda.py",
|
||||
"tools/pre_commit/generate_attention_backend_docs.py",
|
||||
"docs/design/attention_backends.md",
|
||||
]
|
||||
REPO_ROOT = Path(__file__).parent.parent.parent.parent
|
||||
|
||||
BACKENDS_DIR = REPO_ROOT / "vllm" / "v1" / "attention" / "backends"
|
||||
REGISTRY_FILE = BACKENDS_DIR / "registry.py"
|
||||
@@ -55,19 +50,6 @@ BACKEND_KV_DTYPE_EXCLUDES: dict[str, set[str]] = {
|
||||
}
|
||||
|
||||
|
||||
def is_relevant_file(filepath: str) -> bool:
|
||||
"""Check if a file matches any of the relevant patterns."""
|
||||
path = Path(filepath)
|
||||
if path.is_absolute():
|
||||
try:
|
||||
path = path.relative_to(REPO_ROOT)
|
||||
except ValueError:
|
||||
return False
|
||||
path_str = str(path)
|
||||
|
||||
return any(fnmatch.fnmatch(path_str, pattern) for pattern in RELEVANT_PATTERNS)
|
||||
|
||||
|
||||
MLA_PREFILL_DIR = BACKENDS_DIR / "mla" / "prefill"
|
||||
MLA_PREFILL_REGISTRY_FILE = MLA_PREFILL_DIR / "registry.py"
|
||||
MLA_PREFILL_SELECTOR_FILE = MLA_PREFILL_DIR / "selector.py"
|
||||
@@ -960,7 +942,7 @@ def analyze_backend(backend_name: str, class_path: str) -> dict[str, Any] | None
|
||||
try:
|
||||
tree = ast.parse(file_path.read_text())
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not parse {file_path}: {e}", file=sys.stderr)
|
||||
logger.warning("Could not parse %s: %s", file_path, e)
|
||||
return None
|
||||
|
||||
class_name = class_path.rsplit(".", 1)[1]
|
||||
@@ -1657,113 +1639,12 @@ def _render_table(
|
||||
return lines
|
||||
|
||||
|
||||
def generate_markdown_table(
|
||||
backends: list[dict[str, Any]], title: str, is_mla_table: bool = False
|
||||
) -> str:
|
||||
"""Generate a titled markdown table from backend info."""
|
||||
if not backends:
|
||||
return f"## {title}\n\nNo backends found.\n"
|
||||
has_versions = any(b.get("version") for b in backends)
|
||||
columns = _build_columns(is_mla_table, has_versions)
|
||||
lines = [f"## {title}", ""]
|
||||
lines.extend(_render_table(columns, backends))
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown section generators (usage, priority, legend, MLA)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_usage_section() -> str:
|
||||
"""Generate the usage documentation section."""
|
||||
return """## Setting the Attention Backend
|
||||
|
||||
### Command Line
|
||||
|
||||
There are two ways to specify the backend from the command line:
|
||||
|
||||
**Option 1: Using `--attention-backend` (simple)**
|
||||
|
||||
```bash
|
||||
vllm serve <model> --attention-backend FLASH_ATTN
|
||||
```
|
||||
|
||||
**Option 2: Using `--attention-config.backend` / `-ac.backend` (structured config)**
|
||||
|
||||
```bash
|
||||
# Dot notation
|
||||
vllm serve <model> --attention-config.backend FLASH_ATTN
|
||||
vllm serve <model> -ac.backend FLASH_ATTN
|
||||
|
||||
# JSON format
|
||||
vllm serve <model> --attention-config '{"backend": "FLASH_ATTN"}'
|
||||
vllm serve <model> -ac '{"backend": "FLASH_ATTN"}'
|
||||
```
|
||||
|
||||
> **Note:** `--attention-backend` and `--attention-config.backend` are mutually
|
||||
> exclusive. Use one or the other, not both.
|
||||
|
||||
### Python API
|
||||
|
||||
Use `AttentionConfig` with the `LLM` class:
|
||||
|
||||
```python
|
||||
from vllm import LLM
|
||||
from vllm.config import AttentionConfig
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
# Method 1: Using AttentionConfig with enum
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
attention_config=AttentionConfig(backend=AttentionBackendEnum.FLASH_ATTN),
|
||||
)
|
||||
|
||||
# Method 2: Using attention_backend parameter with string
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
attention_backend="FLASH_ATTN",
|
||||
)
|
||||
```
|
||||
|
||||
## Backend Selection Behavior
|
||||
|
||||
### Manual Selection
|
||||
|
||||
When you explicitly set a backend via `--attention-backend` or `AttentionConfig`:
|
||||
|
||||
1. The backend is **validated** against your configuration (model dtype, head
|
||||
size, compute capability, etc.)
|
||||
2. If the backend **doesn't support** your configuration, an error is raised
|
||||
with the specific reason
|
||||
3. If valid, the backend is used
|
||||
|
||||
Example error when selecting an incompatible backend:
|
||||
|
||||
```text
|
||||
ValueError: Selected backend FLASHMLA is not valid for this configuration.
|
||||
Reason: ['compute capability not supported']
|
||||
```
|
||||
|
||||
### Automatic Selection
|
||||
|
||||
When no backend is specified (the default):
|
||||
|
||||
1. vLLM iterates through backends in **priority order** (see tables below)
|
||||
2. Each backend is validated against your configuration
|
||||
3. The **first compatible backend** is selected
|
||||
4. If no backend is compatible, an error is raised listing all backends and
|
||||
their incompatibility reasons
|
||||
"""
|
||||
|
||||
|
||||
def _priority_table(
|
||||
title: str,
|
||||
backends: list[str],
|
||||
annotations: dict[str, str] | None = None,
|
||||
) -> list[str]:
|
||||
"""Generate a priority table for a list of backends."""
|
||||
"""Render a priority table for a list of backends."""
|
||||
|
||||
def _fmt(b: str) -> str:
|
||||
suffix = annotations.get(b, "") if annotations else ""
|
||||
@@ -1779,102 +1660,38 @@ def _priority_table(
|
||||
]
|
||||
|
||||
|
||||
def generate_priority_section(priorities: dict[str, list[str]]) -> str:
|
||||
"""Generate the priority ranking section."""
|
||||
lines = [
|
||||
"## Backend Priority (CUDA)",
|
||||
"",
|
||||
"When no backend is explicitly selected, vLLM chooses the first",
|
||||
"compatible backend from these priority-ordered lists.",
|
||||
"",
|
||||
"Priority is **1 = highest** (tried first).",
|
||||
"",
|
||||
"### Standard Attention (MHA, MQA, GQA)",
|
||||
"",
|
||||
]
|
||||
|
||||
sm100 = "Blackwell (SM 10.x)"
|
||||
ampere = "Ampere/Hopper (SM 8.x-9.x)"
|
||||
|
||||
if "standard_sm100" in priorities:
|
||||
lines.extend(_priority_table(sm100, priorities["standard_sm100"]))
|
||||
if "standard_default" in priorities:
|
||||
lines.extend(_priority_table(ampere, priorities["standard_default"]))
|
||||
|
||||
lines.extend(["### MLA Attention (DeepSeek-style)", ""])
|
||||
|
||||
mla_sm100_annotations = {
|
||||
"FLASHINFER_MLA_SPARSE": "**\\***",
|
||||
}
|
||||
if "mla_sm100" in priorities:
|
||||
lines.extend(
|
||||
_priority_table(sm100, priorities["mla_sm100"], mla_sm100_annotations)
|
||||
)
|
||||
if "mla_default" in priorities:
|
||||
lines.extend(_priority_table(ampere, priorities["mla_default"]))
|
||||
|
||||
if "mla_sm100" in priorities:
|
||||
lines.append(
|
||||
"> **\\*** For sparse MLA, FP8 KV cache always prefers "
|
||||
"`FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` "
|
||||
"is preferred for low query-head counts (<= 16), while "
|
||||
"`FLASHMLA_SPARSE` is preferred otherwise."
|
||||
)
|
||||
lines.append(">")
|
||||
|
||||
lines.append(
|
||||
"> **Note:** ROCm and CPU platforms have their own selection logic. "
|
||||
"See the platform-specific documentation for details."
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
_SM100 = "Blackwell (SM 10.x)"
|
||||
_AMPERE = "Ampere/Hopper (SM 8.x-9.x)"
|
||||
|
||||
|
||||
def generate_legend() -> str:
|
||||
"""Generate a legend explaining the table columns."""
|
||||
return """## Legend
|
||||
|
||||
| Column | Description |
|
||||
| ------ | ----------- |
|
||||
| **Dtypes** | Supported model data types (fp16, bf16, fp32) |
|
||||
| **KV Dtypes** | Supported KV cache data types (`auto`, `fp8`, `fp8_e4m3`, etc.) |
|
||||
| **Block Sizes** | Supported KV cache block sizes (%N means multiples of N) |
|
||||
| **Head Sizes** | Supported attention head sizes |
|
||||
| **Sink** | Attention sink support (for StreamingLLM) |
|
||||
| **Non-Causal** | Non-causal (bidirectional) attention support for decoder models |
|
||||
| **Sparse** | Sparse attention support (MLA only) |
|
||||
| **MM Prefix** | Multimodal prefix full attention support |
|
||||
| **DCP** | Decode Context Parallelism support (`--decode-context-parallel-size`) |
|
||||
| **Attention Types** | Supported attention patterns (Decoder, Encoder, Enc-Dec) |
|
||||
| **Compute Cap.** | Required CUDA compute capability (N/A for non-CUDA backends) |
|
||||
|
||||
**Symbols:** ✅ = Supported, ❌ = Not supported
|
||||
"""
|
||||
|
||||
|
||||
def generate_mla_section(
|
||||
prefill_backends: list[dict[str, Any]],
|
||||
decode_backends: list[dict[str, Any]],
|
||||
v4_decode_backends: list[dict[str, Any]] | None = None,
|
||||
def _priority_block(
|
||||
priorities: dict[str, list[str]],
|
||||
sm100_key: str,
|
||||
default_key: str,
|
||||
sm100_annotations: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Generate the complete MLA section with prefill and decode tables."""
|
||||
"""Render whichever priority tables exist for one attention category."""
|
||||
lines: list[str] = []
|
||||
if sm100_key in priorities:
|
||||
lines += _priority_table(_SM100, priorities[sm100_key], sm100_annotations)
|
||||
if default_key in priorities:
|
||||
lines += _priority_table(_AMPERE, priorities[default_key])
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _feature_table(backends: list[dict[str, Any]], is_mla: bool) -> str:
|
||||
"""Render a backend feature table (header, separator, one row per backend)."""
|
||||
has_versions = any(b.get("version") for b in backends)
|
||||
columns = _build_columns(is_mla, has_versions)
|
||||
return "\n".join(_render_table(columns, backends))
|
||||
|
||||
|
||||
def _mla_prefill_table(prefill_backends: list[dict[str, Any]]) -> str:
|
||||
"""Render the MLA prefill backend table."""
|
||||
lines = [
|
||||
"## MLA (Multi-head Latent Attention) Backends",
|
||||
"",
|
||||
"MLA uses separate backends for prefill and decode phases.",
|
||||
"",
|
||||
"### Prefill Backends",
|
||||
"",
|
||||
"To explicitly select a prefill backend, use",
|
||||
"`-ac.mla_prefill_backend=<BACKEND>` (e.g., `FLASH_ATTN`, `FLASHINFER`).",
|
||||
"Otherwise, the prefill backend is selected automatically at runtime based on",
|
||||
"hardware and configuration.",
|
||||
"",
|
||||
"| Backend | Description | Dtypes | Compute Cap. | Notes |",
|
||||
"| ------- | ----------- | ------ | ------------ | ----- |",
|
||||
]
|
||||
|
||||
for backend in prefill_backends:
|
||||
row = "| `{}`{} | {} | {} | {} | {} |".format(
|
||||
backend["name"],
|
||||
@@ -1885,87 +1702,21 @@ def generate_mla_section(
|
||||
backend.get("notes", ""),
|
||||
)
|
||||
lines.append(row.replace(" ", " "))
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"> **‡** Automatic selection tries FlashAttention first. On Blackwell",
|
||||
"> (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then",
|
||||
"> TokenSpeed MLA. On other GPUs, only FlashAttention is considered.",
|
||||
"",
|
||||
"### Decode Backends",
|
||||
"",
|
||||
"MLA decode backends are selected using the standard",
|
||||
"`-ac.backend=<BACKEND>` argument (e.g., `FLASHMLA`, `TRITON_MLA`).",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
# Reuse data-driven table rendering for decode backends
|
||||
columns = _build_columns(is_mla=True, has_versions=False)
|
||||
lines.extend(_render_table(columns, decode_backends))
|
||||
|
||||
if v4_decode_backends:
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"### DeepSeek V4 Decode Backends",
|
||||
"",
|
||||
"DeepSeek V4 sparse MLA uses its own decode backends, selected via",
|
||||
"`--attention-backend=<BACKEND>` (e.g., `FLASHMLA_SPARSE_DSV4`,",
|
||||
"`FLASHINFER_MLA_SPARSE_DSV4`). They share the V4 sparse-index",
|
||||
"pipeline (compressor + SWA + indexer, 256-token blocks, head 512);",
|
||||
"default on NVIDIA is `FLASHINFER_MLA_SPARSE_DSV4` on SM12x and",
|
||||
"`FLASHMLA_SPARSE_DSV4` on other supported CUDA architectures.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(_render_table(columns, v4_decode_backends))
|
||||
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_minimax_section(backends: list[dict[str, Any]]) -> str:
|
||||
"""Generate the MiniMax M3 sparse attention section."""
|
||||
lines = [
|
||||
"## MiniMax M3 Sparse Attention Backends",
|
||||
"",
|
||||
'Block-sparse GQA backend used by MiniMax M3 sparse ("lightning indexer")',
|
||||
"layers. It is wired in directly by the model and is not part of the",
|
||||
"automatic priority lists above. A lightning indexer scores KV blocks, the",
|
||||
"top-k blocks (plus fixed init/local blocks) are selected, and attention",
|
||||
"attends only to those blocks; index keys live in a separate side cache.",
|
||||
"",
|
||||
]
|
||||
columns = _build_columns(is_mla=False, has_versions=False)
|
||||
lines.extend(_render_table(columns, backends))
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
def build_blocks() -> dict[str, str]:
|
||||
"""Build the generated table blocks keyed by their `gen:` marker name.
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Top-level orchestration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_docs() -> str:
|
||||
"""Generate the complete documentation."""
|
||||
Only the tables are generated here; the surrounding prose lives in the
|
||||
handwritten ``docs/design/attention_backends.md`` page.
|
||||
"""
|
||||
attention_backends_map = parse_registry()
|
||||
|
||||
# Parse priority lists from cuda.py
|
||||
priorities = parse_cuda_priority_lists()
|
||||
|
||||
# Parse FlashAttention FA2/FA3 feature differences
|
||||
fa_features = parse_flash_attn_features()
|
||||
|
||||
# Parse FlashInfer TRTLLM feature differences (native vs TRTLLM on Blackwell)
|
||||
fi_features = parse_flashinfer_trtllm_features()
|
||||
|
||||
# Parse MLA prefill backends
|
||||
mla_prefill_backends = parse_mla_prefill_backends()
|
||||
|
||||
# Collect backend info
|
||||
all_backends = []
|
||||
for backend_name, class_path in attention_backends_map.items():
|
||||
if backend_name in SKIP_BACKENDS:
|
||||
@@ -1973,17 +1724,14 @@ def generate_docs() -> str:
|
||||
info = analyze_backend(backend_name, class_path)
|
||||
if info:
|
||||
all_backends.append(info)
|
||||
|
||||
# Expand backends into version variants
|
||||
if fa_features:
|
||||
all_backends = _expand_flash_attn_variants(all_backends, fa_features)
|
||||
if fi_features:
|
||||
all_backends = _expand_flashinfer_variants(all_backends, fi_features)
|
||||
|
||||
# DeepSeek V4 (*_DSV4) decode backends and MiniMax M3 sparse backends each
|
||||
# get their own subsection rather than mixing into the main MLA / standard
|
||||
# tables (the ROCm V4 backend isn't flagged is_mla by the AST heuristic, so
|
||||
# filter purely on the name).
|
||||
# DeepSeek V4 (*_DSV4) and MiniMax M3 sparse backends get their own tables
|
||||
# rather than mixing into the main MLA / standard tables (the ROCm V4 backend
|
||||
# isn't flagged is_mla by the AST heuristic, so filter purely on the name).
|
||||
def _is_v4(b: dict[str, Any]) -> bool:
|
||||
return b["name"].endswith("_DSV4")
|
||||
|
||||
@@ -1999,112 +1747,21 @@ def generate_docs() -> str:
|
||||
if not b["is_mla"] and not _is_v4(b) and not _is_minimax(b)
|
||||
]
|
||||
|
||||
# Generate documentation
|
||||
script_path = "tools/pre_commit/generate_attention_backend_docs.py"
|
||||
doc_lines = [
|
||||
"# Attention Backend Feature Support",
|
||||
"",
|
||||
f"This document is auto-generated by `{script_path}`.",
|
||||
"It shows the feature support for each registered attention backend",
|
||||
"based on the checks in `AttentionBackend.validate_configuration()`.",
|
||||
"",
|
||||
"**Do not edit this file manually.** Run the following command to",
|
||||
"regenerate it:",
|
||||
"",
|
||||
"```bash",
|
||||
f"python {script_path}",
|
||||
"```",
|
||||
"",
|
||||
]
|
||||
|
||||
# Add usage documentation
|
||||
doc_lines.append(generate_usage_section())
|
||||
|
||||
# Add priority section
|
||||
doc_lines.append(generate_priority_section(priorities))
|
||||
|
||||
# Add legend and feature tables
|
||||
doc_lines.append(generate_legend())
|
||||
standard_title = "Standard Attention (MHA, MQA, GQA) Backends"
|
||||
doc_lines.append(
|
||||
generate_markdown_table(non_mla_backends, standard_title, is_mla_table=False)
|
||||
)
|
||||
# Add footnotes for version/variant distinctions (in table order)
|
||||
footnotes = []
|
||||
if fi_features:
|
||||
footnotes.append(
|
||||
"> **†** FlashInfer Native is the regular FlashInfer path. XQA is the "
|
||||
"SM90 decode path exposed through FlashInfer's TRTLLM decode API. "
|
||||
"trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen "
|
||||
"via `--attention-config.use_trtllm_attention=0`."
|
||||
)
|
||||
if fa_features:
|
||||
footnotes.append(
|
||||
"> **\\*** Specify the FlashAttention version via "
|
||||
"`--attention-config.flash_attn_version=2`, `3`, or `4`. "
|
||||
"Default is FA4 on SM100+ (Blackwell), FA3 on SM90 (Hopper), "
|
||||
"FA2 otherwise."
|
||||
)
|
||||
if footnotes:
|
||||
doc_lines.append("\n>\n".join(footnotes) + "\n")
|
||||
|
||||
# Add MiniMax M3 sparse section (separate category after standard GQA)
|
||||
if minimax_backends:
|
||||
doc_lines.append(generate_minimax_section(minimax_backends))
|
||||
|
||||
# Add MLA section with prefill and decode backends
|
||||
doc_lines.append(
|
||||
generate_mla_section(mla_prefill_backends, mla_backends, v4_decode_backends)
|
||||
)
|
||||
|
||||
return "\n".join(doc_lines)
|
||||
mla_sm100_annotations = {"FLASHINFER_MLA_SPARSE": "**\\***"}
|
||||
return {
|
||||
"priority-standard": _priority_block(
|
||||
priorities, "standard_sm100", "standard_default"
|
||||
),
|
||||
"priority-mla": _priority_block(
|
||||
priorities, "mla_sm100", "mla_default", mla_sm100_annotations
|
||||
),
|
||||
"table-standard": _feature_table(non_mla_backends, is_mla=False),
|
||||
"table-minimax": _feature_table(minimax_backends, is_mla=False),
|
||||
"table-mla-prefill": _mla_prefill_table(mla_prefill_backends),
|
||||
"table-mla-decode": _feature_table(mla_backends, is_mla=True),
|
||||
"table-mla-v4-decode": _feature_table(v4_decode_backends, is_mla=True),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate attention backend documentation table"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=str,
|
||||
default=str(REPO_ROOT / "docs" / "design" / "attention_backends.md"),
|
||||
help="Output file path (default: docs/design/attention_backends.md)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="Check if the documentation is up to date (for pre-commit)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"files",
|
||||
nargs="*",
|
||||
help="Files to check (passed by pre-commit). If none are relevant, skip.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.files and not any(is_relevant_file(f) for f in args.files):
|
||||
sys.exit(0)
|
||||
|
||||
output_path = Path(args.output)
|
||||
new_content = generate_docs()
|
||||
|
||||
if args.check:
|
||||
needs_update = (
|
||||
not output_path.exists() or output_path.read_text() != new_content
|
||||
)
|
||||
if needs_update:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(new_content)
|
||||
print(f"🔄 Regenerated: {output_path}")
|
||||
sys.exit(1)
|
||||
print(f"✅ Up to date: {output_path}")
|
||||
sys.exit(0)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(new_content)
|
||||
print(f"Generated: {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
logger.info("Generating attention backend documentation")
|
||||
fill_markers("design/attention_backends.md", build_blocks())
|
||||
+34
-41
@@ -5,16 +5,15 @@ import logging
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import mkdocs_awesome_nav.nav.directory as _nav_dir
|
||||
import mkdocs_gen_files
|
||||
import regex as re
|
||||
|
||||
logger = logging.getLogger("mkdocs")
|
||||
|
||||
ROOT_DIR = Path(__file__).parent.parent.parent.parent
|
||||
ROOT_DIR_RELATIVE = "../../../../.."
|
||||
EXAMPLE_DIR = ROOT_DIR / "examples"
|
||||
EXAMPLE_DOC_DIR = ROOT_DIR / "docs/examples"
|
||||
|
||||
|
||||
def title(text: str) -> str:
|
||||
@@ -197,44 +196,38 @@ class Example:
|
||||
return content
|
||||
|
||||
|
||||
def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
|
||||
# Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are
|
||||
# title-cased (e.g. "Offline Inference" instead of "Offline inference").
|
||||
import mkdocs_awesome_nav.nav.directory as _nav_dir
|
||||
# Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are
|
||||
# title-cased (e.g. "Offline Inference" instead of "Offline inference").
|
||||
_nav_dir.dirname_to_title = title
|
||||
logger.info("Generating example documentation")
|
||||
logger.debug("Root directory: %s", ROOT_DIR.resolve())
|
||||
logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())
|
||||
|
||||
_nav_dir.dirname_to_title = title
|
||||
logger.info("Generating example documentation")
|
||||
logger.debug("Root directory: %s", ROOT_DIR.resolve())
|
||||
logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())
|
||||
logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve())
|
||||
categories = sorted(
|
||||
p for p in EXAMPLE_DIR.iterdir() if p.is_dir() and not p.name.startswith(".")
|
||||
)
|
||||
|
||||
# Create the EXAMPLE_DOC_DIR if it doesn't exist
|
||||
if not EXAMPLE_DOC_DIR.exists():
|
||||
EXAMPLE_DOC_DIR.mkdir(parents=True)
|
||||
examples = []
|
||||
glob_patterns = ["*.py", "*.md", "*.sh"]
|
||||
# Find categorised examples
|
||||
for category in categories:
|
||||
logger.info("Processing category: %s", category.stem)
|
||||
globs = [category.glob(pattern) for pattern in glob_patterns]
|
||||
for path in itertools.chain(*globs):
|
||||
examples.append(Example(path, category.stem))
|
||||
# Find examples in subdirectories
|
||||
globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns]
|
||||
for path in itertools.chain(*globs):
|
||||
examples.append(Example(path.parent, category.stem))
|
||||
|
||||
categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir())
|
||||
|
||||
examples = []
|
||||
glob_patterns = ["*.py", "*.md", "*.sh"]
|
||||
# Find categorised examples
|
||||
for category in categories:
|
||||
logger.info("Processing category: %s", category.stem)
|
||||
globs = [category.glob(pattern) for pattern in glob_patterns]
|
||||
for path in itertools.chain(*globs):
|
||||
examples.append(Example(path, category.stem))
|
||||
# Find examples in subdirectories
|
||||
globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns]
|
||||
for path in itertools.chain(*globs):
|
||||
examples.append(Example(path.parent, category.stem))
|
||||
|
||||
# Generate the example documentation
|
||||
for example in sorted(examples, key=lambda e: e.path.stem):
|
||||
example_name = f"{example.path.stem}.md"
|
||||
doc_path = EXAMPLE_DOC_DIR / example.category / example_name
|
||||
if not doc_path.parent.exists():
|
||||
doc_path.parent.mkdir(parents=True)
|
||||
# Specify encoding for building on Windows
|
||||
with open(doc_path, "w+", encoding="utf-8") as f:
|
||||
f.write(example.generate())
|
||||
logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
|
||||
logger.info("Total examples generated: %d", len(examples))
|
||||
# Generate the example documentation
|
||||
for example in sorted(examples, key=lambda e: e.path.stem):
|
||||
doc_path = f"examples/{example.category}/{example.path.stem}.md"
|
||||
with mkdocs_gen_files.open(doc_path, "w") as f:
|
||||
f.write(example.generate())
|
||||
if example.main_file is not None:
|
||||
# Point the edit button at the example's source file
|
||||
edit_path = Path("..") / example.main_file.relative_to(ROOT_DIR)
|
||||
mkdocs_gen_files.set_edit_path(doc_path, str(edit_path))
|
||||
logger.debug("Example generated: %s", doc_path)
|
||||
logger.info("Total examples generated: %d", len(examples))
|
||||
@@ -2,27 +2,28 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import ast
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from generated_content import fill_markers # noqa: E402
|
||||
|
||||
logger = logging.getLogger("mkdocs")
|
||||
|
||||
ROOT_DIR = Path(__file__).parent.parent.parent.parent
|
||||
DOCS_DIR = ROOT_DIR / "docs"
|
||||
GENERATED_METRICS_DIR = DOCS_DIR / "generated" / "metrics"
|
||||
|
||||
# Files to scan for metric definitions - each will generate a separate table
|
||||
# Files to scan for metric definitions - each fills a `gen:` marker in
|
||||
# docs/usage/metrics.md with its table (the section heading and any preamble
|
||||
# live in the tracked page next to the marker).
|
||||
METRIC_SOURCE_FILES = [
|
||||
{"path": "vllm/v1/metrics/loggers.py", "output": "general.inc.md"},
|
||||
{
|
||||
"path": "vllm/v1/spec_decode/metrics.py",
|
||||
"output": "spec_decode.inc.md",
|
||||
},
|
||||
{"path": "vllm/v1/metrics/loggers.py", "key": "metrics-general"},
|
||||
{"path": "vllm/v1/spec_decode/metrics.py", "key": "metrics-spec-decode"},
|
||||
{
|
||||
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
|
||||
"output": "nixl_connector.inc.md",
|
||||
"key": "metrics-nixl",
|
||||
},
|
||||
{"path": "vllm/v1/metrics/perf.py", "output": "perf.inc.md"},
|
||||
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
|
||||
]
|
||||
|
||||
|
||||
@@ -110,41 +111,27 @@ def generate_markdown_table(metrics: list[dict[str, str]]) -> str:
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
|
||||
"""Generate metrics documentation tables from source files."""
|
||||
logger.info("Generating metrics documentation")
|
||||
logger.info("Generating metrics documentation")
|
||||
|
||||
# Create generated directory if it doesn't exist
|
||||
GENERATED_METRICS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
blocks = {}
|
||||
total_metrics = 0
|
||||
for source_config in METRIC_SOURCE_FILES:
|
||||
source_path = source_config["path"]
|
||||
|
||||
total_metrics = 0
|
||||
for source_config in METRIC_SOURCE_FILES:
|
||||
source_path = source_config["path"]
|
||||
output_file = source_config["output"]
|
||||
filepath = ROOT_DIR / source_path
|
||||
if not filepath.exists():
|
||||
raise FileNotFoundError(f"Metrics source file not found: {filepath}")
|
||||
|
||||
filepath = ROOT_DIR / source_path
|
||||
if not filepath.exists():
|
||||
raise FileNotFoundError(f"Metrics source file not found: {filepath}")
|
||||
logger.debug("Extracting metrics from: %s", source_path)
|
||||
metrics = extract_metrics_from_file(filepath)
|
||||
logger.debug("Found %d metrics in %s", len(metrics), source_path)
|
||||
|
||||
logger.debug("Extracting metrics from: %s", source_path)
|
||||
metrics = extract_metrics_from_file(filepath)
|
||||
logger.debug("Found %d metrics in %s", len(metrics), source_path)
|
||||
blocks[source_config["key"]] = generate_markdown_table(metrics).strip()
|
||||
total_metrics += len(metrics)
|
||||
|
||||
# Generate and write the markdown table for this source
|
||||
table_content = generate_markdown_table(metrics)
|
||||
output_path = GENERATED_METRICS_DIR / output_file
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
f.write(table_content)
|
||||
|
||||
total_metrics += len(metrics)
|
||||
logger.info(
|
||||
"Generated metrics table: %s (%d metrics)",
|
||||
output_path.relative_to(ROOT_DIR),
|
||||
len(metrics),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Total metrics generated: %d across %d files",
|
||||
total_metrics,
|
||||
len(METRIC_SOURCE_FILES),
|
||||
)
|
||||
fill_markers("usage/metrics.md", blocks)
|
||||
logger.info(
|
||||
"Total metrics generated: %d across %d files",
|
||||
total_metrics,
|
||||
len(METRIC_SOURCE_FILES),
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Inline build-time generated content into existing docs pages.
|
||||
|
||||
Source pages mark where generated content goes with a snippet-style marker,
|
||||
`--8<-- "gen:<key>"`, so the insertion point is explicit and readable. The
|
||||
substitution happens here (at gen-files time, before mkdocs-gen-files shadows
|
||||
the page), not via pymdownx.snippets, so the content can be generated at build
|
||||
time without living in a real file on disk.
|
||||
|
||||
The `gen:` prefix keeps these markers distinct from real pymdownx.snippets
|
||||
includes, and `fill_markers` fails loudly if a marker is missing or left behind
|
||||
(pymdownx.snippets would otherwise silently drop an unsubstituted marker).
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import mkdocs_gen_files
|
||||
import regex as re
|
||||
|
||||
DOCS_DIR = Path(__file__).parent.parent.parent
|
||||
|
||||
_MARKER = '--8<-- "gen:{key}"'
|
||||
_ANY_MARKER = re.compile(r'--8<-- "gen:[^"]*"')
|
||||
|
||||
|
||||
def fill_markers(doc_path: str, blocks: dict[str, str]) -> None:
|
||||
"""Replace `--8<-- "gen:<key>"` markers in a docs page with generated content.
|
||||
|
||||
Args:
|
||||
doc_path: Docs-relative path of the source page to fill.
|
||||
blocks: Mapping of marker key to the markdown to insert in its place.
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: If the source page does not exist.
|
||||
ValueError: If an expected marker is missing, or any `gen:` marker is
|
||||
left unsubstituted after filling.
|
||||
"""
|
||||
source = DOCS_DIR / doc_path
|
||||
if not source.exists():
|
||||
raise FileNotFoundError(f"Cannot fill markers in missing page: {doc_path}")
|
||||
|
||||
text = source.read_text()
|
||||
for key, content in blocks.items():
|
||||
marker = _MARKER.format(key=key)
|
||||
if marker not in text:
|
||||
raise ValueError(f"{doc_path}: missing marker {marker}")
|
||||
text = text.replace(marker, content)
|
||||
|
||||
if leftover := _ANY_MARKER.search(text):
|
||||
raise ValueError(f"{doc_path}: unsubstituted marker {leftover.group()}")
|
||||
|
||||
with mkdocs_gen_files.open(doc_path, "w") as f:
|
||||
f.write(text)
|
||||
# Keep the edit button pointing at the real source page
|
||||
mkdocs_gen_files.set_edit_path(doc_path, doc_path)
|
||||
@@ -19,6 +19,7 @@ The on_page_markdown hook passes the current page context to the preprocessor be
|
||||
each page is converted.
|
||||
"""
|
||||
|
||||
import posixpath
|
||||
from pathlib import Path
|
||||
|
||||
import regex as re
|
||||
@@ -38,18 +39,22 @@ TITLE = r"(?P<title>[^\[\]<>]+?)"
|
||||
REPO = r"(?P<repo>.+?/.+?)"
|
||||
TYPE = r"(?P<type>issues|pull|projects)"
|
||||
NUMBER = r"(?P<number>\d+)"
|
||||
VERSION = r"[^/\s]+"
|
||||
PATH = r"(?P<path>[^\s]+?)"
|
||||
FRAGMENT = r"(?P<fragment>#[^\s]+)?"
|
||||
URL = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}"
|
||||
URL_GITHUB = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}"
|
||||
RELATIVE = rf"(?!(https?|ftp)://|#){PATH}{FRAGMENT}"
|
||||
URL_DOCS = f"https://docs.vllm.ai/en/{VERSION}/{PATH}{FRAGMENT}"
|
||||
|
||||
# Common titles to use for GitHub links when none is provided in the link.
|
||||
TITLES = {"issues": "Issue ", "pull": "Pull Request ", "projects": "Project "}
|
||||
|
||||
# Regex to match GitHub issue, PR, and project links with optional titles.
|
||||
github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL}(\)|>)")
|
||||
github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL_GITHUB}(\)|>)")
|
||||
# Regex to match relative file links with optional titles.
|
||||
relative_link = re.compile(rf"\[{TITLE}\]\({RELATIVE}\)")
|
||||
# Regex to match absolute docs.vllm.ai links (should only exist in CLI).
|
||||
docs_link = re.compile(rf"\[{TITLE}\]\({URL_DOCS}\)")
|
||||
|
||||
|
||||
class UrlSchemesPreprocessor(Preprocessor):
|
||||
@@ -61,7 +66,8 @@ class UrlSchemesPreprocessor(Preprocessor):
|
||||
|
||||
def run(self, lines):
|
||||
page = self.ext.page
|
||||
if page is None or getattr(page.file, "abs_src_path", None) is None:
|
||||
files = self.ext.files
|
||||
if page is None:
|
||||
return lines
|
||||
|
||||
def replace_relative_link(match: re.Match) -> str:
|
||||
@@ -70,7 +76,7 @@ class UrlSchemesPreprocessor(Preprocessor):
|
||||
"""
|
||||
title = match.group("title")
|
||||
path = match.group("path")
|
||||
path = (Path(page.file.abs_src_path).parent / path).resolve()
|
||||
path = ((DOC_DIR / page.file.src_uri).parent / path).resolve()
|
||||
fragment = match.group("fragment") or ""
|
||||
|
||||
# Check if the path exists and is outside the docs dir
|
||||
@@ -105,9 +111,36 @@ class UrlSchemesPreprocessor(Preprocessor):
|
||||
url = f"https://github.com/{repo}/{type}/{number}{fragment}"
|
||||
return f"[{gh_icon} {title}]({url})"
|
||||
|
||||
def replace_docs_link(match: re.Match) -> str:
|
||||
"""Rewrite absolute docs.vllm.ai links as doc-relative links."""
|
||||
title = match.group("title")
|
||||
path = match.group("path").rstrip("/")
|
||||
fragment = match.group("fragment") or ""
|
||||
|
||||
# vllm.config.<Class> API reference -> mkdocstrings cross-reference
|
||||
if path == "api/vllm/config" and re.fullmatch(
|
||||
r"#vllm\.config\.\w+", fragment
|
||||
):
|
||||
ident = fragment[1:]
|
||||
return f"[`{ident}`][{ident}]"
|
||||
|
||||
# Other docs pages -> link relative to the current page, but only
|
||||
# when the target is a known docs page (real or generated); leave
|
||||
# unknown/external URLs untouched. This is correct even when the same
|
||||
# docstring is also rendered on its API reference page.
|
||||
src = f"{path.removesuffix('.html')}.md"
|
||||
if files.get_file_from_path(src) is None:
|
||||
return match.group(0)
|
||||
rel = posixpath.relpath(src, posixpath.dirname(page.file.src_uri))
|
||||
# Auto-wrapped bare URLs use the URL as their title; make it readable.
|
||||
if title.startswith("http"):
|
||||
title = path.removesuffix(".html")
|
||||
return f"[{title}]({rel}{fragment})"
|
||||
|
||||
markdown = "\n".join(lines)
|
||||
markdown = github_link.sub(replace_github_link, markdown)
|
||||
markdown = relative_link.sub(replace_relative_link, markdown)
|
||||
markdown = docs_link.sub(replace_docs_link, markdown)
|
||||
return markdown.split("\n")
|
||||
|
||||
|
||||
@@ -116,6 +149,7 @@ class UrlSchemesExtension(Extension):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.page = None
|
||||
self.files = None
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def extendMarkdown(self, md):
|
||||
@@ -138,4 +172,5 @@ def on_page_markdown(
|
||||
) -> str:
|
||||
"""Pass the current page context to the preprocessor."""
|
||||
_ext.page = page
|
||||
_ext.files = files
|
||||
return markdown
|
||||
|
||||
@@ -28,7 +28,7 @@ DOCS_PATHS=(
|
||||
docs/ # Actual docs content
|
||||
examples/ # Examples are rendered in docs
|
||||
vllm/ # API & CLI reference
|
||||
requirements/test/cuda.txt # CLI reference (see docs/mkdocs/hooks/generate_argparse.py)
|
||||
requirements/test/cuda.txt # CLI reference (see docs/mkdocs/gen_files/generate_argparse.py)
|
||||
mkdocs.yaml # Affects build process
|
||||
.readthedocs.yaml # Affects build process
|
||||
requirements/docs.txt # Affects build process
|
||||
|
||||
@@ -35,21 +35,21 @@ The following metrics are exposed:
|
||||
|
||||
## General Metrics
|
||||
|
||||
--8<-- "docs/generated/metrics/general.inc.md"
|
||||
--8<-- "gen:metrics-general"
|
||||
|
||||
## Speculative Decoding Metrics
|
||||
|
||||
--8<-- "docs/generated/metrics/spec_decode.inc.md"
|
||||
--8<-- "gen:metrics-spec-decode"
|
||||
|
||||
## NIXL KV Connector Metrics
|
||||
|
||||
--8<-- "docs/generated/metrics/nixl_connector.inc.md"
|
||||
--8<-- "gen:metrics-nixl"
|
||||
|
||||
## Model Flops Utilization (MFU) Performance Metrics
|
||||
|
||||
These metrics are available via `--enable-mfu-metrics`:
|
||||
|
||||
--8<-- "docs/generated/metrics/perf.inc.md"
|
||||
--8<-- "gen:metrics-mfu"
|
||||
|
||||
## Deprecation Policy
|
||||
|
||||
|
||||
+7
-10
@@ -3,7 +3,6 @@ site_url: !ENV READTHEDOCS_CANONICAL_URL
|
||||
repo_url: https://github.com/vllm-project/vllm
|
||||
edit_uri: edit/main/docs/
|
||||
exclude_docs: |
|
||||
argparse
|
||||
*.inc.md
|
||||
*.template.md
|
||||
theme:
|
||||
@@ -50,24 +49,22 @@ theme:
|
||||
|
||||
hooks:
|
||||
- docs/mkdocs/hooks/remove_announcement.py
|
||||
- docs/mkdocs/hooks/generate_examples.py
|
||||
- docs/mkdocs/hooks/generate_argparse.py
|
||||
- docs/mkdocs/hooks/generate_metrics.py
|
||||
- docs/mkdocs/hooks/url_schemes.py
|
||||
- docs/mkdocs/hooks/autoref_code.py
|
||||
|
||||
plugins:
|
||||
- meta
|
||||
- search
|
||||
- gen-files:
|
||||
scripts:
|
||||
- docs/mkdocs/gen_files/generate_examples.py
|
||||
- docs/mkdocs/gen_files/generate_argparse.py
|
||||
- docs/mkdocs/gen_files/generate_metrics.py
|
||||
- docs/mkdocs/gen_files/generate_attention_backends.py
|
||||
- autorefs
|
||||
- awesome-nav
|
||||
- glightbox
|
||||
- git-revision-date-localized:
|
||||
# exclude autogenerated files
|
||||
exclude:
|
||||
- api/*
|
||||
- examples/*
|
||||
- generated/*
|
||||
- git-revision-date-localized
|
||||
- minify:
|
||||
minify_html: true
|
||||
minify_js: true
|
||||
|
||||
@@ -137,13 +137,13 @@ class CacheConfig:
|
||||
still be controlled by mamba_cache_dtype). If set to 'auto', the data type
|
||||
for the ssm state will be determined by mamba_cache_dtype."""
|
||||
mamba_cache_mode: MambaCacheMode = "none"
|
||||
"""The cache strategy for Mamba layers.
|
||||
"""The cache strategy for Mamba layers:
|
||||
|
||||
- "none": set when prefix caching is disabled.
|
||||
- "all": cache the mamba state of all tokens at position i * block_size. This is
|
||||
the default behavior (for models that support it) when prefix caching is
|
||||
enabled.
|
||||
the default behavior (for models that support it) when prefix caching is enabled.
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size.
|
||||
when the token is at position i * block_size.
|
||||
"""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM history buffer length B: with use_replayssm, standard decode
|
||||
|
||||
@@ -249,17 +249,19 @@ def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:
|
||||
|
||||
NEEDS_HELP = (
|
||||
any("--help" in arg for arg in sys.argv) # vllm SUBCOMMAND --help
|
||||
or (argv0 := sys.argv[0]).endswith("mkdocs") # mkdocs SUBCOMMAND
|
||||
or argv0.endswith("mkdocs/__main__.py") # python -m mkdocs SUBCOMMAND
|
||||
or "mkdocs" in sys.modules # mkdocs SUBCOMMAND
|
||||
)
|
||||
|
||||
|
||||
def _maybe_add_docs_url(cls: Any) -> str:
|
||||
"""Generate API docs URL for a vllm config class."""
|
||||
if not cls.__module__.startswith("vllm.config"):
|
||||
import vllm.config
|
||||
|
||||
name = cls.__name__
|
||||
if getattr(vllm.config, name, None) is not cls:
|
||||
return ""
|
||||
version = f"v{VLLM_VERSION}" if "dev" not in VLLM_VERSION else "latest"
|
||||
return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{cls.__name__}"
|
||||
return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{name}"
|
||||
|
||||
|
||||
def _expand_json_human_readable_numbers(val: str) -> str:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import inspect
|
||||
|
||||
from vllm.benchmarks.mm_processor import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
@@ -8,13 +9,59 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `mm-processor` subcommand for `vllm bench`."""
|
||||
r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of
|
||||
vision-language models. It measures per-stage latency from the HuggingFace
|
||||
processor through to the encoder forward pass, helping you identify
|
||||
preprocessing bottlenecks and understand how different image resolutions or
|
||||
item counts affect end-to-end request time.
|
||||
|
||||
The benchmark supports two data sources: synthetic random multimodal inputs
|
||||
(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before
|
||||
measurement to ensure stable results.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
vllm bench mm-processor \
|
||||
--model Qwen/Qwen2-VL-7B-Instruct \
|
||||
--dataset-name random-mm \
|
||||
--num-prompts 50 \
|
||||
--random-input-len 300 \
|
||||
--random-output-len 40 \
|
||||
--random-mm-base-items-per-request 2 \
|
||||
--random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \
|
||||
--random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'
|
||||
```
|
||||
|
||||
## Measured Stages
|
||||
|
||||
| Stage | Description |
|
||||
| ----- | ----------- |
|
||||
| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |
|
||||
| `get_cache_missing_items_secs` | Time spent looking up the processor cache |
|
||||
| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |
|
||||
| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |
|
||||
| `apply_prompt_updates_secs` | Time spent updating prompt tokens |
|
||||
| `preprocessor_total_secs` | Total preprocessing time |
|
||||
| `encoder_forward_secs` | Time spent in the encoder model forward pass |
|
||||
| `num_encoder_calls` | Number of encoder invocations per request |
|
||||
|
||||
The benchmark also reports end-to-end latency (TTFT + decode time) per
|
||||
request. Use `--metric-percentiles` to select which percentiles to report
|
||||
(default: p99) and `--output-json` to save results.
|
||||
|
||||
For more examples (HF datasets, warmup, JSON output), see the
|
||||
[multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark).
|
||||
"""
|
||||
|
||||
name = "mm-processor"
|
||||
help = "Benchmark multimodal processor latency across different configurations."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
# The class docstring is the page overview / `--help` description.
|
||||
if cls.__doc__:
|
||||
parser.description = inspect.cleandoc(cls.__doc__)
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
import signal
|
||||
|
||||
import uvloop
|
||||
@@ -36,9 +37,12 @@ class LaunchSubcommandBase(CLISubcommand):
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
"""Add the CLI arguments to the parser.
|
||||
|
||||
By default, adds the standard vLLM serving arguments.
|
||||
By default, uses the subcommand's docstring as the description and adds
|
||||
the standard vLLM serving arguments.
|
||||
Subclasses can override to add component-specific arguments.
|
||||
"""
|
||||
if cls.__doc__:
|
||||
parser.description = inspect.cleandoc(cls.__doc__)
|
||||
make_arg_parser(parser)
|
||||
|
||||
@staticmethod
|
||||
@@ -47,7 +51,17 @@ class LaunchSubcommandBase(CLISubcommand):
|
||||
|
||||
|
||||
class RenderSubcommand(LaunchSubcommandBase):
|
||||
"""The `render` subcommand for `vllm launch`."""
|
||||
"""`vllm launch render` starts a GPU-less rendering server for preprocessing
|
||||
and postprocessing only.
|
||||
|
||||
```bash
|
||||
vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100
|
||||
```
|
||||
|
||||
This command reuses the standard serving parser, so model, frontend,
|
||||
networking, and related CLI options follow the same conventions as
|
||||
[`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/).
|
||||
"""
|
||||
|
||||
name = "render"
|
||||
help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."
|
||||
@@ -93,7 +107,6 @@ class LaunchSubcommand(CLISubcommand):
|
||||
cmd_subparser = launch_subparsers.add_parser(
|
||||
cmd_cls.name,
|
||||
help=cmd_cls.help,
|
||||
description=cmd_cls.help,
|
||||
usage=f"vllm {self.name} {cmd_cls.name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)
|
||||
|
||||
Reference in New Issue
Block a user