diff --git a/.gitignore b/.gitignore index 43787b8cd2..46418c0392 100644 --- a/.gitignore +++ b/.gitignore @@ -173,9 +173,6 @@ venv.bak/ # mkdocs documentation /site -docs/argparse -docs/examples/* -!docs/examples/README.md # mypy .mypy_cache/ diff --git a/.markdownlint.yaml b/.markdownlint.yaml index 937487f473..9140af925e 100644 --- a/.markdownlint.yaml +++ b/.markdownlint.yaml @@ -3,6 +3,9 @@ MD007: MD013: false MD024: siblings_only: true +MD025: + # Allow front matter title to be different from the first heading in the document. + front_matter_title: "" MD031: list_items: false MD033: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d26a51bba..a9f744bfa5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -260,10 +260,6 @@ repos: files: ^docker/(Dockerfile|versions\.json)$ pass_filenames: false additional_dependencies: [dockerfile-parse] - - id: attention-backend-docs - name: Check attention backend documentation is up to date - entry: python tools/pre_commit/generate_attention_backend_docs.py --check - language: python - id: check-boolean-context-manager name: Check for boolean ops in with-statements entry: python tools/pre_commit/check_boolean_context_manager.py diff --git a/docs/.nav.yml b/docs/.nav.yml index 7d985fdeb5..a2fae5d655 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -56,7 +56,9 @@ nav: - API Reference: - api/README.md - api/vllm - - CLI Reference: cli + - CLI Reference: + - cli/README.md + - vllm: cli - Community: - community/* - Governance: governance diff --git a/docs/cli/.nav.yml b/docs/cli/.nav.yml index 586685c5a1..1a6a5f6938 100644 --- a/docs/cli/.nav.yml +++ b/docs/cli/.nav.yml @@ -1,10 +1,8 @@ nav: - - README.md - - serve.md - - chat.md - - complete.md - - run-batch.md - - vllm bench: - - bench/**/*.md - - vllm launch: - - launch/**/*.md + - "*.md" + - bench: + - bench/*.md + - sweep: + - bench/sweep/*.md + - launch: + - launch/*.md diff --git a/docs/cli/bench/latency.md b/docs/cli/bench/latency.md deleted file mode 100644 index 9e1b905339..0000000000 --- a/docs/cli/bench/latency.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench latency - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_latency.inc.md" diff --git a/docs/cli/bench/mm_processor.md b/docs/cli/bench/mm_processor.md deleted file mode 100644 index 26746ce12d..0000000000 --- a/docs/cli/bench/mm_processor.md +++ /dev/null @@ -1,55 +0,0 @@ -# vllm bench mm-processor - -## Overview - -`vllm bench mm-processor` profiles the multimodal input processor pipeline of -vision-language models. It measures per-stage latency from the HuggingFace -processor through to the encoder forward pass, helping you identify -preprocessing bottlenecks and understand how different image resolutions or -item counts affect end-to-end request time. - -The benchmark supports two data sources: synthetic random multimodal inputs -(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before -measurement to ensure stable results. - -## Quick Start - -```bash -vllm bench mm-processor \ - --model Qwen/Qwen2-VL-7B-Instruct \ - --dataset-name random-mm \ - --num-prompts 50 \ - --random-input-len 300 \ - --random-output-len 40 \ - --random-mm-base-items-per-request 2 \ - --random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \ - --random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}' -``` - -## Measured Stages - -| Stage | Description | -| ----- | ----------- | -| `get_mm_hashes_secs` | Time spent hashing multimodal inputs | -| `get_cache_missing_items_secs` | Time spent looking up the processor cache | -| `apply_hf_processor_secs` | Time spent in the HuggingFace processor | -| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs | -| `apply_prompt_updates_secs` | Time spent updating prompt tokens | -| `preprocessor_total_secs` | Total preprocessing time | -| `encoder_forward_secs` | Time spent in the encoder model forward pass | -| `num_encoder_calls` | Number of encoder invocations per request | - -The benchmark also reports end-to-end latency (TTFT + decode time) per -request. Use `--metric-percentiles` to select which percentiles to report -(default: p99) and `--output-json` to save results. - -For more examples (HF datasets, warmup, JSON output), see -[Benchmarking CLI — Multimodal Processor Benchmark](../../benchmarking/cli.md#multimodal-processor-benchmark). - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_mm_processor.inc.md" diff --git a/docs/cli/bench/serve.md b/docs/cli/bench/serve.md deleted file mode 100644 index 792c6e094b..0000000000 --- a/docs/cli/bench/serve.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench serve - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_serve.inc.md" diff --git a/docs/cli/bench/sweep/plot.md b/docs/cli/bench/sweep/plot.md deleted file mode 100644 index d7dc65e6df..0000000000 --- a/docs/cli/bench/sweep/plot.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench sweep plot - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_sweep_plot.inc.md" diff --git a/docs/cli/bench/sweep/plot_pareto.md b/docs/cli/bench/sweep/plot_pareto.md deleted file mode 100644 index 13dffd7f2b..0000000000 --- a/docs/cli/bench/sweep/plot_pareto.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench sweep plot_pareto - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_sweep_plot_pareto.inc.md" diff --git a/docs/cli/bench/sweep/serve.md b/docs/cli/bench/sweep/serve.md deleted file mode 100644 index 6a8182feb4..0000000000 --- a/docs/cli/bench/sweep/serve.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench sweep serve - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_sweep_serve.inc.md" diff --git a/docs/cli/bench/sweep/serve_workload.md b/docs/cli/bench/sweep/serve_workload.md deleted file mode 100644 index 8c21788e8d..0000000000 --- a/docs/cli/bench/sweep/serve_workload.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench sweep serve_workload - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_sweep_serve_workload.inc.md" diff --git a/docs/cli/bench/throughput.md b/docs/cli/bench/throughput.md deleted file mode 100644 index 66434c8781..0000000000 --- a/docs/cli/bench/throughput.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm bench throughput - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/bench_throughput.inc.md" diff --git a/docs/cli/chat.md b/docs/cli/chat.md deleted file mode 100644 index 7b8e718f62..0000000000 --- a/docs/cli/chat.md +++ /dev/null @@ -1,5 +0,0 @@ -# vllm chat - -## Arguments - ---8<-- "docs/generated/argparse/chat.inc.md" diff --git a/docs/cli/complete.md b/docs/cli/complete.md deleted file mode 100644 index 65d953a7c0..0000000000 --- a/docs/cli/complete.md +++ /dev/null @@ -1,5 +0,0 @@ -# vllm complete - -## Arguments - ---8<-- "docs/generated/argparse/complete.inc.md" diff --git a/docs/cli/json_tip.inc.md b/docs/cli/json_tip.inc.md deleted file mode 100644 index 56c9cb2cc8..0000000000 --- a/docs/cli/json_tip.inc.md +++ /dev/null @@ -1,10 +0,0 @@ - -When passing JSON CLI arguments, the following sets of arguments are equivalent: - -- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'` -- `--json-arg.key1 value1 --json-arg.key2.key3 value2` - -Additionally, list elements can be passed individually using `+`: - -- `--json-arg '{"key4": ["value3", "value4", "value5"]}'` -- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'` diff --git a/docs/cli/launch/render.md b/docs/cli/launch/render.md deleted file mode 100644 index 4d15e5f116..0000000000 --- a/docs/cli/launch/render.md +++ /dev/null @@ -1,22 +0,0 @@ -# vllm launch render - -## Overview - -`vllm launch render` starts a GPU-less rendering server for preprocessing and -postprocessing only. - -```bash -vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100 -``` - -This command reuses the standard serving parser, so model, frontend, -networking, and related CLI options follow the same conventions as -[`vllm serve`](../serve.md). - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/launch_render.inc.md" diff --git a/docs/cli/run-batch.md b/docs/cli/run-batch.md deleted file mode 100644 index f2255e6637..0000000000 --- a/docs/cli/run-batch.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm run-batch - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/run-batch.inc.md" diff --git a/docs/cli/serve.md b/docs/cli/serve.md deleted file mode 100644 index 0326fe29ec..0000000000 --- a/docs/cli/serve.md +++ /dev/null @@ -1,9 +0,0 @@ -# vllm serve - -## JSON CLI Arguments - ---8<-- "docs/cli/json_tip.inc.md" - -## Arguments - ---8<-- "docs/generated/argparse/serve.inc.md" diff --git a/docs/configuration/engine_args.md b/docs/configuration/engine_args.md index b619cbf3db..11886cca7b 100644 --- a/docs/configuration/engine_args.md +++ b/docs/configuration/engine_args.md @@ -11,12 +11,4 @@ Engine arguments control the behavior of the vLLM engine. The engine argument classes, [EngineArgs][vllm.engine.arg_utils.EngineArgs] and [AsyncEngineArgs][vllm.engine.arg_utils.AsyncEngineArgs], are a combination of the configuration classes defined in [vllm.config][]. Therefore, if you are interested in developer documentation, we recommend looking at these configuration classes as they are the source of truth for types, defaults and docstrings. ---8<-- "docs/cli/json_tip.inc.md" - -## `EngineArgs` - ---8<-- "docs/generated/argparse/engine_args.inc.md" - -## `AsyncEngineArgs` - ---8<-- "docs/generated/argparse/async_engine_args.inc.md" +--8<-- "gen:engine-args" diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index 9497f70e3b..755ce8961e 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -1,15 +1,9 @@ # Attention Backend Feature Support -This document is auto-generated by `tools/pre_commit/generate_attention_backend_docs.py`. -It shows the feature support for each registered attention backend -based on the checks in `AttentionBackend.validate_configuration()`. - -**Do not edit this file manually.** Run the following command to -regenerate it: - -```bash -python tools/pre_commit/generate_attention_backend_docs.py -``` +The priority and feature tables on this page are auto-generated from the +attention backend registry by +`docs/mkdocs/gen_files/generate_attention_backends.py`, based on the checks in +`AttentionBackend.validate_configuration()`. ## Setting the Attention Backend @@ -98,40 +92,11 @@ Priority is **1 = highest** (tried first). ### Standard Attention (MHA, MQA, GQA) -**Blackwell (SM 10.x):** - -| Priority | Backend | -| -------- | ------- | -| 1 | `FLASHINFER` | -| 2 | `FLASH_ATTN` | -| 3 | `TRITON_ATTN` | -| 4 | `FLEX_ATTENTION` | -| 5 | `TURBOQUANT` | - -**Ampere/Hopper (SM 8.x-9.x):** - -| Priority | Backend | -| -------- | ------- | -| 1 | `FLASH_ATTN` | -| 2 | `FLASHINFER` | -| 3 | `TRITON_ATTN` | -| 4 | `FLEX_ATTENTION` | -| 5 | `TURBOQUANT` | +--8<-- "gen:priority-standard" ### MLA Attention (DeepSeek-style) -**Blackwell (SM 10.x):** - -| Priority | Backend | -| -------- | ------- | -| 1 | `FLASHINFER_MLA` | -| 2 | `TOKENSPEED_MLA` | -| 3 | `CUTLASS_MLA` | -| 4 | `FLASH_ATTN_MLA` | -| 5 | `FLASHMLA` | -| 6 | `TRITON_MLA` | -| 7 | `FLASHINFER_MLA_SPARSE`**\*** | -| 8 | `FLASHMLA_SPARSE` | +--8<-- "gen:priority-mla" > **\*** For sparse MLA, FP8 KV cache always prefers `FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` is preferred for low query-head counts (<= 16), while `FLASHMLA_SPARSE` is preferred otherwise. > @@ -157,24 +122,7 @@ Priority is **1 = highest** (tried first). ## Standard Attention (MHA, MQA, GQA) Backends -| Backend | Version | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. | -| ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ | -| `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ✅ | ❌ | ❌ | All | N/A | -| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 8.x-9.x | -| `FLASHINFER` | XQA† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 9.0 | -| `FLASHINFER` | trtllm-gen† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x | -| `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 | -| `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x | -| `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 | -| `FLASH_ATTN_DIFFKV` | | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ✅ | Decoder | Any | -| `FLEX_ATTENTION` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder Only | Any | -| `HPC_ATTN` | | fp16, bf16 | `auto`, `bfloat16`, `fp8_e4m3` | 64 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | ≥9.0 | -| `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ✅ | ✅ | ❌ | ❌ | Decoder | N/A | -| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A | -| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A | -| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int4_per_token_head`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ✅ | ✅ | ❌ | All | Any | -| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any | -| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any | +--8<-- "gen:table-standard" > **†** FlashInfer Native is the regular FlashInfer path. XQA is the SM90 decode path exposed through FlashInfer's TRTLLM decode API. trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen via `--attention-config.use_trtllm_attention=0`. > @@ -188,9 +136,7 @@ automatic priority lists above. A lightning indexer scores KV blocks, the top-k blocks (plus fixed init/local blocks) are selected, and attention attends only to those blocks; index keys live in a separate side cache. -| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. | -| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ | -| `MINIMAX_M3_SPARSE` | bf16, fp16 | `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 128 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | Any | +--8<-- "gen:table-minimax" ## MLA (Multi-head Latent Attention) Backends @@ -203,12 +149,7 @@ To explicitly select a prefill backend, use Otherwise, the prefill backend is selected automatically at runtime based on hardware and configuration. -| Backend | Description | Dtypes | Compute Cap. | Notes | -| ------- | ----------- | ------ | ------------ | ----- | -| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=64, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) (FA2/FA3 only) | -| `TRTLLM_RAGGED` | TensorRT-LLM ragged attention | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) only | -| `FLASHINFER` | FlashInfer CUTLASS backend | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only | -| `TOKENSPEED_MLA` | | fp16, bf16 | 10.x | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) only | +--8<-- "gen:table-mla-prefill" > **‡** Automatic selection tries FlashAttention first. On Blackwell > (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then @@ -219,22 +160,7 @@ hardware and configuration. MLA decode backends are selected using the standard `-ac.backend=` argument (e.g., `FLASHMLA`, `TRITON_MLA`). -| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. | -| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ | -| `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | -| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | -| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | -| `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x | -| `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x | -| `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x | -| `FLASH_ATTN_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x | -| `FLASH_ATTN_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x | -| `ROCM_AITER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %1 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A | -| `ROCM_AITER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 1, 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | N/A | -| `ROCM_AITER_TRITON_MLA` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A | -| `TOKENSPEED_MLA` | fp16, bf16 | `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | -| `TRITON_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | Any | -| `XPU_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | Any | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | Any | +--8<-- "gen:table-mla-decode" ### DeepSeek V4 Decode Backends @@ -245,8 +171,4 @@ pipeline (compressor + SWA + indexer, 256-token blocks, head 512); default on NVIDIA is `FLASHINFER_MLA_SPARSE_DSV4` on SM12x and `FLASHMLA_SPARSE_DSV4` on other supported CUDA architectures. -| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. | -| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ | -| `FLASHINFER_MLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 256 | 512 | ✅ | ❌ | ✅ | ❌ | ❌ | Decoder | 10.x, 12.x | -| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `fp8_ds_mla`, `fp8` | 256 | 512 | ✅ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x | -| `ROCM_FLASHMLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A | +--8<-- "gen:table-mla-v4-decode" diff --git a/docs/mkdocs/hooks/generate_argparse.py b/docs/mkdocs/gen_files/generate_argparse.py similarity index 51% rename from docs/mkdocs/hooks/generate_argparse.py rename to docs/mkdocs/gen_files/generate_argparse.py index 4548e33f88..c0fa707a1c 100644 --- a/docs/mkdocs/hooks/generate_argparse.py +++ b/docs/mkdocs/gen_files/generate_argparse.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib.metadata import importlib.util +import inspect import logging import sys import textwrap @@ -10,17 +11,21 @@ from argparse import SUPPRESS, Action, HelpFormatter from collections.abc import Callable, Iterable from importlib.machinery import ModuleSpec from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING from unittest.mock import MagicMock, patch +import mkdocs_gen_files +import regex as re from pydantic_core import core_schema logger = logging.getLogger("mkdocs") ROOT_DIR = Path(__file__).parent.parent.parent.parent -ARGPARSE_DOC_DIR = ROOT_DIR / "docs/generated/argparse" sys.path.insert(0, str(ROOT_DIR)) +sys.path.insert(0, str(Path(__file__).parent)) + +from generated_content import fill_markers # noqa: E402 def mock_if_no_torch(mock_module: str, mock: MagicMock): @@ -132,8 +137,8 @@ def auto_mock(module_name: str, attr: str, max_mocks: int = 100): bench_latency = auto_mock("vllm.benchmarks", "latency") -bench_mm_processor = auto_mock("vllm.benchmarks", "mm_processor") bench_serve = auto_mock("vllm.benchmarks", "serve") +bench_startup = auto_mock("vllm.benchmarks", "startup") bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs") bench_sweep_plot_pareto = auto_mock( "vllm.benchmarks.sweep.plot_pareto", "SweepPlotParetoArgs" @@ -142,12 +147,28 @@ bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs") bench_sweep_serve_workload = auto_mock( "vllm.benchmarks.sweep.serve_workload", "SweepServeWorkloadArgs" ) +bench_sweep_startup = auto_mock("vllm.benchmarks.sweep.startup", "SweepStartupArgs") bench_throughput = auto_mock("vllm.benchmarks", "throughput") AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs") EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs") ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand") CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand") +BenchmarkSubcommand = auto_mock( + "vllm.entrypoints.cli.benchmark.main", "BenchmarkSubcommand" +) +import_bench_subcommands = auto_mock( + "vllm.entrypoints.cli.benchmark.main", "_import_bench_subcommand_modules" +) +BenchmarkSubcommandBase = auto_mock( + "vllm.entrypoints.cli.benchmark.base", "BenchmarkSubcommandBase" +) +BenchmarkMMProcessorSubcommand = auto_mock( + "vllm.entrypoints.cli.benchmark.mm_processor", "BenchmarkMMProcessorSubcommand" +) +LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase") +launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION") RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand") +sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS") openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args") openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch") @@ -179,7 +200,7 @@ class MarkdownFormatter(HelpFormatter): def add_text(self, text: str): if text: - self._markdown_output.append(f"{text.strip()}\n\n") + self._markdown_output.append(f"{inspect.cleandoc(text)}\n\n") def add_usage(self, usage, actions, groups, prefix=None): pass @@ -241,49 +262,163 @@ def create_parser(add_cli_args, **kwargs) -> FlexibleArgumentParser: return _parser or parser -def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool): - logger.info("Generating argparse documentation") - logger.debug("Root directory: %s", ROOT_DIR.resolve()) - logger.debug("Output directory: %s", ARGPARSE_DOC_DIR.resolve()) - - # Create the ARGPARSE_DOC_DIR if it doesn't exist - if not ARGPARSE_DOC_DIR.exists(): - ARGPARSE_DOC_DIR.mkdir(parents=True) - - # Create parsers to document - parsers = { - # Engine args - "engine_args": create_parser(EngineArgs.add_cli_args), - "async_engine_args": create_parser( - AsyncEngineArgs.add_cli_args, async_args_only=True - ), - # CLI - "serve": create_parser(openai_cli_args.make_arg_parser), - "chat": create_parser(ChatCommand.add_cli_args), - "complete": create_parser(CompleteCommand.add_cli_args), - "launch_render": create_parser(RenderSubcommand.add_cli_args), - "run-batch": create_parser(openai_run_batch.make_arg_parser), - # Benchmark CLI - "bench_latency": create_parser(bench_latency.add_cli_args), - "bench_mm_processor": create_parser(bench_mm_processor.add_cli_args), - "bench_serve": create_parser(bench_serve.add_cli_args), - "bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args), - "bench_sweep_plot_pareto": create_parser(bench_sweep_plot_pareto.add_cli_args), - "bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args), - "bench_sweep_serve_workload": create_parser( - bench_sweep_serve_workload.add_cli_args - ), - "bench_throughput": create_parser(bench_throughput.add_cli_args), - } - - # Generate documentation for each parser - for stem, parser in parsers.items(): - doc_path = ARGPARSE_DOC_DIR / f"{stem}.inc.md" - # Specify encoding for building on Windows - with open(doc_path, "w", encoding="utf-8") as f: - f.write(super(type(parser), parser).format_help()) - logger.info("Argparse generated: %s", doc_path.relative_to(ROOT_DIR)) +def format_help(parser: FlexibleArgumentParser) -> str: + """Format a parser's help as markdown using `MarkdownFormatter`.""" + return super(type(parser), parser).format_help() -if __name__ == "__main__": - on_startup("build", False) +# Absolute docs URLs are kept in the help text because they are useful in the +# terminal. Wrap them as markdown links so the `url_schemes` hook can rewrite +# them into doc-relative links / cross-references at render time. +_DOCS_URL = re.compile(r"https://docs\.vllm\.ai/en/[^/\s]+/[^\s)>]+") + + +def linkify_docs_urls(text: str) -> str: + """Wrap bare docs.vllm.ai URLs in help text as markdown links.""" + return _DOCS_URL.sub(lambda m: f"[{m.group()}]({m.group()})", text) + + +logger.info("Generating argparse documentation") +logger.debug("Root directory: %s", ROOT_DIR.resolve()) + +# The JSON tip is always rendered immediately before generated argument content, +# and the generator is its only consumer, so it lives here rather than in a +# separate snippet file. (The runtime terminal equivalent is +# `FlexibleArgumentParser._json_tip` in vllm/utils/argparse_utils.py.) +JSON_TIP = """## JSON CLI Arguments + +When passing JSON CLI arguments, the following sets of arguments are equivalent: + +- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'` +- `--json-arg.key1 value1 --json-arg.key2.key3 value2` + +Additionally, list elements can be passed individually using `+`: + +- `--json-arg '{"key4": ["value3", "value4", "value5"]}'` +- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'` + +""" + +# Argument sections filled into `gen:` markers on handwritten pages +engine_args = create_parser(EngineArgs.add_cli_args) +async_engine_args = create_parser(AsyncEngineArgs.add_cli_args, async_args_only=True) +fill_markers( + "configuration/engine_args.md", + { + "engine-args": ( + f"{JSON_TIP}## `EngineArgs`\n\n" + f"{linkify_docs_urls(format_help(engine_args))}" + f"## `AsyncEngineArgs`\n\n" + f"{linkify_docs_urls(format_help(async_engine_args))}" + ) + }, +) + +# CLI reference pages generated entirely from their parser: page -> (parser, JSON tip) +pages = { + "cli/serve.md": (create_parser(openai_cli_args.make_arg_parser), True), + "cli/chat.md": (create_parser(ChatCommand.add_cli_args), False), + "cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False), + "cli/run-batch.md": (create_parser(openai_run_batch.make_arg_parser), True), + "cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True), + "cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True), + # URL kept as `mm_processor` for back-compat; command name is `mm-processor` + "cli/bench/mm_processor.md": ( + create_parser(BenchmarkMMProcessorSubcommand.add_cli_args), + True, + ), + "cli/bench/serve.md": (create_parser(bench_serve.add_cli_args), True), + "cli/bench/startup.md": (create_parser(bench_startup.add_cli_args), True), + "cli/bench/throughput.md": (create_parser(bench_throughput.add_cli_args), True), + "cli/bench/sweep/plot.md": (create_parser(bench_sweep_plot.add_cli_args), True), + "cli/bench/sweep/plot_pareto.md": ( + create_parser(bench_sweep_plot_pareto.add_cli_args), + True, + ), + "cli/bench/sweep/serve.md": (create_parser(bench_sweep_serve.add_cli_args), True), + "cli/bench/sweep/serve_workload.md": ( + create_parser(bench_sweep_serve_workload.add_cli_args), + True, + ), + "cli/bench/sweep/startup.md": ( + create_parser(bench_sweep_startup.add_cli_args), + True, + ), +} + +# Command name for pages whose file stem differs (URL kept for back-compat). +COMMAND_NAMES = {"cli/bench/mm_processor.md": "mm-processor"} + +for doc_path, (parser, json_tip) in pages.items(): + segments = Path(doc_path).relative_to("cli").with_suffix("").parts + label = COMMAND_NAMES.get(doc_path, segments[-1]) + command = " ".join([*segments[:-1], label]) + # `title` frontmatter keeps the nav label to just this command's segment, + # while the H1 stays the full `vllm ...` command for the page heading. + content = f"---\ntitle: {label}\n---\n\n" + content += f"# vllm {command}\n\n" + if parser.description: + content += f"## Overview\n\n{parser.description}\n\n" + # Rendered above instead of at the top of the Arguments section + parser.description = None + if json_tip: + content += JSON_TIP + content += f"## Arguments\n\n{linkify_docs_urls(format_help(parser))}" + with mkdocs_gen_files.open(doc_path, "w") as f: + f.write(content) + logger.debug("CLI reference generated: %s", doc_path) + +logger.info("Total argparse docs generated: %d", len(pages) + 2) + + +# --- Bare subcommand (group) pages ------------------------------------------- +# Mirror `vllm --help`: an overview plus a table of child subcommands, +# each linked to its reference page. Children are read from the CLI registries +# so the listing can never drift from the actual subcommands. Each page is the +# `README.md` of its command directory so it becomes that section's index and is +# picked up by the existing nav globs. +import_bench_subcommands() # populate BenchmarkSubcommandBase.__subclasses__() +bench_subcommands = BenchmarkSubcommandBase.__subclasses__() +bench_children = [(cmd.name, cmd.help) for cmd in bench_subcommands] + +groups = { + "cli/bench/README.md": (BenchmarkSubcommand.help, bench_children), + "cli/launch/README.md": ( + launch_description, + [(cmd.name, cmd.help) for cmd in LaunchSubcommandBase.__subclasses__()], + ), + "cli/bench/sweep/README.md": ( + dict(bench_children).get("sweep"), + [(args.parser_name, args.parser_help) for args, _ in sweep_subcommands], + ), +} + +# Doc paths that exist, so we only link a child that has a reference page. +existing_pages = set(pages) | set(groups) + + +def child_link(group_doc: str, name: str) -> str | None: + group_dir = Path(group_doc).parent # cli/bench/README.md -> cli/bench + for stem in (name, name.replace("-", "_")): + # A leaf page (bench/latency.md) or a nested group index (sweep/README.md) + for candidate in (group_dir / f"{stem}.md", group_dir / stem / "README.md"): + if candidate.as_posix() in existing_pages: + return candidate.relative_to(group_dir).as_posix() + return None + + +for doc_path, (overview, children) in groups.items(): + title = "vllm " + Path(doc_path).parent.relative_to("cli").as_posix() + lines = [f"# {title.replace('/', ' ')}", ""] + if overview: + lines += ["## Overview", "", overview.strip(), ""] + lines += ["## Subcommands", "", "| Command | Description |", "| --- | --- |"] + for name, summary in children: + link = child_link(doc_path, name) + command = f"[`{name}`]({link})" if link else f"`{name}`" + lines.append(f"| {command} | {(summary or '').strip()} |") + with mkdocs_gen_files.open(doc_path, "w") as f: + f.write("\n".join(lines) + "\n") + logger.debug("CLI group reference generated: %s", doc_path) + +logger.info("CLI group reference pages generated: %d", len(groups)) diff --git a/tools/pre_commit/generate_attention_backend_docs.py b/docs/mkdocs/gen_files/generate_attention_backends.py similarity index 81% rename from tools/pre_commit/generate_attention_backend_docs.py rename to docs/mkdocs/gen_files/generate_attention_backends.py index 875d19a0f5..942165eaa9 100644 --- a/tools/pre_commit/generate_attention_backend_docs.py +++ b/docs/mkdocs/gen_files/generate_attention_backends.py @@ -9,33 +9,28 @@ based on the checks in AttentionBackend.validate_configuration(). This approach avoids requiring CUDA/ROCm/GPU libraries to be installed. -When used as a pre-commit hook, this script receives filenames as arguments -and only runs the check if any of the relevant files were modified. +It runs as an mkdocs-gen-files script, so the page is generated at docs build +time rather than being committed to the repository. """ -import argparse import ast -import fnmatch +import logging import sys from collections.abc import Callable from pathlib import Path from typing import Any +sys.path.insert(0, str(Path(__file__).parent)) + +from generated_content import fill_markers # noqa: E402 + +logger = logging.getLogger("mkdocs") + # --------------------------------------------------------------------------- # Constants and file paths # --------------------------------------------------------------------------- -REPO_ROOT = Path(__file__).parent.parent.parent - -RELEVANT_PATTERNS = [ - "vllm/v1/attention/backends/*.py", - "vllm/v1/attention/backends/**/*.py", - "vllm/models/minimax_m3/common/sparse_attention.py", - "vllm/model_executor/layers/attention/mla_attention.py", - "vllm/platforms/cuda.py", - "tools/pre_commit/generate_attention_backend_docs.py", - "docs/design/attention_backends.md", -] +REPO_ROOT = Path(__file__).parent.parent.parent.parent BACKENDS_DIR = REPO_ROOT / "vllm" / "v1" / "attention" / "backends" REGISTRY_FILE = BACKENDS_DIR / "registry.py" @@ -55,19 +50,6 @@ BACKEND_KV_DTYPE_EXCLUDES: dict[str, set[str]] = { } -def is_relevant_file(filepath: str) -> bool: - """Check if a file matches any of the relevant patterns.""" - path = Path(filepath) - if path.is_absolute(): - try: - path = path.relative_to(REPO_ROOT) - except ValueError: - return False - path_str = str(path) - - return any(fnmatch.fnmatch(path_str, pattern) for pattern in RELEVANT_PATTERNS) - - MLA_PREFILL_DIR = BACKENDS_DIR / "mla" / "prefill" MLA_PREFILL_REGISTRY_FILE = MLA_PREFILL_DIR / "registry.py" MLA_PREFILL_SELECTOR_FILE = MLA_PREFILL_DIR / "selector.py" @@ -960,7 +942,7 @@ def analyze_backend(backend_name: str, class_path: str) -> dict[str, Any] | None try: tree = ast.parse(file_path.read_text()) except Exception as e: - print(f" Warning: Could not parse {file_path}: {e}", file=sys.stderr) + logger.warning("Could not parse %s: %s", file_path, e) return None class_name = class_path.rsplit(".", 1)[1] @@ -1657,113 +1639,12 @@ def _render_table( return lines -def generate_markdown_table( - backends: list[dict[str, Any]], title: str, is_mla_table: bool = False -) -> str: - """Generate a titled markdown table from backend info.""" - if not backends: - return f"## {title}\n\nNo backends found.\n" - has_versions = any(b.get("version") for b in backends) - columns = _build_columns(is_mla_table, has_versions) - lines = [f"## {title}", ""] - lines.extend(_render_table(columns, backends)) - lines.append("") - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Markdown section generators (usage, priority, legend, MLA) -# --------------------------------------------------------------------------- - - -def generate_usage_section() -> str: - """Generate the usage documentation section.""" - return """## Setting the Attention Backend - -### Command Line - -There are two ways to specify the backend from the command line: - -**Option 1: Using `--attention-backend` (simple)** - -```bash -vllm serve --attention-backend FLASH_ATTN -``` - -**Option 2: Using `--attention-config.backend` / `-ac.backend` (structured config)** - -```bash -# Dot notation -vllm serve --attention-config.backend FLASH_ATTN -vllm serve -ac.backend FLASH_ATTN - -# JSON format -vllm serve --attention-config '{"backend": "FLASH_ATTN"}' -vllm serve -ac '{"backend": "FLASH_ATTN"}' -``` - -> **Note:** `--attention-backend` and `--attention-config.backend` are mutually -> exclusive. Use one or the other, not both. - -### Python API - -Use `AttentionConfig` with the `LLM` class: - -```python -from vllm import LLM -from vllm.config import AttentionConfig -from vllm.v1.attention.backends.registry import AttentionBackendEnum - -# Method 1: Using AttentionConfig with enum -llm = LLM( - model="Qwen/Qwen3-0.6B", - attention_config=AttentionConfig(backend=AttentionBackendEnum.FLASH_ATTN), -) - -# Method 2: Using attention_backend parameter with string -llm = LLM( - model="Qwen/Qwen3-0.6B", - attention_backend="FLASH_ATTN", -) -``` - -## Backend Selection Behavior - -### Manual Selection - -When you explicitly set a backend via `--attention-backend` or `AttentionConfig`: - -1. The backend is **validated** against your configuration (model dtype, head - size, compute capability, etc.) -2. If the backend **doesn't support** your configuration, an error is raised - with the specific reason -3. If valid, the backend is used - -Example error when selecting an incompatible backend: - -```text -ValueError: Selected backend FLASHMLA is not valid for this configuration. -Reason: ['compute capability not supported'] -``` - -### Automatic Selection - -When no backend is specified (the default): - -1. vLLM iterates through backends in **priority order** (see tables below) -2. Each backend is validated against your configuration -3. The **first compatible backend** is selected -4. If no backend is compatible, an error is raised listing all backends and - their incompatibility reasons -""" - - def _priority_table( title: str, backends: list[str], annotations: dict[str, str] | None = None, ) -> list[str]: - """Generate a priority table for a list of backends.""" + """Render a priority table for a list of backends.""" def _fmt(b: str) -> str: suffix = annotations.get(b, "") if annotations else "" @@ -1779,102 +1660,38 @@ def _priority_table( ] -def generate_priority_section(priorities: dict[str, list[str]]) -> str: - """Generate the priority ranking section.""" - lines = [ - "## Backend Priority (CUDA)", - "", - "When no backend is explicitly selected, vLLM chooses the first", - "compatible backend from these priority-ordered lists.", - "", - "Priority is **1 = highest** (tried first).", - "", - "### Standard Attention (MHA, MQA, GQA)", - "", - ] - - sm100 = "Blackwell (SM 10.x)" - ampere = "Ampere/Hopper (SM 8.x-9.x)" - - if "standard_sm100" in priorities: - lines.extend(_priority_table(sm100, priorities["standard_sm100"])) - if "standard_default" in priorities: - lines.extend(_priority_table(ampere, priorities["standard_default"])) - - lines.extend(["### MLA Attention (DeepSeek-style)", ""]) - - mla_sm100_annotations = { - "FLASHINFER_MLA_SPARSE": "**\\***", - } - if "mla_sm100" in priorities: - lines.extend( - _priority_table(sm100, priorities["mla_sm100"], mla_sm100_annotations) - ) - if "mla_default" in priorities: - lines.extend(_priority_table(ampere, priorities["mla_default"])) - - if "mla_sm100" in priorities: - lines.append( - "> **\\*** For sparse MLA, FP8 KV cache always prefers " - "`FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` " - "is preferred for low query-head counts (<= 16), while " - "`FLASHMLA_SPARSE` is preferred otherwise." - ) - lines.append(">") - - lines.append( - "> **Note:** ROCm and CPU platforms have their own selection logic. " - "See the platform-specific documentation for details." - ) - lines.append("") - - return "\n".join(lines) +_SM100 = "Blackwell (SM 10.x)" +_AMPERE = "Ampere/Hopper (SM 8.x-9.x)" -def generate_legend() -> str: - """Generate a legend explaining the table columns.""" - return """## Legend - -| Column | Description | -| ------ | ----------- | -| **Dtypes** | Supported model data types (fp16, bf16, fp32) | -| **KV Dtypes** | Supported KV cache data types (`auto`, `fp8`, `fp8_e4m3`, etc.) | -| **Block Sizes** | Supported KV cache block sizes (%N means multiples of N) | -| **Head Sizes** | Supported attention head sizes | -| **Sink** | Attention sink support (for StreamingLLM) | -| **Non-Causal** | Non-causal (bidirectional) attention support for decoder models | -| **Sparse** | Sparse attention support (MLA only) | -| **MM Prefix** | Multimodal prefix full attention support | -| **DCP** | Decode Context Parallelism support (`--decode-context-parallel-size`) | -| **Attention Types** | Supported attention patterns (Decoder, Encoder, Enc-Dec) | -| **Compute Cap.** | Required CUDA compute capability (N/A for non-CUDA backends) | - -**Symbols:** ✅ = Supported, ❌ = Not supported -""" - - -def generate_mla_section( - prefill_backends: list[dict[str, Any]], - decode_backends: list[dict[str, Any]], - v4_decode_backends: list[dict[str, Any]] | None = None, +def _priority_block( + priorities: dict[str, list[str]], + sm100_key: str, + default_key: str, + sm100_annotations: dict[str, str] | None = None, ) -> str: - """Generate the complete MLA section with prefill and decode tables.""" + """Render whichever priority tables exist for one attention category.""" + lines: list[str] = [] + if sm100_key in priorities: + lines += _priority_table(_SM100, priorities[sm100_key], sm100_annotations) + if default_key in priorities: + lines += _priority_table(_AMPERE, priorities[default_key]) + return "\n".join(lines).strip() + + +def _feature_table(backends: list[dict[str, Any]], is_mla: bool) -> str: + """Render a backend feature table (header, separator, one row per backend).""" + has_versions = any(b.get("version") for b in backends) + columns = _build_columns(is_mla, has_versions) + return "\n".join(_render_table(columns, backends)) + + +def _mla_prefill_table(prefill_backends: list[dict[str, Any]]) -> str: + """Render the MLA prefill backend table.""" lines = [ - "## MLA (Multi-head Latent Attention) Backends", - "", - "MLA uses separate backends for prefill and decode phases.", - "", - "### Prefill Backends", - "", - "To explicitly select a prefill backend, use", - "`-ac.mla_prefill_backend=` (e.g., `FLASH_ATTN`, `FLASHINFER`).", - "Otherwise, the prefill backend is selected automatically at runtime based on", - "hardware and configuration.", - "", "| Backend | Description | Dtypes | Compute Cap. | Notes |", "| ------- | ----------- | ------ | ------------ | ----- |", ] - for backend in prefill_backends: row = "| `{}`{} | {} | {} | {} | {} |".format( backend["name"], @@ -1885,87 +1702,21 @@ def generate_mla_section( backend.get("notes", ""), ) lines.append(row.replace(" ", " ")) - - lines.extend( - [ - "", - "> **‡** Automatic selection tries FlashAttention first. On Blackwell", - "> (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then", - "> TokenSpeed MLA. On other GPUs, only FlashAttention is considered.", - "", - "### Decode Backends", - "", - "MLA decode backends are selected using the standard", - "`-ac.backend=` argument (e.g., `FLASHMLA`, `TRITON_MLA`).", - "", - ] - ) - - # Reuse data-driven table rendering for decode backends - columns = _build_columns(is_mla=True, has_versions=False) - lines.extend(_render_table(columns, decode_backends)) - - if v4_decode_backends: - lines.extend( - [ - "", - "### DeepSeek V4 Decode Backends", - "", - "DeepSeek V4 sparse MLA uses its own decode backends, selected via", - "`--attention-backend=` (e.g., `FLASHMLA_SPARSE_DSV4`,", - "`FLASHINFER_MLA_SPARSE_DSV4`). They share the V4 sparse-index", - "pipeline (compressor + SWA + indexer, 256-token blocks, head 512);", - "default on NVIDIA is `FLASHINFER_MLA_SPARSE_DSV4` on SM12x and", - "`FLASHMLA_SPARSE_DSV4` on other supported CUDA architectures.", - "", - ] - ) - lines.extend(_render_table(columns, v4_decode_backends)) - - lines.append("") return "\n".join(lines) -def generate_minimax_section(backends: list[dict[str, Any]]) -> str: - """Generate the MiniMax M3 sparse attention section.""" - lines = [ - "## MiniMax M3 Sparse Attention Backends", - "", - 'Block-sparse GQA backend used by MiniMax M3 sparse ("lightning indexer")', - "layers. It is wired in directly by the model and is not part of the", - "automatic priority lists above. A lightning indexer scores KV blocks, the", - "top-k blocks (plus fixed init/local blocks) are selected, and attention", - "attends only to those blocks; index keys live in a separate side cache.", - "", - ] - columns = _build_columns(is_mla=False, has_versions=False) - lines.extend(_render_table(columns, backends)) - lines.append("") - return "\n".join(lines) +def build_blocks() -> dict[str, str]: + """Build the generated table blocks keyed by their `gen:` marker name. - -# --------------------------------------------------------------------------- -# Top-level orchestration -# --------------------------------------------------------------------------- - - -def generate_docs() -> str: - """Generate the complete documentation.""" + Only the tables are generated here; the surrounding prose lives in the + handwritten ``docs/design/attention_backends.md`` page. + """ attention_backends_map = parse_registry() - - # Parse priority lists from cuda.py priorities = parse_cuda_priority_lists() - - # Parse FlashAttention FA2/FA3 feature differences fa_features = parse_flash_attn_features() - - # Parse FlashInfer TRTLLM feature differences (native vs TRTLLM on Blackwell) fi_features = parse_flashinfer_trtllm_features() - - # Parse MLA prefill backends mla_prefill_backends = parse_mla_prefill_backends() - # Collect backend info all_backends = [] for backend_name, class_path in attention_backends_map.items(): if backend_name in SKIP_BACKENDS: @@ -1973,17 +1724,14 @@ def generate_docs() -> str: info = analyze_backend(backend_name, class_path) if info: all_backends.append(info) - - # Expand backends into version variants if fa_features: all_backends = _expand_flash_attn_variants(all_backends, fa_features) if fi_features: all_backends = _expand_flashinfer_variants(all_backends, fi_features) - # DeepSeek V4 (*_DSV4) decode backends and MiniMax M3 sparse backends each - # get their own subsection rather than mixing into the main MLA / standard - # tables (the ROCm V4 backend isn't flagged is_mla by the AST heuristic, so - # filter purely on the name). + # DeepSeek V4 (*_DSV4) and MiniMax M3 sparse backends get their own tables + # rather than mixing into the main MLA / standard tables (the ROCm V4 backend + # isn't flagged is_mla by the AST heuristic, so filter purely on the name). def _is_v4(b: dict[str, Any]) -> bool: return b["name"].endswith("_DSV4") @@ -1999,112 +1747,21 @@ def generate_docs() -> str: if not b["is_mla"] and not _is_v4(b) and not _is_minimax(b) ] - # Generate documentation - script_path = "tools/pre_commit/generate_attention_backend_docs.py" - doc_lines = [ - "# Attention Backend Feature Support", - "", - f"This document is auto-generated by `{script_path}`.", - "It shows the feature support for each registered attention backend", - "based on the checks in `AttentionBackend.validate_configuration()`.", - "", - "**Do not edit this file manually.** Run the following command to", - "regenerate it:", - "", - "```bash", - f"python {script_path}", - "```", - "", - ] - - # Add usage documentation - doc_lines.append(generate_usage_section()) - - # Add priority section - doc_lines.append(generate_priority_section(priorities)) - - # Add legend and feature tables - doc_lines.append(generate_legend()) - standard_title = "Standard Attention (MHA, MQA, GQA) Backends" - doc_lines.append( - generate_markdown_table(non_mla_backends, standard_title, is_mla_table=False) - ) - # Add footnotes for version/variant distinctions (in table order) - footnotes = [] - if fi_features: - footnotes.append( - "> **†** FlashInfer Native is the regular FlashInfer path. XQA is the " - "SM90 decode path exposed through FlashInfer's TRTLLM decode API. " - "trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen " - "via `--attention-config.use_trtllm_attention=0`." - ) - if fa_features: - footnotes.append( - "> **\\*** Specify the FlashAttention version via " - "`--attention-config.flash_attn_version=2`, `3`, or `4`. " - "Default is FA4 on SM100+ (Blackwell), FA3 on SM90 (Hopper), " - "FA2 otherwise." - ) - if footnotes: - doc_lines.append("\n>\n".join(footnotes) + "\n") - - # Add MiniMax M3 sparse section (separate category after standard GQA) - if minimax_backends: - doc_lines.append(generate_minimax_section(minimax_backends)) - - # Add MLA section with prefill and decode backends - doc_lines.append( - generate_mla_section(mla_prefill_backends, mla_backends, v4_decode_backends) - ) - - return "\n".join(doc_lines) + mla_sm100_annotations = {"FLASHINFER_MLA_SPARSE": "**\\***"} + return { + "priority-standard": _priority_block( + priorities, "standard_sm100", "standard_default" + ), + "priority-mla": _priority_block( + priorities, "mla_sm100", "mla_default", mla_sm100_annotations + ), + "table-standard": _feature_table(non_mla_backends, is_mla=False), + "table-minimax": _feature_table(minimax_backends, is_mla=False), + "table-mla-prefill": _mla_prefill_table(mla_prefill_backends), + "table-mla-decode": _feature_table(mla_backends, is_mla=True), + "table-mla-v4-decode": _feature_table(v4_decode_backends, is_mla=True), + } -def main(): - parser = argparse.ArgumentParser( - description="Generate attention backend documentation table" - ) - parser.add_argument( - "--output", - "-o", - type=str, - default=str(REPO_ROOT / "docs" / "design" / "attention_backends.md"), - help="Output file path (default: docs/design/attention_backends.md)", - ) - parser.add_argument( - "--check", - action="store_true", - help="Check if the documentation is up to date (for pre-commit)", - ) - parser.add_argument( - "files", - nargs="*", - help="Files to check (passed by pre-commit). If none are relevant, skip.", - ) - args = parser.parse_args() - - if args.files and not any(is_relevant_file(f) for f in args.files): - sys.exit(0) - - output_path = Path(args.output) - new_content = generate_docs() - - if args.check: - needs_update = ( - not output_path.exists() or output_path.read_text() != new_content - ) - if needs_update: - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(new_content) - print(f"🔄 Regenerated: {output_path}") - sys.exit(1) - print(f"✅ Up to date: {output_path}") - sys.exit(0) - - output_path.parent.mkdir(parents=True, exist_ok=True) - output_path.write_text(new_content) - print(f"Generated: {output_path}") - - -if __name__ == "__main__": - main() +logger.info("Generating attention backend documentation") +fill_markers("design/attention_backends.md", build_blocks()) diff --git a/docs/mkdocs/hooks/generate_examples.py b/docs/mkdocs/gen_files/generate_examples.py similarity index 76% rename from docs/mkdocs/hooks/generate_examples.py rename to docs/mkdocs/gen_files/generate_examples.py index 07fbd7e4d5..c9c10018ec 100644 --- a/docs/mkdocs/hooks/generate_examples.py +++ b/docs/mkdocs/gen_files/generate_examples.py @@ -5,16 +5,15 @@ import logging from dataclasses import dataclass from functools import cached_property from pathlib import Path -from typing import Literal +import mkdocs_awesome_nav.nav.directory as _nav_dir +import mkdocs_gen_files import regex as re logger = logging.getLogger("mkdocs") ROOT_DIR = Path(__file__).parent.parent.parent.parent -ROOT_DIR_RELATIVE = "../../../../.." EXAMPLE_DIR = ROOT_DIR / "examples" -EXAMPLE_DOC_DIR = ROOT_DIR / "docs/examples" def title(text: str) -> str: @@ -197,44 +196,38 @@ class Example: return content -def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool): - # Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are - # title-cased (e.g. "Offline Inference" instead of "Offline inference"). - import mkdocs_awesome_nav.nav.directory as _nav_dir +# Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are +# title-cased (e.g. "Offline Inference" instead of "Offline inference"). +_nav_dir.dirname_to_title = title +logger.info("Generating example documentation") +logger.debug("Root directory: %s", ROOT_DIR.resolve()) +logger.debug("Example directory: %s", EXAMPLE_DIR.resolve()) - _nav_dir.dirname_to_title = title - logger.info("Generating example documentation") - logger.debug("Root directory: %s", ROOT_DIR.resolve()) - logger.debug("Example directory: %s", EXAMPLE_DIR.resolve()) - logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve()) +categories = sorted( + p for p in EXAMPLE_DIR.iterdir() if p.is_dir() and not p.name.startswith(".") +) - # Create the EXAMPLE_DOC_DIR if it doesn't exist - if not EXAMPLE_DOC_DIR.exists(): - EXAMPLE_DOC_DIR.mkdir(parents=True) +examples = [] +glob_patterns = ["*.py", "*.md", "*.sh"] +# Find categorised examples +for category in categories: + logger.info("Processing category: %s", category.stem) + globs = [category.glob(pattern) for pattern in glob_patterns] + for path in itertools.chain(*globs): + examples.append(Example(path, category.stem)) + # Find examples in subdirectories + globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns] + for path in itertools.chain(*globs): + examples.append(Example(path.parent, category.stem)) - categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir()) - - examples = [] - glob_patterns = ["*.py", "*.md", "*.sh"] - # Find categorised examples - for category in categories: - logger.info("Processing category: %s", category.stem) - globs = [category.glob(pattern) for pattern in glob_patterns] - for path in itertools.chain(*globs): - examples.append(Example(path, category.stem)) - # Find examples in subdirectories - globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns] - for path in itertools.chain(*globs): - examples.append(Example(path.parent, category.stem)) - - # Generate the example documentation - for example in sorted(examples, key=lambda e: e.path.stem): - example_name = f"{example.path.stem}.md" - doc_path = EXAMPLE_DOC_DIR / example.category / example_name - if not doc_path.parent.exists(): - doc_path.parent.mkdir(parents=True) - # Specify encoding for building on Windows - with open(doc_path, "w+", encoding="utf-8") as f: - f.write(example.generate()) - logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR)) - logger.info("Total examples generated: %d", len(examples)) +# Generate the example documentation +for example in sorted(examples, key=lambda e: e.path.stem): + doc_path = f"examples/{example.category}/{example.path.stem}.md" + with mkdocs_gen_files.open(doc_path, "w") as f: + f.write(example.generate()) + if example.main_file is not None: + # Point the edit button at the example's source file + edit_path = Path("..") / example.main_file.relative_to(ROOT_DIR) + mkdocs_gen_files.set_edit_path(doc_path, str(edit_path)) + logger.debug("Example generated: %s", doc_path) +logger.info("Total examples generated: %d", len(examples)) diff --git a/docs/mkdocs/hooks/generate_metrics.py b/docs/mkdocs/gen_files/generate_metrics.py similarity index 63% rename from docs/mkdocs/hooks/generate_metrics.py rename to docs/mkdocs/gen_files/generate_metrics.py index 97282aaee7..b952f53977 100644 --- a/docs/mkdocs/hooks/generate_metrics.py +++ b/docs/mkdocs/gen_files/generate_metrics.py @@ -2,27 +2,28 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import ast import logging +import sys from pathlib import Path -from typing import Literal + +sys.path.insert(0, str(Path(__file__).parent)) + +from generated_content import fill_markers # noqa: E402 logger = logging.getLogger("mkdocs") ROOT_DIR = Path(__file__).parent.parent.parent.parent -DOCS_DIR = ROOT_DIR / "docs" -GENERATED_METRICS_DIR = DOCS_DIR / "generated" / "metrics" -# Files to scan for metric definitions - each will generate a separate table +# Files to scan for metric definitions - each fills a `gen:` marker in +# docs/usage/metrics.md with its table (the section heading and any preamble +# live in the tracked page next to the marker). METRIC_SOURCE_FILES = [ - {"path": "vllm/v1/metrics/loggers.py", "output": "general.inc.md"}, - { - "path": "vllm/v1/spec_decode/metrics.py", - "output": "spec_decode.inc.md", - }, + {"path": "vllm/v1/metrics/loggers.py", "key": "metrics-general"}, + {"path": "vllm/v1/spec_decode/metrics.py", "key": "metrics-spec-decode"}, { "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py", - "output": "nixl_connector.inc.md", + "key": "metrics-nixl", }, - {"path": "vllm/v1/metrics/perf.py", "output": "perf.inc.md"}, + {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"}, ] @@ -110,41 +111,27 @@ def generate_markdown_table(metrics: list[dict[str, str]]) -> str: return "\n".join(lines) + "\n" -def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool): - """Generate metrics documentation tables from source files.""" - logger.info("Generating metrics documentation") +logger.info("Generating metrics documentation") - # Create generated directory if it doesn't exist - GENERATED_METRICS_DIR.mkdir(parents=True, exist_ok=True) +blocks = {} +total_metrics = 0 +for source_config in METRIC_SOURCE_FILES: + source_path = source_config["path"] - total_metrics = 0 - for source_config in METRIC_SOURCE_FILES: - source_path = source_config["path"] - output_file = source_config["output"] + filepath = ROOT_DIR / source_path + if not filepath.exists(): + raise FileNotFoundError(f"Metrics source file not found: {filepath}") - filepath = ROOT_DIR / source_path - if not filepath.exists(): - raise FileNotFoundError(f"Metrics source file not found: {filepath}") + logger.debug("Extracting metrics from: %s", source_path) + metrics = extract_metrics_from_file(filepath) + logger.debug("Found %d metrics in %s", len(metrics), source_path) - logger.debug("Extracting metrics from: %s", source_path) - metrics = extract_metrics_from_file(filepath) - logger.debug("Found %d metrics in %s", len(metrics), source_path) + blocks[source_config["key"]] = generate_markdown_table(metrics).strip() + total_metrics += len(metrics) - # Generate and write the markdown table for this source - table_content = generate_markdown_table(metrics) - output_path = GENERATED_METRICS_DIR / output_file - with open(output_path, "w", encoding="utf-8") as f: - f.write(table_content) - - total_metrics += len(metrics) - logger.info( - "Generated metrics table: %s (%d metrics)", - output_path.relative_to(ROOT_DIR), - len(metrics), - ) - - logger.info( - "Total metrics generated: %d across %d files", - total_metrics, - len(METRIC_SOURCE_FILES), - ) +fill_markers("usage/metrics.md", blocks) +logger.info( + "Total metrics generated: %d across %d files", + total_metrics, + len(METRIC_SOURCE_FILES), +) diff --git a/docs/mkdocs/gen_files/generated_content.py b/docs/mkdocs/gen_files/generated_content.py new file mode 100644 index 0000000000..fa3f31b294 --- /dev/null +++ b/docs/mkdocs/gen_files/generated_content.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Inline build-time generated content into existing docs pages. + +Source pages mark where generated content goes with a snippet-style marker, +`--8<-- "gen:"`, so the insertion point is explicit and readable. The +substitution happens here (at gen-files time, before mkdocs-gen-files shadows +the page), not via pymdownx.snippets, so the content can be generated at build +time without living in a real file on disk. + +The `gen:` prefix keeps these markers distinct from real pymdownx.snippets +includes, and `fill_markers` fails loudly if a marker is missing or left behind +(pymdownx.snippets would otherwise silently drop an unsubstituted marker). +""" + +from pathlib import Path + +import mkdocs_gen_files +import regex as re + +DOCS_DIR = Path(__file__).parent.parent.parent + +_MARKER = '--8<-- "gen:{key}"' +_ANY_MARKER = re.compile(r'--8<-- "gen:[^"]*"') + + +def fill_markers(doc_path: str, blocks: dict[str, str]) -> None: + """Replace `--8<-- "gen:"` markers in a docs page with generated content. + + Args: + doc_path: Docs-relative path of the source page to fill. + blocks: Mapping of marker key to the markdown to insert in its place. + + Raises: + FileNotFoundError: If the source page does not exist. + ValueError: If an expected marker is missing, or any `gen:` marker is + left unsubstituted after filling. + """ + source = DOCS_DIR / doc_path + if not source.exists(): + raise FileNotFoundError(f"Cannot fill markers in missing page: {doc_path}") + + text = source.read_text() + for key, content in blocks.items(): + marker = _MARKER.format(key=key) + if marker not in text: + raise ValueError(f"{doc_path}: missing marker {marker}") + text = text.replace(marker, content) + + if leftover := _ANY_MARKER.search(text): + raise ValueError(f"{doc_path}: unsubstituted marker {leftover.group()}") + + with mkdocs_gen_files.open(doc_path, "w") as f: + f.write(text) + # Keep the edit button pointing at the real source page + mkdocs_gen_files.set_edit_path(doc_path, doc_path) diff --git a/docs/mkdocs/hooks/url_schemes.py b/docs/mkdocs/hooks/url_schemes.py index e6faf95cd4..d208b9d677 100644 --- a/docs/mkdocs/hooks/url_schemes.py +++ b/docs/mkdocs/hooks/url_schemes.py @@ -19,6 +19,7 @@ The on_page_markdown hook passes the current page context to the preprocessor be each page is converted. """ +import posixpath from pathlib import Path import regex as re @@ -38,18 +39,22 @@ TITLE = r"(?P[^\[\]<>]+?)" REPO = r"(?P<repo>.+?/.+?)" TYPE = r"(?P<type>issues|pull|projects)" NUMBER = r"(?P<number>\d+)" +VERSION = r"[^/\s]+" PATH = r"(?P<path>[^\s]+?)" FRAGMENT = r"(?P<fragment>#[^\s]+)?" -URL = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}" +URL_GITHUB = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}" RELATIVE = rf"(?!(https?|ftp)://|#){PATH}{FRAGMENT}" +URL_DOCS = f"https://docs.vllm.ai/en/{VERSION}/{PATH}{FRAGMENT}" # Common titles to use for GitHub links when none is provided in the link. TITLES = {"issues": "Issue ", "pull": "Pull Request ", "projects": "Project "} # Regex to match GitHub issue, PR, and project links with optional titles. -github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL}(\)|>)") +github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL_GITHUB}(\)|>)") # Regex to match relative file links with optional titles. relative_link = re.compile(rf"\[{TITLE}\]\({RELATIVE}\)") +# Regex to match absolute docs.vllm.ai links (should only exist in CLI). +docs_link = re.compile(rf"\[{TITLE}\]\({URL_DOCS}\)") class UrlSchemesPreprocessor(Preprocessor): @@ -61,7 +66,8 @@ class UrlSchemesPreprocessor(Preprocessor): def run(self, lines): page = self.ext.page - if page is None or getattr(page.file, "abs_src_path", None) is None: + files = self.ext.files + if page is None: return lines def replace_relative_link(match: re.Match) -> str: @@ -70,7 +76,7 @@ class UrlSchemesPreprocessor(Preprocessor): """ title = match.group("title") path = match.group("path") - path = (Path(page.file.abs_src_path).parent / path).resolve() + path = ((DOC_DIR / page.file.src_uri).parent / path).resolve() fragment = match.group("fragment") or "" # Check if the path exists and is outside the docs dir @@ -105,9 +111,36 @@ class UrlSchemesPreprocessor(Preprocessor): url = f"https://github.com/{repo}/{type}/{number}{fragment}" return f"[{gh_icon} {title}]({url})" + def replace_docs_link(match: re.Match) -> str: + """Rewrite absolute docs.vllm.ai links as doc-relative links.""" + title = match.group("title") + path = match.group("path").rstrip("/") + fragment = match.group("fragment") or "" + + # vllm.config.<Class> API reference -> mkdocstrings cross-reference + if path == "api/vllm/config" and re.fullmatch( + r"#vllm\.config\.\w+", fragment + ): + ident = fragment[1:] + return f"[`{ident}`][{ident}]" + + # Other docs pages -> link relative to the current page, but only + # when the target is a known docs page (real or generated); leave + # unknown/external URLs untouched. This is correct even when the same + # docstring is also rendered on its API reference page. + src = f"{path.removesuffix('.html')}.md" + if files.get_file_from_path(src) is None: + return match.group(0) + rel = posixpath.relpath(src, posixpath.dirname(page.file.src_uri)) + # Auto-wrapped bare URLs use the URL as their title; make it readable. + if title.startswith("http"): + title = path.removesuffix(".html") + return f"[{title}]({rel}{fragment})" + markdown = "\n".join(lines) markdown = github_link.sub(replace_github_link, markdown) markdown = relative_link.sub(replace_relative_link, markdown) + markdown = docs_link.sub(replace_docs_link, markdown) return markdown.split("\n") @@ -116,6 +149,7 @@ class UrlSchemesExtension(Extension): def __init__(self, **kwargs): self.page = None + self.files = None super().__init__(**kwargs) def extendMarkdown(self, md): @@ -138,4 +172,5 @@ def on_page_markdown( ) -> str: """Pass the current page context to the preprocessor.""" _ext.page = page + _ext.files = files return markdown diff --git a/docs/pre_run_check.sh b/docs/pre_run_check.sh index d55f8c8db1..d611e61607 100644 --- a/docs/pre_run_check.sh +++ b/docs/pre_run_check.sh @@ -28,7 +28,7 @@ DOCS_PATHS=( docs/ # Actual docs content examples/ # Examples are rendered in docs vllm/ # API & CLI reference - requirements/test/cuda.txt # CLI reference (see docs/mkdocs/hooks/generate_argparse.py) + requirements/test/cuda.txt # CLI reference (see docs/mkdocs/gen_files/generate_argparse.py) mkdocs.yaml # Affects build process .readthedocs.yaml # Affects build process requirements/docs.txt # Affects build process diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 44c9c7cbfe..305c21478f 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -35,21 +35,21 @@ The following metrics are exposed: ## General Metrics ---8<-- "docs/generated/metrics/general.inc.md" +--8<-- "gen:metrics-general" ## Speculative Decoding Metrics ---8<-- "docs/generated/metrics/spec_decode.inc.md" +--8<-- "gen:metrics-spec-decode" ## NIXL KV Connector Metrics ---8<-- "docs/generated/metrics/nixl_connector.inc.md" +--8<-- "gen:metrics-nixl" ## Model Flops Utilization (MFU) Performance Metrics These metrics are available via `--enable-mfu-metrics`: ---8<-- "docs/generated/metrics/perf.inc.md" +--8<-- "gen:metrics-mfu" ## Deprecation Policy diff --git a/mkdocs.yaml b/mkdocs.yaml index a5c03c9e45..f3b9deab78 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -3,7 +3,6 @@ site_url: !ENV READTHEDOCS_CANONICAL_URL repo_url: https://github.com/vllm-project/vllm edit_uri: edit/main/docs/ exclude_docs: | - argparse *.inc.md *.template.md theme: @@ -50,24 +49,22 @@ theme: hooks: - docs/mkdocs/hooks/remove_announcement.py - - docs/mkdocs/hooks/generate_examples.py - - docs/mkdocs/hooks/generate_argparse.py - - docs/mkdocs/hooks/generate_metrics.py - docs/mkdocs/hooks/url_schemes.py - docs/mkdocs/hooks/autoref_code.py plugins: - meta - search + - gen-files: + scripts: + - docs/mkdocs/gen_files/generate_examples.py + - docs/mkdocs/gen_files/generate_argparse.py + - docs/mkdocs/gen_files/generate_metrics.py + - docs/mkdocs/gen_files/generate_attention_backends.py - autorefs - awesome-nav - glightbox - - git-revision-date-localized: - # exclude autogenerated files - exclude: - - api/* - - examples/* - - generated/* + - git-revision-date-localized - minify: minify_html: true minify_js: true diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 45c9362418..d267e9af35 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -137,13 +137,13 @@ class CacheConfig: still be controlled by mamba_cache_dtype). If set to 'auto', the data type for the ssm state will be determined by mamba_cache_dtype.""" mamba_cache_mode: MambaCacheMode = "none" - """The cache strategy for Mamba layers. + """The cache strategy for Mamba layers: + - "none": set when prefix caching is disabled. - "all": cache the mamba state of all tokens at position i * block_size. This is - the default behavior (for models that support it) when prefix caching is - enabled. + the default behavior (for models that support it) when prefix caching is enabled. - "align": only cache the mamba state of the last token of each scheduler step and - when the token is at position i * block_size. + when the token is at position i * block_size. """ replayssm_buffer_len: int = Field(default=16, gt=0) """ReplaySSM history buffer length B: with use_replayssm, standard decode diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index a14de27190..73a724c442 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -249,17 +249,19 @@ def get_type_hints(type_hint: TypeHint) -> set[TypeHint]: NEEDS_HELP = ( any("--help" in arg for arg in sys.argv) # vllm SUBCOMMAND --help - or (argv0 := sys.argv[0]).endswith("mkdocs") # mkdocs SUBCOMMAND - or argv0.endswith("mkdocs/__main__.py") # python -m mkdocs SUBCOMMAND + or "mkdocs" in sys.modules # mkdocs SUBCOMMAND ) def _maybe_add_docs_url(cls: Any) -> str: """Generate API docs URL for a vllm config class.""" - if not cls.__module__.startswith("vllm.config"): + import vllm.config + + name = cls.__name__ + if getattr(vllm.config, name, None) is not cls: return "" version = f"v{VLLM_VERSION}" if "dev" not in VLLM_VERSION else "latest" - return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{cls.__name__}" + return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{name}" def _expand_json_human_readable_numbers(val: str) -> str: diff --git a/vllm/entrypoints/cli/benchmark/mm_processor.py b/vllm/entrypoints/cli/benchmark/mm_processor.py index 26b93aacdc..de0d62e743 100644 --- a/vllm/entrypoints/cli/benchmark/mm_processor.py +++ b/vllm/entrypoints/cli/benchmark/mm_processor.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse +import inspect from vllm.benchmarks.mm_processor import add_cli_args, main from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase @@ -8,13 +9,59 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase): - """The `mm-processor` subcommand for `vllm bench`.""" + r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of + vision-language models. It measures per-stage latency from the HuggingFace + processor through to the encoder forward pass, helping you identify + preprocessing bottlenecks and understand how different image resolutions or + item counts affect end-to-end request time. + + The benchmark supports two data sources: synthetic random multimodal inputs + (`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before + measurement to ensure stable results. + + ## Quick Start + + ```bash + vllm bench mm-processor \ + --model Qwen/Qwen2-VL-7B-Instruct \ + --dataset-name random-mm \ + --num-prompts 50 \ + --random-input-len 300 \ + --random-output-len 40 \ + --random-mm-base-items-per-request 2 \ + --random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \ + --random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}' + ``` + + ## Measured Stages + + | Stage | Description | + | ----- | ----------- | + | `get_mm_hashes_secs` | Time spent hashing multimodal inputs | + | `get_cache_missing_items_secs` | Time spent looking up the processor cache | + | `apply_hf_processor_secs` | Time spent in the HuggingFace processor | + | `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs | + | `apply_prompt_updates_secs` | Time spent updating prompt tokens | + | `preprocessor_total_secs` | Total preprocessing time | + | `encoder_forward_secs` | Time spent in the encoder model forward pass | + | `num_encoder_calls` | Number of encoder invocations per request | + + The benchmark also reports end-to-end latency (TTFT + decode time) per + request. Use `--metric-percentiles` to select which percentiles to report + (default: p99) and `--output-json` to save results. + + For more examples (HF datasets, warmup, JSON output), see the + [multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark). + """ name = "mm-processor" help = "Benchmark multimodal processor latency across different configurations." @classmethod def add_cli_args(cls, parser: FlexibleArgumentParser) -> None: + # The class docstring is the page overview / `--help` description. + if cls.__doc__: + parser.description = inspect.cleandoc(cls.__doc__) add_cli_args(parser) @staticmethod diff --git a/vllm/entrypoints/cli/launch.py b/vllm/entrypoints/cli/launch.py index d13a0c67c8..91d4bf094b 100644 --- a/vllm/entrypoints/cli/launch.py +++ b/vllm/entrypoints/cli/launch.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import argparse +import inspect import signal import uvloop @@ -36,9 +37,12 @@ class LaunchSubcommandBase(CLISubcommand): def add_cli_args(cls, parser: FlexibleArgumentParser) -> None: """Add the CLI arguments to the parser. - By default, adds the standard vLLM serving arguments. + By default, uses the subcommand's docstring as the description and adds + the standard vLLM serving arguments. Subclasses can override to add component-specific arguments. """ + if cls.__doc__: + parser.description = inspect.cleandoc(cls.__doc__) make_arg_parser(parser) @staticmethod @@ -47,7 +51,17 @@ class LaunchSubcommandBase(CLISubcommand): class RenderSubcommand(LaunchSubcommandBase): - """The `render` subcommand for `vllm launch`.""" + """`vllm launch render` starts a GPU-less rendering server for preprocessing + and postprocessing only. + + ```bash + vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100 + ``` + + This command reuses the standard serving parser, so model, frontend, + networking, and related CLI options follow the same conventions as + [`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/). + """ name = "render" help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)." @@ -93,7 +107,6 @@ class LaunchSubcommand(CLISubcommand): cmd_subparser = launch_subparsers.add_parser( cmd_cls.name, help=cmd_cls.help, - description=cmd_cls.help, usage=f"vllm {self.name} {cmd_cls.name} [options]", ) cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)