[Docs] Use gen-files for generated docs content (#49587)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-25 14:43:21 +00:00
committed by GitHub
parent d1a8ba63d9
commit 3e74c60b9c
34 changed files with 515 additions and 861 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.metadata
import importlib.util
import inspect
import logging
import sys
import textwrap
@@ -10,17 +11,21 @@ from argparse import SUPPRESS, Action, HelpFormatter
from collections.abc import Callable, Iterable
from importlib.machinery import ModuleSpec
from pathlib import Path
from typing import TYPE_CHECKING, Literal
from typing import TYPE_CHECKING
from unittest.mock import MagicMock, patch
import mkdocs_gen_files
import regex as re
from pydantic_core import core_schema
logger = logging.getLogger("mkdocs")
ROOT_DIR = Path(__file__).parent.parent.parent.parent
ARGPARSE_DOC_DIR = ROOT_DIR / "docs/generated/argparse"
sys.path.insert(0, str(ROOT_DIR))
sys.path.insert(0, str(Path(__file__).parent))
from generated_content import fill_markers # noqa: E402
def mock_if_no_torch(mock_module: str, mock: MagicMock):
@@ -132,8 +137,8 @@ def auto_mock(module_name: str, attr: str, max_mocks: int = 100):
bench_latency = auto_mock("vllm.benchmarks", "latency")
bench_mm_processor = auto_mock("vllm.benchmarks", "mm_processor")
bench_serve = auto_mock("vllm.benchmarks", "serve")
bench_startup = auto_mock("vllm.benchmarks", "startup")
bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs")
bench_sweep_plot_pareto = auto_mock(
"vllm.benchmarks.sweep.plot_pareto", "SweepPlotParetoArgs"
@@ -142,12 +147,28 @@ bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs")
bench_sweep_serve_workload = auto_mock(
"vllm.benchmarks.sweep.serve_workload", "SweepServeWorkloadArgs"
)
bench_sweep_startup = auto_mock("vllm.benchmarks.sweep.startup", "SweepStartupArgs")
bench_throughput = auto_mock("vllm.benchmarks", "throughput")
AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs")
EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs")
ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand")
CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand")
BenchmarkSubcommand = auto_mock(
"vllm.entrypoints.cli.benchmark.main", "BenchmarkSubcommand"
)
import_bench_subcommands = auto_mock(
"vllm.entrypoints.cli.benchmark.main", "_import_bench_subcommand_modules"
)
BenchmarkSubcommandBase = auto_mock(
"vllm.entrypoints.cli.benchmark.base", "BenchmarkSubcommandBase"
)
BenchmarkMMProcessorSubcommand = auto_mock(
"vllm.entrypoints.cli.benchmark.mm_processor", "BenchmarkMMProcessorSubcommand"
)
LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase")
launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION")
RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand")
sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS")
openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args")
openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch")
@@ -179,7 +200,7 @@ class MarkdownFormatter(HelpFormatter):
def add_text(self, text: str):
if text:
self._markdown_output.append(f"{text.strip()}\n\n")
self._markdown_output.append(f"{inspect.cleandoc(text)}\n\n")
def add_usage(self, usage, actions, groups, prefix=None):
pass
@@ -241,49 +262,163 @@ def create_parser(add_cli_args, **kwargs) -> FlexibleArgumentParser:
return _parser or parser
def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
logger.info("Generating argparse documentation")
logger.debug("Root directory: %s", ROOT_DIR.resolve())
logger.debug("Output directory: %s", ARGPARSE_DOC_DIR.resolve())
# Create the ARGPARSE_DOC_DIR if it doesn't exist
if not ARGPARSE_DOC_DIR.exists():
ARGPARSE_DOC_DIR.mkdir(parents=True)
# Create parsers to document
parsers = {
# Engine args
"engine_args": create_parser(EngineArgs.add_cli_args),
"async_engine_args": create_parser(
AsyncEngineArgs.add_cli_args, async_args_only=True
),
# CLI
"serve": create_parser(openai_cli_args.make_arg_parser),
"chat": create_parser(ChatCommand.add_cli_args),
"complete": create_parser(CompleteCommand.add_cli_args),
"launch_render": create_parser(RenderSubcommand.add_cli_args),
"run-batch": create_parser(openai_run_batch.make_arg_parser),
# Benchmark CLI
"bench_latency": create_parser(bench_latency.add_cli_args),
"bench_mm_processor": create_parser(bench_mm_processor.add_cli_args),
"bench_serve": create_parser(bench_serve.add_cli_args),
"bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args),
"bench_sweep_plot_pareto": create_parser(bench_sweep_plot_pareto.add_cli_args),
"bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args),
"bench_sweep_serve_workload": create_parser(
bench_sweep_serve_workload.add_cli_args
),
"bench_throughput": create_parser(bench_throughput.add_cli_args),
}
# Generate documentation for each parser
for stem, parser in parsers.items():
doc_path = ARGPARSE_DOC_DIR / f"{stem}.inc.md"
# Specify encoding for building on Windows
with open(doc_path, "w", encoding="utf-8") as f:
f.write(super(type(parser), parser).format_help())
logger.info("Argparse generated: %s", doc_path.relative_to(ROOT_DIR))
def format_help(parser: FlexibleArgumentParser) -> str:
"""Format a parser's help as markdown using `MarkdownFormatter`."""
return super(type(parser), parser).format_help()
if __name__ == "__main__":
on_startup("build", False)
# Absolute docs URLs are kept in the help text because they are useful in the
# terminal. Wrap them as markdown links so the `url_schemes` hook can rewrite
# them into doc-relative links / cross-references at render time.
_DOCS_URL = re.compile(r"https://docs\.vllm\.ai/en/[^/\s]+/[^\s)>]+")
def linkify_docs_urls(text: str) -> str:
"""Wrap bare docs.vllm.ai URLs in help text as markdown links."""
return _DOCS_URL.sub(lambda m: f"[{m.group()}]({m.group()})", text)
logger.info("Generating argparse documentation")
logger.debug("Root directory: %s", ROOT_DIR.resolve())
# The JSON tip is always rendered immediately before generated argument content,
# and the generator is its only consumer, so it lives here rather than in a
# separate snippet file. (The runtime terminal equivalent is
# `FlexibleArgumentParser._json_tip` in vllm/utils/argparse_utils.py.)
JSON_TIP = """## JSON CLI Arguments
When passing JSON CLI arguments, the following sets of arguments are equivalent:
- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`
- `--json-arg.key1 value1 --json-arg.key2.key3 value2`
Additionally, list elements can be passed individually using `+`:
- `--json-arg '{"key4": ["value3", "value4", "value5"]}'`
- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`
"""
# Argument sections filled into `gen:` markers on handwritten pages
engine_args = create_parser(EngineArgs.add_cli_args)
async_engine_args = create_parser(AsyncEngineArgs.add_cli_args, async_args_only=True)
fill_markers(
"configuration/engine_args.md",
{
"engine-args": (
f"{JSON_TIP}## `EngineArgs`\n\n"
f"{linkify_docs_urls(format_help(engine_args))}"
f"## `AsyncEngineArgs`\n\n"
f"{linkify_docs_urls(format_help(async_engine_args))}"
)
},
)
# CLI reference pages generated entirely from their parser: page -> (parser, JSON tip)
pages = {
"cli/serve.md": (create_parser(openai_cli_args.make_arg_parser), True),
"cli/chat.md": (create_parser(ChatCommand.add_cli_args), False),
"cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False),
"cli/run-batch.md": (create_parser(openai_run_batch.make_arg_parser), True),
"cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True),
"cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True),
# URL kept as `mm_processor` for back-compat; command name is `mm-processor`
"cli/bench/mm_processor.md": (
create_parser(BenchmarkMMProcessorSubcommand.add_cli_args),
True,
),
"cli/bench/serve.md": (create_parser(bench_serve.add_cli_args), True),
"cli/bench/startup.md": (create_parser(bench_startup.add_cli_args), True),
"cli/bench/throughput.md": (create_parser(bench_throughput.add_cli_args), True),
"cli/bench/sweep/plot.md": (create_parser(bench_sweep_plot.add_cli_args), True),
"cli/bench/sweep/plot_pareto.md": (
create_parser(bench_sweep_plot_pareto.add_cli_args),
True,
),
"cli/bench/sweep/serve.md": (create_parser(bench_sweep_serve.add_cli_args), True),
"cli/bench/sweep/serve_workload.md": (
create_parser(bench_sweep_serve_workload.add_cli_args),
True,
),
"cli/bench/sweep/startup.md": (
create_parser(bench_sweep_startup.add_cli_args),
True,
),
}
# Command name for pages whose file stem differs (URL kept for back-compat).
COMMAND_NAMES = {"cli/bench/mm_processor.md": "mm-processor"}
for doc_path, (parser, json_tip) in pages.items():
segments = Path(doc_path).relative_to("cli").with_suffix("").parts
label = COMMAND_NAMES.get(doc_path, segments[-1])
command = " ".join([*segments[:-1], label])
# `title` frontmatter keeps the nav label to just this command's segment,
# while the H1 stays the full `vllm ...` command for the page heading.
content = f"---\ntitle: {label}\n---\n\n"
content += f"# vllm {command}\n\n"
if parser.description:
content += f"## Overview\n\n{parser.description}\n\n"
# Rendered above instead of at the top of the Arguments section
parser.description = None
if json_tip:
content += JSON_TIP
content += f"## Arguments\n\n{linkify_docs_urls(format_help(parser))}"
with mkdocs_gen_files.open(doc_path, "w") as f:
f.write(content)
logger.debug("CLI reference generated: %s", doc_path)
logger.info("Total argparse docs generated: %d", len(pages) + 2)
# --- Bare subcommand (group) pages -------------------------------------------
# Mirror `vllm <group> --help`: an overview plus a table of child subcommands,
# each linked to its reference page. Children are read from the CLI registries
# so the listing can never drift from the actual subcommands. Each page is the
# `README.md` of its command directory so it becomes that section's index and is
# picked up by the existing nav globs.
import_bench_subcommands() # populate BenchmarkSubcommandBase.__subclasses__()
bench_subcommands = BenchmarkSubcommandBase.__subclasses__()
bench_children = [(cmd.name, cmd.help) for cmd in bench_subcommands]
groups = {
"cli/bench/README.md": (BenchmarkSubcommand.help, bench_children),
"cli/launch/README.md": (
launch_description,
[(cmd.name, cmd.help) for cmd in LaunchSubcommandBase.__subclasses__()],
),
"cli/bench/sweep/README.md": (
dict(bench_children).get("sweep"),
[(args.parser_name, args.parser_help) for args, _ in sweep_subcommands],
),
}
# Doc paths that exist, so we only link a child that has a reference page.
existing_pages = set(pages) | set(groups)
def child_link(group_doc: str, name: str) -> str | None:
group_dir = Path(group_doc).parent # cli/bench/README.md -> cli/bench
for stem in (name, name.replace("-", "_")):
# A leaf page (bench/latency.md) or a nested group index (sweep/README.md)
for candidate in (group_dir / f"{stem}.md", group_dir / stem / "README.md"):
if candidate.as_posix() in existing_pages:
return candidate.relative_to(group_dir).as_posix()
return None
for doc_path, (overview, children) in groups.items():
title = "vllm " + Path(doc_path).parent.relative_to("cli").as_posix()
lines = [f"# {title.replace('/', ' ')}", ""]
if overview:
lines += ["## Overview", "", overview.strip(), ""]
lines += ["## Subcommands", "", "| Command | Description |", "| --- | --- |"]
for name, summary in children:
link = child_link(doc_path, name)
command = f"[`{name}`]({link})" if link else f"`{name}`"
lines.append(f"| {command} | {(summary or '').strip()} |")
with mkdocs_gen_files.open(doc_path, "w") as f:
f.write("\n".join(lines) + "\n")
logger.debug("CLI group reference generated: %s", doc_path)
logger.info("CLI group reference pages generated: %d", len(groups))
File diff suppressed because it is too large Load Diff
@@ -5,16 +5,15 @@ import logging
from dataclasses import dataclass
from functools import cached_property
from pathlib import Path
from typing import Literal
import mkdocs_awesome_nav.nav.directory as _nav_dir
import mkdocs_gen_files
import regex as re
logger = logging.getLogger("mkdocs")
ROOT_DIR = Path(__file__).parent.parent.parent.parent
ROOT_DIR_RELATIVE = "../../../../.."
EXAMPLE_DIR = ROOT_DIR / "examples"
EXAMPLE_DOC_DIR = ROOT_DIR / "docs/examples"
def title(text: str) -> str:
@@ -197,44 +196,38 @@ class Example:
return content
def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
# Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are
# title-cased (e.g. "Offline Inference" instead of "Offline inference").
import mkdocs_awesome_nav.nav.directory as _nav_dir
# Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are
# title-cased (e.g. "Offline Inference" instead of "Offline inference").
_nav_dir.dirname_to_title = title
logger.info("Generating example documentation")
logger.debug("Root directory: %s", ROOT_DIR.resolve())
logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())
_nav_dir.dirname_to_title = title
logger.info("Generating example documentation")
logger.debug("Root directory: %s", ROOT_DIR.resolve())
logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())
logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve())
categories = sorted(
p for p in EXAMPLE_DIR.iterdir() if p.is_dir() and not p.name.startswith(".")
)
# Create the EXAMPLE_DOC_DIR if it doesn't exist
if not EXAMPLE_DOC_DIR.exists():
EXAMPLE_DOC_DIR.mkdir(parents=True)
examples = []
glob_patterns = ["*.py", "*.md", "*.sh"]
# Find categorised examples
for category in categories:
logger.info("Processing category: %s", category.stem)
globs = [category.glob(pattern) for pattern in glob_patterns]
for path in itertools.chain(*globs):
examples.append(Example(path, category.stem))
# Find examples in subdirectories
globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns]
for path in itertools.chain(*globs):
examples.append(Example(path.parent, category.stem))
categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir())
examples = []
glob_patterns = ["*.py", "*.md", "*.sh"]
# Find categorised examples
for category in categories:
logger.info("Processing category: %s", category.stem)
globs = [category.glob(pattern) for pattern in glob_patterns]
for path in itertools.chain(*globs):
examples.append(Example(path, category.stem))
# Find examples in subdirectories
globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns]
for path in itertools.chain(*globs):
examples.append(Example(path.parent, category.stem))
# Generate the example documentation
for example in sorted(examples, key=lambda e: e.path.stem):
example_name = f"{example.path.stem}.md"
doc_path = EXAMPLE_DOC_DIR / example.category / example_name
if not doc_path.parent.exists():
doc_path.parent.mkdir(parents=True)
# Specify encoding for building on Windows
with open(doc_path, "w+", encoding="utf-8") as f:
f.write(example.generate())
logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
logger.info("Total examples generated: %d", len(examples))
# Generate the example documentation
for example in sorted(examples, key=lambda e: e.path.stem):
doc_path = f"examples/{example.category}/{example.path.stem}.md"
with mkdocs_gen_files.open(doc_path, "w") as f:
f.write(example.generate())
if example.main_file is not None:
# Point the edit button at the example's source file
edit_path = Path("..") / example.main_file.relative_to(ROOT_DIR)
mkdocs_gen_files.set_edit_path(doc_path, str(edit_path))
logger.debug("Example generated: %s", doc_path)
logger.info("Total examples generated: %d", len(examples))
@@ -2,27 +2,28 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
import logging
import sys
from pathlib import Path
from typing import Literal
sys.path.insert(0, str(Path(__file__).parent))
from generated_content import fill_markers # noqa: E402
logger = logging.getLogger("mkdocs")
ROOT_DIR = Path(__file__).parent.parent.parent.parent
DOCS_DIR = ROOT_DIR / "docs"
GENERATED_METRICS_DIR = DOCS_DIR / "generated" / "metrics"
# Files to scan for metric definitions - each will generate a separate table
# Files to scan for metric definitions - each fills a `gen:` marker in
# docs/usage/metrics.md with its table (the section heading and any preamble
# live in the tracked page next to the marker).
METRIC_SOURCE_FILES = [
{"path": "vllm/v1/metrics/loggers.py", "output": "general.inc.md"},
{
"path": "vllm/v1/spec_decode/metrics.py",
"output": "spec_decode.inc.md",
},
{"path": "vllm/v1/metrics/loggers.py", "key": "metrics-general"},
{"path": "vllm/v1/spec_decode/metrics.py", "key": "metrics-spec-decode"},
{
"path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",
"output": "nixl_connector.inc.md",
"key": "metrics-nixl",
},
{"path": "vllm/v1/metrics/perf.py", "output": "perf.inc.md"},
{"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},
]
@@ -110,41 +111,27 @@ def generate_markdown_table(metrics: list[dict[str, str]]) -> str:
return "\n".join(lines) + "\n"
def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
"""Generate metrics documentation tables from source files."""
logger.info("Generating metrics documentation")
logger.info("Generating metrics documentation")
# Create generated directory if it doesn't exist
GENERATED_METRICS_DIR.mkdir(parents=True, exist_ok=True)
blocks = {}
total_metrics = 0
for source_config in METRIC_SOURCE_FILES:
source_path = source_config["path"]
total_metrics = 0
for source_config in METRIC_SOURCE_FILES:
source_path = source_config["path"]
output_file = source_config["output"]
filepath = ROOT_DIR / source_path
if not filepath.exists():
raise FileNotFoundError(f"Metrics source file not found: {filepath}")
filepath = ROOT_DIR / source_path
if not filepath.exists():
raise FileNotFoundError(f"Metrics source file not found: {filepath}")
logger.debug("Extracting metrics from: %s", source_path)
metrics = extract_metrics_from_file(filepath)
logger.debug("Found %d metrics in %s", len(metrics), source_path)
logger.debug("Extracting metrics from: %s", source_path)
metrics = extract_metrics_from_file(filepath)
logger.debug("Found %d metrics in %s", len(metrics), source_path)
blocks[source_config["key"]] = generate_markdown_table(metrics).strip()
total_metrics += len(metrics)
# Generate and write the markdown table for this source
table_content = generate_markdown_table(metrics)
output_path = GENERATED_METRICS_DIR / output_file
with open(output_path, "w", encoding="utf-8") as f:
f.write(table_content)
total_metrics += len(metrics)
logger.info(
"Generated metrics table: %s (%d metrics)",
output_path.relative_to(ROOT_DIR),
len(metrics),
)
logger.info(
"Total metrics generated: %d across %d files",
total_metrics,
len(METRIC_SOURCE_FILES),
)
fill_markers("usage/metrics.md", blocks)
logger.info(
"Total metrics generated: %d across %d files",
total_metrics,
len(METRIC_SOURCE_FILES),
)
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inline build-time generated content into existing docs pages.
Source pages mark where generated content goes with a snippet-style marker,
`--8<-- "gen:<key>"`, so the insertion point is explicit and readable. The
substitution happens here (at gen-files time, before mkdocs-gen-files shadows
the page), not via pymdownx.snippets, so the content can be generated at build
time without living in a real file on disk.
The `gen:` prefix keeps these markers distinct from real pymdownx.snippets
includes, and `fill_markers` fails loudly if a marker is missing or left behind
(pymdownx.snippets would otherwise silently drop an unsubstituted marker).
"""
from pathlib import Path
import mkdocs_gen_files
import regex as re
DOCS_DIR = Path(__file__).parent.parent.parent
_MARKER = '--8<-- "gen:{key}"'
_ANY_MARKER = re.compile(r'--8<-- "gen:[^"]*"')
def fill_markers(doc_path: str, blocks: dict[str, str]) -> None:
"""Replace `--8<-- "gen:<key>"` markers in a docs page with generated content.
Args:
doc_path: Docs-relative path of the source page to fill.
blocks: Mapping of marker key to the markdown to insert in its place.
Raises:
FileNotFoundError: If the source page does not exist.
ValueError: If an expected marker is missing, or any `gen:` marker is
left unsubstituted after filling.
"""
source = DOCS_DIR / doc_path
if not source.exists():
raise FileNotFoundError(f"Cannot fill markers in missing page: {doc_path}")
text = source.read_text()
for key, content in blocks.items():
marker = _MARKER.format(key=key)
if marker not in text:
raise ValueError(f"{doc_path}: missing marker {marker}")
text = text.replace(marker, content)
if leftover := _ANY_MARKER.search(text):
raise ValueError(f"{doc_path}: unsubstituted marker {leftover.group()}")
with mkdocs_gen_files.open(doc_path, "w") as f:
f.write(text)
# Keep the edit button pointing at the real source page
mkdocs_gen_files.set_edit_path(doc_path, doc_path)
+39 -4
View File
@@ -19,6 +19,7 @@ The on_page_markdown hook passes the current page context to the preprocessor be
each page is converted.
"""
import posixpath
from pathlib import Path
import regex as re
@@ -38,18 +39,22 @@ TITLE = r"(?P<title>[^\[\]<>]+?)"
REPO = r"(?P<repo>.+?/.+?)"
TYPE = r"(?P<type>issues|pull|projects)"
NUMBER = r"(?P<number>\d+)"
VERSION = r"[^/\s]+"
PATH = r"(?P<path>[^\s]+?)"
FRAGMENT = r"(?P<fragment>#[^\s]+)?"
URL = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}"
URL_GITHUB = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}"
RELATIVE = rf"(?!(https?|ftp)://|#){PATH}{FRAGMENT}"
URL_DOCS = f"https://docs.vllm.ai/en/{VERSION}/{PATH}{FRAGMENT}"
# Common titles to use for GitHub links when none is provided in the link.
TITLES = {"issues": "Issue ", "pull": "Pull Request ", "projects": "Project "}
# Regex to match GitHub issue, PR, and project links with optional titles.
github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL}(\)|>)")
github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL_GITHUB}(\)|>)")
# Regex to match relative file links with optional titles.
relative_link = re.compile(rf"\[{TITLE}\]\({RELATIVE}\)")
# Regex to match absolute docs.vllm.ai links (should only exist in CLI).
docs_link = re.compile(rf"\[{TITLE}\]\({URL_DOCS}\)")
class UrlSchemesPreprocessor(Preprocessor):
@@ -61,7 +66,8 @@ class UrlSchemesPreprocessor(Preprocessor):
def run(self, lines):
page = self.ext.page
if page is None or getattr(page.file, "abs_src_path", None) is None:
files = self.ext.files
if page is None:
return lines
def replace_relative_link(match: re.Match) -> str:
@@ -70,7 +76,7 @@ class UrlSchemesPreprocessor(Preprocessor):
"""
title = match.group("title")
path = match.group("path")
path = (Path(page.file.abs_src_path).parent / path).resolve()
path = ((DOC_DIR / page.file.src_uri).parent / path).resolve()
fragment = match.group("fragment") or ""
# Check if the path exists and is outside the docs dir
@@ -105,9 +111,36 @@ class UrlSchemesPreprocessor(Preprocessor):
url = f"https://github.com/{repo}/{type}/{number}{fragment}"
return f"[{gh_icon} {title}]({url})"
def replace_docs_link(match: re.Match) -> str:
"""Rewrite absolute docs.vllm.ai links as doc-relative links."""
title = match.group("title")
path = match.group("path").rstrip("/")
fragment = match.group("fragment") or ""
# vllm.config.<Class> API reference -> mkdocstrings cross-reference
if path == "api/vllm/config" and re.fullmatch(
r"#vllm\.config\.\w+", fragment
):
ident = fragment[1:]
return f"[`{ident}`][{ident}]"
# Other docs pages -> link relative to the current page, but only
# when the target is a known docs page (real or generated); leave
# unknown/external URLs untouched. This is correct even when the same
# docstring is also rendered on its API reference page.
src = f"{path.removesuffix('.html')}.md"
if files.get_file_from_path(src) is None:
return match.group(0)
rel = posixpath.relpath(src, posixpath.dirname(page.file.src_uri))
# Auto-wrapped bare URLs use the URL as their title; make it readable.
if title.startswith("http"):
title = path.removesuffix(".html")
return f"[{title}]({rel}{fragment})"
markdown = "\n".join(lines)
markdown = github_link.sub(replace_github_link, markdown)
markdown = relative_link.sub(replace_relative_link, markdown)
markdown = docs_link.sub(replace_docs_link, markdown)
return markdown.split("\n")
@@ -116,6 +149,7 @@ class UrlSchemesExtension(Extension):
def __init__(self, **kwargs):
self.page = None
self.files = None
super().__init__(**kwargs)
def extendMarkdown(self, md):
@@ -138,4 +172,5 @@ def on_page_markdown(
) -> str:
"""Pass the current page context to the preprocessor."""
_ext.page = page
_ext.files = files
return markdown