Merge branch 'main' into patches

This commit is contained in:
2026-10-18 16:04:36 +07:00
299 changed files with 51483 additions and 2195 deletions
+16 -6
View File
@@ -2124,7 +2124,6 @@ class rocm_aiter_ops:
VLLM_ROCM_USE_AITER_MHA: Controls MHA ops including flash_attn_varlen.
VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: Controls Triton unified attention.
VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply.
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: Controls Triton sparse MLA (gfx950).
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
@@ -2196,8 +2195,6 @@ class rocm_aiter_ops:
_FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM
_FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM
_LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
# TODO: Consolidate under _LINEAR_ENABLED
_FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
_TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA
@@ -2228,7 +2225,6 @@ class rocm_aiter_ops:
cls._FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM
cls._FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM
cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
cls._TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
@@ -2565,10 +2561,10 @@ class rocm_aiter_ops:
@classmethod
@if_aiter_supported
def is_asm_fp4_gemm_dynamic_quant_enabled(cls) -> bool:
def is_asm_fp4_gemm_supported(cls) -> bool:
from vllm.platforms.rocm import on_gfx950
return cls._AITER_ENABLED and cls._FP4_GEMM_DYNAMIC_QUANT_ASM and on_gfx950()
return cls._AITER_ENABLED and on_gfx950()
@classmethod
@if_aiter_supported
@@ -4360,6 +4356,20 @@ class rocm_aiter_ops:
except (AssertionError, ImportError):
return False
@staticmethod
@functools.cache
def is_asm_fp4_gemm_tuned(n: int, k_bytes: int) -> bool:
if not current_platform.is_rocm():
return False
try:
from aiter.ops.gemm_op_a4w4 import get_GEMM_config
return all(
get_GEMM_config(m, n, 2 * k_bytes) is not None for m in (1024, 8192)
)
except ImportError:
return False
@staticmethod
@functools.cache
def is_shuffled_per_token_w8a8_gemm_tuned(
+110
View File
@@ -339,6 +339,17 @@ def fused_add_rms_norm(
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
def muse_spark_mul_tanh_soft_cap(
x: torch.Tensor,
multiplier: float,
soft_cap_temp: float,
) -> torch.Tensor:
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
out = torch.empty_like(x, dtype=torch.float32)
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
return out
def fused_qk_norm_rope(
qkv: torch.Tensor,
num_heads_q: int,
@@ -2598,6 +2609,105 @@ def reshape_and_cache_flash(
)
def fused_qknorm_rope_cache(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
slot_mapping: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
num_q_heads: int,
num_kv_heads: int,
head_dim: int,
rotary_dim: int,
eps: float,
norm_weight_bias: float = 0.0,
kv_cache_dtype: str = "auto",
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
Norm weights may be FP32 independently of QKV. A zero rotary dimension
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
"""
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
v = torch.empty_like(k)
cos, sin = cos_sin_cache.chunk(2, dim=-1)
torch.ops._C.fused_qknorm_rope_cache(
q,
k,
v,
qkv,
q_weight,
k_weight,
cos,
sin,
positions,
slot_mapping,
key_cache,
value_cache,
head_dim,
num_q_heads,
num_kv_heads,
rotary_dim // 2,
eps,
norm_weight_bias,
kv_cache_dtype,
k_scale,
v_scale,
)
return q, k, v
def fused_muse_spark_qknorm_rope_kv_insert(
qkv: torch.Tensor,
positions: torch.Tensor,
num_heads: int,
num_kv_heads: int,
head_dim: int,
eps: float,
cos_sin_cache: torch.Tensor | None = None,
rotary_dim: int = 0,
slot_mapping: torch.Tensor | None = None,
kv_cache: torch.Tensor | None = None,
block_size: int = 0,
kv_cache_dtype: str = "auto",
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
q_fp8_out: torch.Tensor | None = None,
q_scale: torch.Tensor | None = None,
) -> None:
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
"""
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
num_heads,
num_kv_heads,
head_dim,
eps,
cos_sin_cache,
rotary_dim,
slot_mapping,
kv_cache,
block_size,
kv_cache_dtype,
k_scale,
v_scale,
q_fp8_out,
q_scale,
)
def fused_minimax_m3_qknorm_rope_kv_insert(
qkv: torch.Tensor,
q_norm_weight: torch.Tensor,
+18
View File
@@ -55,6 +55,8 @@ class ForwardBench:
iters: int,
samples: int,
state_views: list[torch.Tensor],
name: str,
profile_prefix: str | None = None,
):
self.runner = runner
self.batch = batch
@@ -62,6 +64,8 @@ class ForwardBench:
self.iters = iters
self.samples = samples
self.state_views = state_views
self.name = name
self.profile_prefix = profile_prefix
self.phase = "warmup"
self.result: dict[str, Any] = {}
@@ -193,6 +197,20 @@ class ForwardBench:
),
)
self.result["latency_us"] = self._time(graph)
from vllm.benchmarks.forward.profile import profile_graph
# Each replay restores the state views, then (multi-rank) rendezvouses,
# before the timed forward.
skip = len(self.state_views) + (world.world_size > 1)
summary = profile_graph(graph, self.name, self.profile_prefix, skip)
if summary is not None:
self.result["profile"] = {
"summary": f"{self.profile_prefix}.kernels.json",
"span_us": summary["span_us"],
"busy_us": summary["busy_us"],
"activities": summary["activities"],
"top_kernels": summary["kernels"][:10],
}
del graph
return output
+24
View File
@@ -74,6 +74,13 @@ def add_cli_args(parser: FlexibleArgumentParser) -> None:
parser.add_argument("--iters", type=int, default=20, help="Replays per sample.")
parser.add_argument("--samples", type=int, default=5)
parser.add_argument("--output-json", help="Write results to this file.")
parser.add_argument(
"--profile-dir",
help="After timing, trace a few replays of each case and write "
"`<case>.rank<r>.trace.json.gz` (Perfetto) and `<case>.rank<r>.kernels.json` "
"(GPU time per kernel per forward) here. Summarize or compare with "
"`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.",
)
EngineArgs.add_cli_args(parser)
parser.set_defaults(load_format="dummy", enable_prefix_caching=False)
@@ -146,6 +153,13 @@ def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any
dcp_size,
)
zero_kv_blocks(context, kv_config.kv_cache_groups, batch, kv_config.num_blocks)
profile_prefix = None
if args.profile_dir:
from vllm.distributed.parallel_state import get_world_group
os.makedirs(args.profile_dir, exist_ok=True)
rank = get_world_group().rank
profile_prefix = os.path.join(args.profile_dir, f"{case.name}.rank{rank}")
bench = ForwardBench(
runner,
batch,
@@ -154,6 +168,8 @@ def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any
touched_kv_views(
context, kv_config.kv_cache_groups, batch, kv_config.num_blocks, dcp_size
),
case.name,
profile_prefix,
)
routing, balanced = install_routing(
runner, args.routing, args.load_format == "dummy", batch.num_tokens, args.seed
@@ -221,6 +237,14 @@ def main(args: argparse.Namespace) -> None:
f"eager rel rms {rank0['graph_vs_eager']['relative_rms']:.1e}",
flush=True,
)
if profile := rank0.get("profile"):
from vllm.benchmarks.forward.profile import format_summary
print(
f"[forward] {case.name} rank 0 kernels ({profile['summary']}):\n"
+ format_summary({**profile, "kernels": profile["top_kernels"]}),
flush=True,
)
if latency["sample_spread_pct"] > 0.5:
logger.warning(
"%s: unstable timing (sample medians spread %.2f%%).",
+236
View File
@@ -0,0 +1,236 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GPU kernel profiles of `vllm bench forward` cases.
With `--profile-dir`, each case's graph is replayed a few more times under
`torch.profiler` after timing. Each rank writes a Perfetto trace and a kernel
summary (`<case>.rank<r>.kernels.json`): per-kernel time per forward, sorted.
Summarize one file or compare two (e.g. before/after a change):
`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.
The same replays run between `cudaProfilerStart`/`Stop` in an NVTX range per
case (with or without `--profile-dir`), so Nsight Systems can record just them:
`nsys profile --capture-range=cudaProfilerApi --capture-range-end=repeat
--cuda-graph-trace=node vllm bench forward ...`.
"""
import argparse
import gzip
import json
import statistics
from collections import defaultdict
from typing import Any
import regex as re
import torch
_GPU_CATEGORIES = ("kernel", "gpu_memcpy", "gpu_memset")
# Kernels named after a generic launcher are identified by their first
# template argument instead.
_GENERIC = re.compile(r"(^|::)(device_kernel|Kernel|kernel|KernelWrapper)$")
# CuTe DSL kernel names embed per-shape layouts after these markers.
_CUTE_DSL_SUFFIX = re.compile(r"_object_at_|_tensorptr")
def _match_back(text: str, end: int, open_ch: str, close_ch: str) -> int:
"""Index of the bracket opening the one closing at `end`, skipping
brackets of the other kind nested inside parentheses."""
depth, parens = 0, 0
for i in range(end, -1, -1):
ch = text[i]
if open_ch == "<" and ch in "()":
parens += 1 if ch == ")" else -1
continue
if parens:
continue
if ch == close_ch:
depth += 1
elif ch == open_ch:
depth -= 1
if depth == 0:
return i
return -1
def short_name(name: str) -> str:
"""`void ns::foo<T, 4>(args)` -> `ns::foo`: return type, template and
function arguments dropped. Generic launchers (e.g.
`cutlass::device_kernel<Op<...>>`) keep their first template argument."""
if name.startswith("kernel_cutlass_"):
return _CUTE_DSL_SUFFIX.split(name, maxsplit=1)[0]
head = name.strip()
if head.endswith(")"):
head = head[: max(_match_back(head, len(head) - 1, "(", ")"), 0)].rstrip()
template = ""
if head.endswith(">"):
start = _match_back(head, len(head) - 1, "<", ">")
if start > 0:
head, template = head[:start], head[start + 1 : -1]
base = head.split()[-1] if head.split() else name
if _GENERIC.search(base) and template:
arg = re.split(r"[<>(,]", template, maxsplit=1)[0].strip()
if arg:
return f"{base}<{arg.removeprefix('void ')}>"
return base
def summarize(events: list[dict], replays: int, skip: int) -> dict[str, Any]:
"""Per-forward kernel times from the GPU activities of `replays` replays.
Each replay first runs `skip` activities that aren't part of the timed
forward (the KV state restore and the rank rendezvous); they are reported
separately as `pre_forward_us`.
"""
acts = sorted(
(e for e in events if e.get("cat") in _GPU_CATEGORIES and "dur" in e),
key=lambda e: e["ts"],
)
if not acts:
raise ValueError(
"No GPU activity recorded. Another profiler (e.g. nsys) may hold "
"CUPTI; don't combine --profile-dir with it."
)
if len(acts) % replays:
raise ValueError(
f"{len(acts)} GPU activities don't split into {replays} replays"
)
per_replay = len(acts) // replays
if per_replay <= skip:
raise ValueError(f"{per_replay} activities per replay, {skip} to skip")
kernels: dict[str, dict[str, Any]] = defaultdict(
lambda: {"total_us": 0.0, "calls": 0, "example": ""}
)
spans, busy, pre = [], [], []
for r in range(replays):
chunk = acts[r * per_replay : (r + 1) * per_replay]
pre.append(sum(e["dur"] for e in chunk[:skip]))
forward = chunk[skip:]
spans.append(
max(e["ts"] + e["dur"] for e in forward) - min(e["ts"] for e in forward)
)
busy.append(sum(e["dur"] for e in forward))
for e in forward:
name = short_name(e["name"]) if e["cat"] == "kernel" else e["name"]
k = kernels[name]
k["total_us"] += e["dur"]
k["calls"] += 1
k["example"] = k["example"] or e["name"][:300]
total = sum(k["total_us"] for k in kernels.values())
rows = [
{
"name": name,
"us": round(k["total_us"] / replays, 2),
"pct": round(100 * k["total_us"] / total, 2),
"calls": k["calls"] // replays,
"example": k["example"],
}
for name, k in kernels.items()
]
rows.sort(key=lambda row: -row["us"])
return {
"replays": replays,
# First to last forward activity; busy can exceed it when streams overlap.
"span_us": round(statistics.median(spans), 2),
"busy_us": round(statistics.median(busy), 2),
"pre_forward_us": round(statistics.median(pre), 2),
"activities": per_replay - skip,
"kernels": rows,
}
def profile_graph(
graph: torch.cuda.CUDAGraph,
name: str,
prefix: str | None,
skip: int,
replays: int = 3,
) -> dict[str, Any] | None:
"""Replay `replays` times between `cudaProfilerStart`/`Stop`, in an NVTX
range named after the case, for `nsys --capture-range=cudaProfilerApi`.
With `prefix`, also trace them with `torch.profiler` and write
`prefix.trace.json.gz` and `prefix.kernels.json`. Every rank must call
this (replays may contain collectives).
"""
def run() -> None:
torch.cuda.profiler.start()
with torch.cuda.nvtx.range(f"forward/{name}"):
for _ in range(replays):
graph.replay()
torch.accelerator.synchronize()
torch.cuda.profiler.stop()
if prefix is None:
run()
return None
activities = [torch.profiler.ProfilerActivity.CUDA]
with torch.profiler.profile(activities=activities) as prof:
run()
trace = f"{prefix}.trace.json.gz"
prof.export_chrome_trace(trace)
with gzip.open(trace, "rt") as f:
events = json.load(f)["traceEvents"]
summary = {"trace": trace, **summarize(events, replays, skip)}
with open(f"{prefix}.kernels.json", "w") as f:
json.dump(summary, f, indent=1)
return summary
def format_summary(summary: dict[str, Any], top: int = 20) -> str:
lines = [
f"span {summary['span_us']:.1f} us, busy {summary['busy_us']:.1f} us, "
f"{summary['activities']} activities/forward",
f"{'us':>10} {'%':>6} {'calls':>6} kernel",
]
for row in summary["kernels"][:top]:
lines.append(
f"{row['us']:10.1f} {row['pct']:6.2f} {row['calls']:6d} {row['name']}"
)
return "\n".join(lines)
def format_diff(a: dict[str, Any], b: dict[str, Any], top: int = 20) -> str:
rows_a = {row["name"]: row for row in a["kernels"]}
rows_b = {row["name"]: row for row in b["kernels"]}
deltas = []
for name in rows_a.keys() | rows_b.keys():
ra, rb = rows_a.get(name), rows_b.get(name)
us_a, us_b = ra["us"] if ra else 0.0, rb["us"] if rb else 0.0
calls_a, calls_b = ra["calls"] if ra else 0, rb["calls"] if rb else 0
deltas.append((us_b - us_a, us_a, us_b, calls_a, calls_b, name))
deltas.sort(key=lambda d: -abs(d[0]))
lines = [
f"span {a['span_us']:.1f} -> {b['span_us']:.1f} us "
f"({b['span_us'] - a['span_us']:+.1f}), busy {a['busy_us']:.1f} -> "
f"{b['busy_us']:.1f} us ({b['busy_us'] - a['busy_us']:+.1f})",
f"{'delta us':>10} {'A us':>10} {'B us':>10} {'A n':>5} {'B n':>5} kernel",
]
for delta, us_a, us_b, calls_a, calls_b, name in deltas[:top]:
lines.append(
f"{delta:+10.1f} {us_a:10.1f} {us_b:10.1f} {calls_a:5d} {calls_b:5d} "
f"{name}"
)
return "\n".join(lines)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("summaries", nargs="+", help="One or two *.kernels.json.")
parser.add_argument("--top", type=int, default=20)
args = parser.parse_args()
if len(args.summaries) > 2:
parser.error("Pass one summary, or two to compare.")
loaded = []
for path in args.summaries:
with open(path) as f:
loaded.append(json.load(f))
if len(loaded) == 1:
print(format_summary(loaded[0], args.top))
else:
print(format_diff(loaded[0], loaded[1], args.top))
if __name__ == "__main__":
main()
+16 -2
View File
@@ -122,8 +122,22 @@ async def _align_prompts_to_server_tokenizer(
try:
first_tokens = await _tokenize(input_requests[0].prompt)
except Exception:
print("WARNING: /tokenize unavailable, skipping alignment.")
except aiohttp.ClientConnectionError as e:
print(f"WARNING: {base_url} unreachable ({e!r}), skipping alignment.")
return input_requests
except aiohttp.ClientResponseError as e:
if e.status == 404:
hint = (
"404 Not Found: either this server has no /tokenize route,"
f" or it does not serve a model named `{model_id}`"
" (its served names are listed by GET /v1/models)"
)
else:
hint = f"HTTP {e.status}"
print(f"WARNING: /tokenize unavailable ({hint}), skipping alignment.")
return input_requests
except Exception as e:
print(f"WARNING: /tokenize probe failed ({e!r}), skipping alignment.")
return input_requests
expected = input_requests[0].prompt_len
+23
View File
@@ -1063,6 +1063,28 @@ class VllmConfig:
)
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
def _maybe_disable_dynamic_sd_for_v1(self) -> None:
speculative_config = self.speculative_config
# V2 truncates drafts centrally. V1 requires proposer support.
if (
speculative_config is None
or not speculative_config.uses_dynamic_speculative_decoding()
or self.use_v2_model_runner
or speculative_config.method
in ("ngram", "eagle", "eagle3", "mtp", "draft_model", "dflash")
):
return
logger.warning_once(
"Dynamic speculative decoding is not supported with the '%s' "
"speculative method on Model Runner V1. "
"Disabling num_speculative_tokens_per_batch_size "
"and falling back to static num_speculative_tokens=%d.",
speculative_config.method,
speculative_config.num_speculative_tokens,
)
speculative_config.num_speculative_tokens_per_batch_size = None
def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
speculative_config = self.speculative_config
if (
@@ -1836,6 +1858,7 @@ class VllmConfig:
)
self._maybe_disable_dynamic_sd_for_data_parallel()
self._maybe_disable_dynamic_sd_for_v1()
self._maybe_override_dynamic_sd_cudagraph_mode()
if (
@@ -156,10 +156,12 @@ def _resolve_fi_ar_backend(group: ProcessGroup) -> tuple[str | None, bool]:
node_count = _node_count(group)
if node_count == 1:
# TRTLLM supports single-node allreduce on Hopper and Blackwell. Do not
# create an MNNVL workspace when the process group is node-local.
backend = "trtllm"
allow_trtllm_fallback = False
# MNNVL decodes faster than TRTLLM on single-node NVSwitch systems
# (about 3-5% throughput at TP4/TP8 on H200 and B200). Fall back to
# TRTLLM where the node lacks NVSwitch multicast and MNNVL cannot
# create its workspace.
backend = "mnnvl"
allow_trtllm_fallback = True
elif current_platform.has_device_capability(100):
# MNNVL requires Blackwell-class GPUs and is the only supported
# FlashInfer backend for a multi-node allreduce process group.
+9 -1
View File
@@ -354,6 +354,7 @@ class NixlEplbCommunicator(EplbCommunicator):
# Per-rank expert_id -> physical row; set by set_transfer_context.
self._expert_to_src_row: list[dict[int, int]] | None = None
self._layer_idx: int | None = None
self.set_stream(None)
nixl_agent_config = nixl_utils.nixl_agent_config
config = (
@@ -403,7 +404,7 @@ class NixlEplbCommunicator(EplbCommunicator):
return f"eplb-{self._rank}{pp_suffix}-{uid}"
def set_stream(self, stream: torch.Stream | None) -> None:
pass
self._stream = stream
def add_send(
self,
@@ -421,6 +422,13 @@ class NixlEplbCommunicator(EplbCommunicator):
f"pending transfers from layer {self._layer_idx}; "
f"execute() was not called after previous add_recv() calls"
)
# NIXL READs bypass device streams; wait for pending copies out of
# the receive buffer before add_recv overwrites it.
if current_platform.is_rocm():
with gpu_sync_allowed():
torch.accelerator.current_stream().synchronize()
if self._stream is not None:
self._stream.synchronize()
self._layer_idx = layer_idx
n = self._num_local_experts
rank_experts = old_indices[: self._world_size * n].reshape(self._world_size, n)
@@ -748,11 +748,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
"""Drain HB / completion notifs forwarded by the writer thread.
The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG
notifs are handled there. Everything else is forwarded here for
existing accounting.
notifs are handled there. P learns of finished pushes from its WRITE
handles, so this never reports finished sends.
"""
assert self.transfer_topo is not None
notified_req_ids: set[str] = set()
while True:
try:
notif = self._pending_completion_notifs.get_nowait()
@@ -766,46 +764,27 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
req_id, tp_size = msg.rsplit(":", 1)
# Not tracked as a P-side send/process for this notif.
if req_id not in self._reqs_to_send and req_id not in self._reqs_to_process:
if (meta := self._recving_metadata.get(req_id)) is not None:
# Consumer waits for one notif per producer rank writing
# here: pp_size stages * producers-per-consumer (>1 when
# producer TP > consumer TP; tp_size is the producer TP).
producers_per_consumer = max(1, int(tp_size) // self.world_size)
expected_notifs = meta.pp_size * producers_per_consumer
self.consumer_notification_counts_by_req[req_id] += 1
notifs = self.consumer_notification_counts_by_req[req_id]
if notifs < expected_notifs:
continue
del self.consumer_notification_counts_by_req[req_id]
# P drove the transfer (we own no NIXL handle), so
# materialise an empty ``_recving_transfers`` entry for
# ``_pop_done_transfers`` to report done.
self._recving_transfers.setdefault(req_id, [])
else:
# Not tracked on either side (lease may have expired
# before the notif arrived). Log and skip.
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Unrecognized request %s notif (may have expired).",
req_id,
)
if (meta := self._recving_metadata.get(req_id)) is None:
# The lease may have expired before the notif arrived.
self.xfer_stats.record_notification_after_expiry()
logger.error(
"Unrecognized request %s notif (may have expired).", req_id
)
continue
n_consumers = int(tp_size)
tp_ratio = self.transfer_topo.tp_ratio(n_consumers)
consumers_per_producer = -tp_ratio if n_consumers > self.world_size else 1
# Consumer waits for one notif per producer rank writing
# here: pp_size stages * producers-per-consumer (>1 when
# producer TP > consumer TP; tp_size is the producer TP).
producers_per_consumer = max(1, int(tp_size) // self.world_size)
expected_notifs = meta.pp_size * producers_per_consumer
self.consumer_notification_counts_by_req[req_id] += 1
if (
self.consumer_notification_counts_by_req[req_id]
== consumers_per_producer
):
notified_req_ids.add(req_id)
del self.consumer_notification_counts_by_req[req_id]
self._reqs_to_process.remove(req_id)
self._reqs_to_send.pop(req_id, None)
return notified_req_ids
if self.consumer_notification_counts_by_req[req_id] < expected_notifs:
continue
del self.consumer_notification_counts_by_req[req_id]
# P drove the transfer (we own no NIXL handle), so
# materialise an empty ``_recving_transfers`` entry for
# ``_pop_done_transfers`` to report done.
self._recving_transfers.setdefault(req_id, [])
return set()
def get_transfer_results(self) -> KVConnectorTransferResults:
# Engine main thread asking for completions: also wake the writer
@@ -835,7 +814,6 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
for req_id in done_pushing:
self._reqs_to_send.pop(req_id, None)
self._reqs_to_process.discard(req_id)
self.consumer_notification_counts_by_req.pop(req_id, None)
done_sending.add(req_id)
# Tell the writer to drop any state it still holds for any
@@ -236,10 +236,9 @@ class SchedulerOffloadConfig(NamedTuple):
)
if use_eagle_block_drop and not eagle_groups:
# No group is annotated as holding drafter layers. This happens
# for shared-group MTP models (e.g. Qwen3.5-style), whose drafter
# is a regular decoder layer merged into the target's
# full-attention group, and for separate-draft models that miss
# annotation. Treating every group as a drafter group here makes
# when the drafter owns no KV cache (e.g. KV-sharing drafters) or
# the model runner does not tag draft layers' KV cache specs.
# Treating every group as a drafter group here makes
# the volatile-tail pop apply to ALL groups, which can zero the
# whole request's offload hit whenever any group's servable
# window is a single chunk (issue #52735). Drafter KV served one
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
import math
from collections.abc import Callable, Sequence
import torch
@@ -104,14 +105,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
structured_output_backend: StructuredOutputBackend | None = None
structured_output_key = None
structured_output_bitmask = None
if params.structured_outputs is not None:
(
structured_output_backend,
structured_output_key,
structured_output_bitmask,
) = self._init_beam_search_structured_output(
params.structured_outputs, tokenizer
)
# generate 2 * beam_width candidates at each step
# following the huggingface transformers implementation
@@ -124,6 +117,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
detokenize=False,
skip_clone=True, # Internal beam search, safe to skip clone
)
params.configure_logprobs(
base_sampling_params, self.llm_engine.input_processor.vllm_config
)
if params.structured_outputs is not None:
(
structured_output_backend,
structured_output_key,
structured_output_bitmask,
) = self._init_beam_search_structured_output(
params.structured_outputs, tokenizer
)
instances: list[BeamSearchInstance] = []
for lora_req, prompt in zip(lora_requests, engine_inputs):
@@ -287,7 +292,12 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
# tokens outside the grammar's allowed set. This filtering is also
# the only grammar enforcement for beams whose allowed set exceeds
# the engine-side allowed_token_ids cap.
allowed_sets: list[set[int] | None] = [None] * len(all_beams)
request_allowed = (
set(base_sampling_params.logprob_token_ids)
if base_sampling_params.logprob_token_ids is not None
else None
)
allowed_sets: list[set[int] | None] = [request_allowed] * len(all_beams)
if structured_output_backend is not None:
for i, entry in enumerate(beam_entries):
if entry is not None:
@@ -316,6 +326,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
if result.outputs[0].logprobs is not None:
logprobs = result.outputs[0].logprobs[0]
if request_allowed is not None:
logprobs = {
token: score
for token, score in sorted(
logprobs.items(),
key=lambda item: item[1].logprob,
reverse=True,
)
if token in request_allowed and math.isfinite(score.logprob)
}
for score in logprobs.values():
score.rank = None
allowed = allowed_sets[i]
for token_id, logprob_obj in logprobs.items():
if allowed is not None and token_id not in allowed:
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import math
from abc import ABC
from collections.abc import AsyncGenerator, Mapping
@@ -16,7 +17,10 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.utils import random_uuid
from vllm.utils.async_utils import collect_from_async_generator
from .utils import BeamSearchSequence, create_sort_beams_key_function
from .utils import (
BeamSearchSequence,
create_sort_beams_key_function,
)
class BeamSearchOnlineMixin(ABC):
@@ -66,6 +70,10 @@ class BeamSearchOnlineMixin(ABC):
watermarking=False,
detokenize=False,
)
params.configure_logprobs(
sampling_params, self.engine_client.input_processor.vllm_config
)
allowed = set(params.allowed_token_ids) if params.allowed_token_ids else None
all_beams = [
BeamSearchSequence(
orig_prompt=prompt,
@@ -135,6 +143,18 @@ class BeamSearchOnlineMixin(ABC):
if result.outputs[0].logprobs is not None:
logprobs = result.outputs[0].logprobs[0]
if allowed is not None:
logprobs = {
token: score
for token, score in sorted(
logprobs.items(),
key=lambda item: item[1].logprob,
reverse=True,
)
if token in allowed and math.isfinite(score.logprob)
}
for score in logprobs.values():
score.rank = None
for token_id, logprob_obj in logprobs.items():
candidate_logprob = (
current_beam.cum_logprob + logprob_obj.logprob
@@ -710,6 +710,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
return BeamSearchParams(
beam_width=n,
allowed_token_ids=self.allowed_token_ids,
max_tokens=max_tokens,
ignore_eos=self.ignore_eos,
temperature=temperature,
@@ -334,6 +334,7 @@ class CompletionRequest(OpenAIBaseModel):
return BeamSearchParams(
beam_width=n,
allowed_token_ids=self.allowed_token_ids,
max_tokens=max_tokens,
ignore_eos=self.ignore_eos,
temperature=temperature,
-6
View File
@@ -146,7 +146,6 @@ if TYPE_CHECKING:
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["auto", "asm", "segmented"] = "auto"
VLLM_ROCM_USE_AITER_MHA: bool = True
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
VLLM_ROCM_USE_AITER_FP8BMM: bool = False
@@ -1346,11 +1345,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MHA": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1")
),
# Whether to use aiter fp4 gemm asm.
# By default is disabled.
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1")
),
# Whether sparse MLA prefill and decode run on aiter's Triton kernel, which
# reads the KV cache as stored (bf16 or fp8, paged or flat). Used by the
# ROCM_AITER_MLA_SPARSE backend (DeepSeek V3.2, GLM-5.x) and DeepSeek
@@ -1424,6 +1424,9 @@ def override_envs_for_invariance():
# torch.compile settings
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
import torch._inductor.config as inductor_config
inductor_config.batch_invariant = True
def init_batch_invariance():
@@ -159,7 +159,8 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
def __init__(self, config: MxFp4LinearLayerConfig) -> None:
super().__init__(config)
self.use_asm_gemm = rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled()
self.asm_gemm_supported = rocm_aiter_ops.is_asm_fp4_gemm_supported()
self.use_asm_gemm = False
self.out_dtype = torch.get_default_dtype()
@classmethod
@@ -199,6 +200,13 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Use preshuffled ASM (P) and Triton (D) GEMM when tuned in aiter
if self.asm_gemm_supported:
n, k_bytes = layer.weight.shape
self.use_asm_gemm = rocm_aiter_ops.is_asm_fp4_gemm_tuned(
n, k_bytes
) and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(n, k_bytes)
if self.use_asm_gemm and not _asm_fp4_scale_swizzle_supported(
layer.weight_scale.data
):
@@ -1,8 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16).
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16, MXFP4 x BF16).
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153:
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153
(BF16) and PR #3987 (FP4 block-scale):
* The W13 (gate_up) LoRA delta is passed directly as ``gemm1_lora_delta`` to
the routed kernel, which fuses it into FC1 before SwiGLU (BiasType::Mn).
@@ -13,11 +14,12 @@ Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153:
Constraints (matching the PR support matrix; final gating lives in the oracle):
* SM100+ (Blackwell), gated SwiGLU, shuffled weights only;
* BF16 only;
* BF16 weights, or MXFP4 weights with BF16 activations;
* routing must be computed outside the MoE (the Modular path satisfies this).
"""
import functools
from abc import abstractmethod
import torch
@@ -37,9 +39,17 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
view_as_block_major_k,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsBase,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
@@ -224,6 +234,15 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
"""
raise NotImplementedError
def prepare_gemm1_lora_delta(
self,
gemm1_lora_delta: torch.Tensor,
lora_context: MoELoRAContext,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Hook to rewrite the W13 delta before it is handed to the kernel."""
return gemm1_lora_delta
def apply(
self,
output: torch.Tensor,
@@ -247,7 +266,8 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
num_tokens = hidden_states.size(0)
top_k = self.topk
intermediate_size = self.intermediate_size_per_partition
K = output.size(1)
# LoRA buffers use the padded hidden size; `output` may be narrower.
K = self.hidden_dim
# ---- Base-model fast path ----
# When no token in the batch selects a LoRA adapter, skip the LoRA machinery
@@ -288,8 +308,7 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
w13_meta = (None, None, None, None)
# zeros (not empty): under EP the punica expand kernel only writes
# slots whose expert is local to this rank; non-local (token, top_k)
# slots must stay 0 so they contribute no bias when fed to flashinfer.
# slots whose expert is local to this rank; the kernel skips the rest.
gemm1_lora_delta = torch.zeros(
num_tokens,
top_k,
@@ -326,6 +345,9 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
add_inputs=False,
swap_w13_slices=True,
)
gemm1_lora_delta = self.prepare_gemm1_lora_delta(
gemm1_lora_delta, lora_context, topk_ids
)
# ---- 2) Call the routed flashinfer kernel ----
ret = self.invoke_routed_moe(
@@ -466,7 +488,9 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
(``expert_weights`` in expanded order, ``idx_map < 0`` dropped), scale by
``scale``, and add the already-weighted ``w2_delta`` reduced over top_k.
"""
K = gemm2_permuted.size(1)
# Inputs may carry hidden padding that `output` does not.
K = output.size(1)
assert gemm2_permuted.size(1) >= K and w2_delta.size(2) >= K
BLOCK_K = 512
grid = (num_tokens, triton.cdiv(K, BLOCK_K))
_finalize_lora_kernel[grid](
@@ -550,3 +574,158 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
return list(ret)
# do_finalize=True finalized directly into `output`.
return [output]
# MXFP4 weights x BF16 activations trtllm MoE + LoRA
class TrtLlmMxfp4LoRAExperts(_TrtLlmLoRAExpertsBase, TrtLlmMxfp4ExpertsBase):
"""MXFP4-weight, BF16-activation trtllm MoE + LoRA. Not MXFP8 activations:
the kernel then returns the FC1 activation quantized, without its scales."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
_TrtLlmLoRAExpertsBase.__init__(self, moe_config, quant_config)
# gemm1_alpha / gemm1_beta / gemm1_clamp_limit, hidden_dim_unpadded.
TrtLlmMxfp4ExpertsBase.__init__(self, moe_config, quant_config)
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key == kMxfp4Static and activation_key is None
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
@staticmethod
def _supports_routing_method(routing_method, weight_key, activation_key) -> bool:
return True # routing is done outside the kernel
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
# DP gathers tokens, but the no-LoRA fast path reads the local batch.
p = moe_parallel_config
return p.dp_size == 1 and not p.use_all2all_kernels and not p.enable_eplb
def workspace_shapes(
self,
M,
N,
K,
topk,
global_num_experts,
local_num_experts,
expert_tokens_meta,
activation,
):
# flashinfer manages its own workspace and drops the hidden padding.
return (0,), (0,), (M, self.hidden_dim_unpadded)
def output_hidden_dim(self, hidden_dim: int) -> int:
# FusedMoEExpertsModular precedes TrtLlmMxfp4ExpertsBase in the MRO.
return self.hidden_dim_unpadded
@functools.cached_property
def _w13_bias_up_gate_index(self) -> torch.Tensor:
"""Columns of the kernel-format FC1 bias in [up | gate] order."""
from flashinfer.fused_moe.core import get_w2_permute_indices_with_cache
# Kernel format is the (up_i, gate_i) bias permuted for the tile.
assert self.w1_bias is not None
bias = self.w1_bias.reshape(self.local_num_experts, -1)[0]
permute = get_w2_permute_indices_with_cache({}, bias.reshape(-1, 1), 128)
interleaved = torch.argsort(permute.to(bias.device))
return torch.cat([interleaved[0::2], interleaved[1::2]])
def prepare_gemm1_lora_delta(
self,
gemm1_lora_delta: torch.Tensor,
lora_context: MoELoRAContext,
topk_ids: torch.Tensor,
) -> torch.Tensor:
"""Bring the W13 delta to [up | gate] halves and fold in the FC1 bias."""
intermediate_size = gemm1_lora_delta.size(-1) // 2
# View as (I, 2) pairs of the first and second w13 half. SWIGLUOAI
# weights, and the deltas that follow them, are interleaved.
if self.moe_config.activation == MoEActivation.SWIGLUOAI:
pairs = gemm1_lora_delta.unflatten(-1, (intermediate_size, 2))
else:
pairs = gemm1_lora_delta.unflatten(-1, (2, intermediate_size)).mT
# Per-expert deltas arrive as (up, gate); fused 3D ones follow the
# checkpoint (gate, up) order.
if lora_context.w13_num_slices != 2:
pairs = pairs.flip(-1)
delta = pairs.mT.flatten(-2) # [up | gate]
# The kernel has a single FC1 bias slot, shared by the static bias and
# the per-token delta, so the static bias rides along in the delta.
if self.w1_bias is not None:
bias = self.w1_bias.reshape(self.local_num_experts, -1)
bias = bias[:, self._w13_bias_up_gate_index].float()
local_ids = topk_ids.long() - self.ep_rank * self.local_num_experts
local_ids = local_ids.clamp_(0, self.local_num_experts - 1)
delta = (delta.float() + bias[local_ids]).to(torch.bfloat16)
return delta.contiguous()
def invoke_routed_moe(
self,
*,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
gemm1_lora_delta: torch.Tensor | None,
global_num_experts: int,
a1q_scale: torch.Tensor | None,
output: torch.Tensor,
) -> list[torch.Tensor]:
from flashinfer import trtllm_fp4_block_scale_routed_moe
assert a1q_scale is None and hidden_states.dtype == torch.bfloat16
topk_ids, topk_weights = topk_ids_and_weights
# See TrtLlmBf16LoRAExperts.invoke_routed_moe for the return contract.
do_finalize = gemm1_lora_delta is None
ret = trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids.to(torch.int32), topk_weights),
routing_bias=None,
hidden_states=hidden_states,
hidden_states_scale=None,
gemm1_weights=w1,
gemm1_weights_scale=self.w1_scale,
# With a delta the bias is folded into it (prepare_gemm1_lora_delta).
gemm1_bias=self.w1_bias if do_finalize else None,
gemm1_alpha=self.gemm1_alpha,
gemm1_beta=self.gemm1_beta,
gemm1_clamp_limit=self.gemm1_clamp_limit,
gemm2_weights=w2,
gemm2_weights_scale=self.w2_scale,
gemm2_bias=self.w2_bias,
output1_scale_scalar=None,
output1_scale_gate_scalar=None,
output2_scale_scalar=None,
num_experts=global_num_experts,
top_k=self.topk,
n_group=None,
topk_group=None,
intermediate_size=self.intermediate_size_per_partition,
local_expert_offset=self.ep_rank * self.local_num_experts,
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
# Routing is already done; Renormalize is a safe default.
routing_method_type=RoutingMethodType.Renormalize,
do_finalize=do_finalize,
enable_pdl=True,
activation_type=self._flashinfer_activation_type(
self.moe_config.activation
),
output=output if do_finalize else None,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
gemm1_lora_delta=gemm1_lora_delta,
)
return [output] if do_finalize else list(ret)
@@ -122,6 +122,9 @@ class TrtLlmMxfp4ExpertsBase:
p = current_platform
return p.is_cuda() and p.is_device_capability_family(100) and has_flashinfer()
def output_hidden_dim(self, hidden_dim: int) -> int:
return self.hidden_dim_unpadded
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@@ -843,6 +843,11 @@ class FusedMoEExpertsModular(FusedMoEExperts):
"""Workspace type: The dtype to use for the workspace tensors."""
return act_dtype
def output_hidden_dim(self, hidden_dim: int) -> int:
"""Hidden size of the finalized output. Experts that drop the hidden
padding return the unpadded size."""
return hidden_dim
@abstractmethod
def workspace_shapes(
self,
@@ -1321,7 +1326,8 @@ class FusedMoEKernelModularImpl:
# low-latency kernels are always batched and can never run into
# the tensor.numel() == 0 case.
if M_full == 0:
return torch.empty_like(a1q, dtype=in_dtype)
out_dim = self.fused_experts.output_hidden_dim(a1q.size(-1))
return a1q.new_empty((*a1q.shape[:-1], out_dim), dtype=in_dtype)
workspace13, workspace2, fused_out, fused_out_is_specialized = (
self._allocate_buffers(
@@ -1505,7 +1511,12 @@ class FusedMoEKernelModularImpl:
a fused consumer.
"""
output = torch.empty_like(hidden_states)
output = hidden_states.new_empty(
(
*hidden_states.shape[:-1],
self.fused_experts.output_hidden_dim(hidden_states.size(-1)),
)
)
local_num_experts = w1.shape[0]
if global_num_experts == -1:
@@ -535,6 +535,30 @@ def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
return f"{base} since {reason}." if reason else f"{base}."
def _requested_kernel_cls(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
) -> list[type[mk.FusedMoEExperts]]:
"""Kernel classes for an explicitly requested backend. The LoRA-aware
TRT-LLM experts are opt-in: "auto" keeps its current LoRA selection."""
candidates = backend_to_kernel_cls(backend)
if not config.is_lora_enabled or backend not in TRTLLM_BACKENDS:
return candidates
if backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8:
raise ValueError(
_make_log_unsupported(
backend,
"expert LoRA is not supported with MXFP8 activations; use "
"moe_backend='flashinfer_trtllm'",
)
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
TrtLlmMxfp4LoRAExperts,
)
return [TrtLlmMxfp4LoRAExperts, *candidates]
def _return_or_raise(
backend: Mxfp4MoeBackend,
config: FusedMoEConfig,
@@ -544,7 +568,7 @@ def _return_or_raise(
scope: Literal["process", "global", "local"] = "local",
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
reason: str | None = None
for k_cls in backend_to_kernel_cls(backend):
for k_cls in _requested_kernel_cls(backend, config):
supported, reason = k_cls.is_supported_config(
k_cls, config, weight_key, activation_key, activation_format
)
+45 -10
View File
@@ -1663,6 +1663,13 @@ class Gemma4ForConditionalGeneration(
batch_size, num_patches = src.shape[0], src.shape[1]
dst[:batch_size, :num_patches].copy_(src)
configured_max_tokens = self.config.vision_config.default_output_length
# One graph per row width (soft tokens of one image or video frame),
# so a budget's rows are not all padded to the widest image.
row_widths = tuple(
t for t in _SUPPORTED_SOFT_TOKENS if t <= configured_max_tokens
)
return EncoderCudaGraphConfig(
modalities=["image", "video"],
buffer_keys=[
@@ -1676,6 +1683,7 @@ class Gemma4ForConditionalGeneration(
"pixel_values": pad_pixel_values,
"pixel_position_ids": pad_pixel_position_ids,
},
capture_axes=(row_widths,),
)
def get_encoder_cudagraph_budget_range(
@@ -1712,10 +1720,15 @@ class Gemma4ForConditionalGeneration(
if modality == "image":
pixel_values = mm_kwargs["pixel_values"]
if isinstance(pixel_values, list):
# A batch pads its images to the widest one, so pack by that.
row_tokens = (
max((pv.shape[0] for pv in pixel_values), default=0) // pool_ratio
)
return [
EncoderItemSpec(
input_size=pv.shape[0],
output_tokens=pv.shape[0] // pool_ratio,
path_output_tokens={"default": row_tokens},
)
for pv in pixel_values
]
@@ -1750,10 +1763,25 @@ class Gemma4ForConditionalGeneration(
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
from vllm.v1.worker.encoder_cudagraph_defs import (
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG,
)
vision_cfg = self.vision_tower.config
pool_ratio = getattr(vision_cfg, "pooling_kernel_size", 2) ** 2
modality = self.get_input_modality(mm_kwargs)
if modality == "image":
pixel_values = mm_kwargs["pixel_values"]
pixel_position_ids = mm_kwargs["pixel_position_ids"]
if isinstance(pixel_values, list):
row_patches = max(
(pixel_values[i].shape[0] for i in indices), default=0
)
else:
row_patches = pixel_values.shape[1]
axis_keys = (row_patches // pool_ratio,)
if len(indices) == 0:
is_pv_list = isinstance(pixel_values, list)
is_pp_list = isinstance(pixel_position_ids, list)
@@ -1762,20 +1790,24 @@ class Gemma4ForConditionalGeneration(
"pixel_position_ids": (
[] if is_pp_list else pixel_position_ids[:0]
),
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
}
if isinstance(pixel_values, list):
return {
"pixel_values": [pixel_values[i] for i in indices],
"pixel_position_ids": [pixel_position_ids[i] for i in indices],
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
}
return {
"pixel_values": pixel_values[indices],
"pixel_position_ids": pixel_position_ids[indices],
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
}
elif modality == "video":
pixel_values_videos = mm_kwargs["pixel_values_videos"]
pixel_position_ids_videos = mm_kwargs["pixel_position_ids_videos"]
video_frame_counts = mm_kwargs["video_frame_counts"]
axis_keys = (pixel_values_videos.shape[1] // pool_ratio,)
if len(indices) == 0:
is_fc_tensor = isinstance(video_frame_counts, torch.Tensor)
@@ -1785,6 +1817,7 @@ class Gemma4ForConditionalGeneration(
"video_frame_counts": (
video_frame_counts[:0] if is_fc_tensor else []
),
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
}
fc_list = (
@@ -1819,6 +1852,7 @@ class Gemma4ForConditionalGeneration(
"pixel_values_videos": selected_pv,
"pixel_position_ids_videos": selected_pp,
"video_frame_counts": selected_fc,
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
}
raise ValueError(f"Unknown modality: {modality}")
@@ -1839,20 +1873,21 @@ class Gemma4ForConditionalGeneration(
dtype = dtype or torch.float32
max_size = max(max_batch_size, max_frames_per_batch)
assert axis_keys is not None
(per_item_output,) = axis_keys
assert isinstance(per_item_output, int)
# Every row of a batch replayed on this graph is an image or video
# frame of per_item_output tokens, so a batch that fits token_budget
# never needs more rows than this. Capture needs at least one row.
max_size = min(
max(max_batch_size, max_frames_per_batch),
max(1, token_budget // per_item_output),
)
vision_cfg = self.vision_tower.config
pool_ratio = getattr(vision_cfg, "pooling_kernel_size", 2) ** 2
# Retrieve the model's actual configured maximum tokens:
configured_max_tokens = getattr(
self.config.vision_config,
"num_soft_tokens",
_SUPPORTED_SOFT_TOKENS[2],
)
# Dynamically compute the slot capacity per item bounded by both the
# current graph budget and the user's maximum config:
per_item_output = min(token_budget, configured_max_tokens)
# Satisfy k^2 * per_item_output = per_item_patches
per_item_patches = per_item_output * pool_ratio
+19 -1
View File
@@ -79,6 +79,24 @@ def _make_deepseek_v4_vl_weights_mapper(
)
def stream_language_model_first(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
"""Yield ``language_model.*`` weights as they arrive, then the rest.
Keeps the language model's weights in one contiguous group without
materializing the stream. Deferred weights are cloned so they do not keep
the loader's per-shard storage alive until the end of the load.
"""
rest = []
for name, tensor in weights:
if name.startswith("language_model."):
yield name, tensor
else:
rest.append((name, tensor.clone()))
yield from rest
@MULTIMODAL_REGISTRY.register_processor(
DeepseekV4VLMultiModalProcessor,
info=DeepseekV4VLProcessingInfo,
@@ -334,7 +352,7 @@ class DeepseekV4ForConditionalGeneration(
if child_finalizes:
# A child which finalizes inside load_weights must see all of its
# weights in one contiguous delegation from AutoWeightsLoader.
mapped = iter(sorted(mapped, key=lambda x: x[0]))
mapped = stream_language_model_first(mapped)
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(mapped)
self._weights_finalized = child_finalizes
+4 -6
View File
@@ -41,6 +41,7 @@ from vllm.models.deepseek_v4.common.vision import (
DeepseekV4ViT,
run_dp_sharded_vision_tower,
)
from vllm.models.deepseek_v4.common.vl_model import stream_language_model_first
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -317,12 +318,9 @@ class DeepseekV41ForCausalLM(
return self.language_model.get_mtp_target_hidden_states()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Map HF names into this wrapper's namespace up front and sort, so
# the "language_model." group reaches the child loader as one
# contiguous block (AutoWeightsLoader delegates per contiguous group,
# and the child's load_weights finalizes fused expert weights, which
# must not run on a partially loaded model).
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
# The child finalizes in load_weights, so its weights must reach it as
# one contiguous group.
mapped = stream_language_model_first(self.hf_to_vllm_mapper.apply(weights))
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(mapped)
# The child's load_weights already ran its post-load finalization.
+16 -13
View File
@@ -75,7 +75,7 @@ from vllm.model_executor.layers.quantization import (
resolve_quant_method,
)
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
from vllm.model_executor.utils import replace_parameter
from vllm.model_executor.utils import is_weights_pre_processed, replace_parameter
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.kimi_k3.nvidia.low_latency_gemm import try_low_latency_gemm
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
@@ -440,18 +440,21 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
projected into latent space by ``W_UK_T`` and the attention output is
projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V.
"""
W_UK, W_UV = split_kv_b_proj(
self.kv_b_proj,
act_dtype,
self.kv_lora_rank,
self.num_local_heads,
self.qk_nope_head_dim,
self.v_head_dim,
)
# (L, N, V) -> (N, L, V)
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
# (L, N, P) -> (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
# The weight cache daemon exports the absorbed weights. Repeating the
# absorption would mutate its shared CUDA allocation.
if not (is_weights_pre_processed() and getattr(self, "W_UV", None) is not None):
W_UK, W_UV = split_kv_b_proj(
self.kv_b_proj,
act_dtype,
self.kv_lora_rank,
self.num_local_heads,
self.qk_nope_head_dim,
self.v_head_dim,
)
# (L, N, V) -> (N, L, V)
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
# (L, N, P) -> (N, P, L)
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
quant_method = (
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
+5
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiMo's compact XML parameters contain verbatim string values."""
import functools
import json
from dataclasses import replace
@@ -10,6 +11,7 @@ from vllm.parser.qwen3 import (
_PARTIAL_PARAM_RE,
CHATML_TURN_BOUNDARIES,
Qwen3Parser,
_param_value_extender,
qwen3_config,
)
@@ -37,6 +39,9 @@ class MiMoParser(Qwen3Parser):
turn_boundary_tokens=CHATML_TURN_BOUNDARIES,
),
arg_converter=_mimo_arg_converter,
arg_value_extender=functools.partial(
_param_value_extender, trim_newlines=False
),
),
)
super().__init__(tokenizer, tools, **kwargs)
+60
View File
@@ -25,6 +25,7 @@ import regex as re
from vllm.parser.engine.events import EventType
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import (
ArgValueExtender,
ParserEngineConfig,
ParserState,
Transition,
@@ -86,6 +87,64 @@ def _qwen3_arg_converter(raw_args: str, partial: bool) -> str:
return json.dumps(params, ensure_ascii=False)
# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener.
_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s*=")
def _param_value_end_window(text: str) -> str:
"""Return the suffix of *text* that may still grow into a value end."""
start = text.rfind("<")
if start == -1:
return ""
tail = text[start:]
return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else ""
def _param_value_extender(
raw_args: str, trim_newlines: bool
) -> ArgValueExtender | None:
"""Extend the open trailing ``<parameter=...>`` value of *raw_args*.
The extender streams exactly what the converter would for the same text.
With *trim_newlines* it follows ``_trim_wrapping_newlines``: it drops a
leading newline of the value and holds back a trailing one.
"""
match = _PARTIAL_PARAM_RE.search(_PARAM_RE.sub("", raw_args))
if match is None:
return None
name, value = match.group(1), match.group(2)
# A repeated key keeps its first position, so its value is not last.
if any(m.group(1) == name for m in _PARAM_RE.finditer(raw_args)):
return None
window = _param_value_end_window(value)
started = bool(value)
held = "\n" if trim_newlines and value.endswith("\n") and value != "\n" else ""
def extend(delta: str) -> str | None:
nonlocal window, started, held
text = window + delta
if _PARAM_VALUE_END_RE.search(text):
return None
window = _param_value_end_window(text)
if trim_newlines:
text = held + delta
if not started and text:
started = True
text = text.removeprefix("\n")
held = "\n" if text.endswith("\n") else ""
text = text[: len(text) - len(held)]
else:
text = delta
return json.dumps(text, ensure_ascii=False)[1:-1]
return extend
def _qwen3_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
return _param_value_extender(raw_args, trim_newlines=True)
@functools.cache
def qwen3_config(
thinking: bool = True,
@@ -223,6 +282,7 @@ def qwen3_config(
),
},
arg_converter=_qwen3_arg_converter,
arg_value_extender=_qwen3_arg_value_extender,
stream_arg_deltas=True,
strip_trailing_reasoning_whitespace=False,
tool_args_json=False,
+6
View File
@@ -405,6 +405,12 @@ if (
):
os.environ["HIP_ONLINE_TUNING"] = "1"
# RCCL 2.30's RAS thread can abort the process at exit ("terminate called
# without an active exception") when its shutdown notification fails.
# https://github.com/ROCm/rocm-systems/issues/13572
if torch.version.hip:
os.environ.setdefault("NCCL_RAS_ENABLE", "0")
@cache
def use_rocm_custom_paged_attention(
+8
View File
@@ -834,6 +834,9 @@ class OnlineDerenderer:
total_completion_tokens = 0
index = 0
top_limit = _completion_top_logprobs_limit(completion_request)
return_token_ids = (
completion_request is not None and completion_request.return_token_ids
)
for gen, pt, seed_ids_override in zip(
generate_responses, prompt_tokens_list, prompt_token_ids_list
@@ -877,6 +880,7 @@ class OnlineDerenderer:
finish_reason=choice.finish_reason,
stop_reason=choice.stop_reason,
logprobs=completion_logprobs,
token_ids=choice.token_ids if return_token_ids else None,
)
)
total_completion_tokens += len(choice.token_ids)
@@ -942,6 +946,9 @@ class OnlineDerenderer:
updated_state = state
top_limit = _completion_top_logprobs_limit(completion_request)
return_token_ids = (
completion_request is not None and completion_request.return_token_ids
)
for choice in generate_chunk.choices:
delta_tids = choice.token_ids or []
new_text, updated_state = await self._detokenize_delta_async(
@@ -982,6 +989,7 @@ class OnlineDerenderer:
logprobs=completion_logprobs,
finish_reason=choice.finish_reason,
stop_reason=choice.stop_reason,
token_ids=choice.token_ids if return_token_ids else None,
)
)
+33 -1
View File
@@ -8,7 +8,7 @@ import math
from dataclasses import field
from enum import Enum, IntEnum
from functools import cached_property
from typing import Annotated, Any
from typing import TYPE_CHECKING, Annotated, Any
import msgspec
import numpy as np
@@ -24,6 +24,9 @@ from vllm.utils.mistral import is_mistral_tokenizer
from vllm.utils.torch_utils import make_ndarray_with_pad
from vllm.v1.serial_utils import PydanticMsgspecMixin
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
_SAMPLING_EPS = 1e-5
@@ -1507,6 +1510,35 @@ class BeamSearchParams(
structured_outputs: StructuredOutputsParams | None = None
skip_special_tokens: bool = True
watermarking: bool | None = None
allowed_token_ids: list[int] | None = None
def __post_init__(self) -> None:
_verify_num_sequences(self.beam_width, "beam_width")
def configure_logprobs(
self, sampling_params: SamplingParams, config: "VllmConfig"
) -> None:
ids = self.allowed_token_ids
if ids is None:
return
if not ids:
raise ValueError("Beam search allowed_token_ids must not be empty")
if self.structured_outputs is not None:
# Grammar candidates can exceed the exact-ID limit of 128 tokens.
raise ValueError(
"Beam allowlists with structured outputs are not supported"
)
if config.speculative_config is not None:
# Rejection sampling does not yet fully support exact-ID logprobs.
raise ValueError(
"Beam allowlists with speculative decoding are not supported"
)
if config.model_config.is_diffusion:
# The diffusion sampler does not consume logprob_token_ids.
raise ValueError("Beam allowlists with diffusion models are not supported")
if not config.use_v2_model_runner:
# V1 exact-ID logprobs overwrite other rows in a mixed batch.
raise ValueError("Beam allowlists require the V2 model runner")
# Score the original distribution without applying a sampling mask.
sampling_params.logprobs = None
sampling_params.logprob_token_ids = ids
+12 -111
View File
@@ -9,7 +9,7 @@ from dataclasses import replace
from functools import partial, reduce
from typing import Any
from vllm.config import VllmConfig
from vllm.config import SpeculativeConfig, VllmConfig
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.mem_utils import format_gib
@@ -1038,120 +1038,20 @@ def _get_packed_kv_cache_groups(
assert group_spec is not None
groups.append(KVCacheGroupSpec(group_layer_names, group_spec))
_annotate_eagle_groups(
vllm_config,
kv_cache_spec,
groups,
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
)
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
return groups
def _uses_trailing_mtp_layers(vllm_config: VllmConfig) -> bool:
"""Whether the drafter's KV layers can be located positionally.
MTP drafters register their KV layers after the target layers, without a
distinguishing spec marker. DeepseekV4/V4.1 do this with ``dspark`` too.
The annotator separately checks that the groups partition the layers exactly.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle():
return False
if spec_config.method == "mtp":
return True
model_config = vllm_config.model_config
return model_config is not None and model_config.hf_config.model_type in (
"deepseek_v4",
"deepseek_v41",
)
def _groups_partition_layers_exactly(
kv_cache_spec: dict[str, KVCacheSpec],
kv_cache_groups: list[KVCacheGroupSpec],
) -> bool:
"""Whether the groups cover every layer of ``kv_cache_spec`` exactly once.
The trailing-layer draft fallback is only meaningful when the last
registered layer is guaranteed to appear in exactly one group.
"""
names = [name for group in kv_cache_groups for name in group.layer_names]
return len(names) == len(kv_cache_spec) and set(names) == set(kv_cache_spec)
def _annotate_eagle_groups(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
kv_cache_groups: list[KVCacheGroupSpec],
use_trailing_layer_fallback: bool = False,
) -> None:
"""Flag the KV cache groups that hold drafter attention layers.
Two detection rules, in order of preference:
1. Spec-driven. ``non_causal_multi_token_decode`` is declared on
MLAAttentionSpec and set by drafter attention layers that run a
non-causal multi-token decode (today only Kimi-K3 DSpark). It survives
MLAAttentionSpec.merge, so it still identifies a group after per-group
spec merging, wherever grouping happens to land. It is sufficient but
not necessary: a drafter whose spec is indistinguishable from the
target's cannot be found this way.
2. Positional fallback for MTP drafters (including DeepseekV4/V4.1 DSpark), whose
MTP block reuses the target's own decoder layer and so carries no spec
marker. Their draft attention layers always register after every
target layer, so flag whichever group holds the last registered layer.
This rule is only valid where the groups partition exactly the layers
of ``kv_cache_spec``, which is re-checked here before applying it.
When a drafter's trailing caches span several groups, this rule flags
only the group holding the very last layer and must be generalized.
FIXME(yifan): avoid/generalize this hacky check.
Args:
vllm_config: Config supplying the speculative method, if any.
kv_cache_spec: The kv cache spec of each attention layer, in layer
registration order. Only read by rule 2.
kv_cache_groups: Groups to annotate in place.
use_trailing_layer_fallback: Enable rule 2. Callers gate this on
``_uses_trailing_mtp_layers``.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle_block_drop():
return
for group in kv_cache_groups:
if any(
getattr(spec, "non_causal_multi_token_decode", False)
for spec in iter_layer_specs(group.kv_cache_spec)
):
group.is_eagle_group = True
if not use_trailing_layer_fallback:
return
if not _groups_partition_layers_exactly(kv_cache_spec, kv_cache_groups):
return
last_layer = next(reversed(kv_cache_spec))
for group in kv_cache_groups:
if last_layer in group.layer_names:
group.is_eagle_group = True
break
def _warn_if_unannotated_eagle_mamba(
vllm_config: VllmConfig,
spec_config: SpeculativeConfig,
kv_cache_groups: list[KVCacheGroupSpec],
) -> None:
"""Warn when no KV cache group could be identified as the draft model's.
Args:
vllm_config: Config supplying the speculative method, if any.
spec_config: Config supplying the speculative method.
kv_cache_groups: Groups as they will be handed to consumers.
"""
spec_config = vllm_config.speculative_config
if spec_config is None or not spec_config.use_eagle():
return
if any(group.is_eagle_group for group in kv_cache_groups):
return
mamba_groups = [
@@ -1297,13 +1197,6 @@ def get_kv_cache_groups(
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
groups.append(KVCacheGroupSpec([name], aligned))
_annotate_eagle_groups(
vllm_config,
kv_cache_spec,
groups,
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
)
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
return groups
@@ -1556,7 +1449,7 @@ def _project_kv_cache_groups_to_worker(
KVCacheGroupSpec(
worker_layer_names,
group_spec,
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
is_eagle_group=group.is_eagle_group,
host_resident=group.host_resident,
enable_kv_transfer=group.enable_kv_transfer,
role=group.role,
@@ -1636,6 +1529,14 @@ def get_kv_cache_configs(
global_kv_cache_groups = get_kv_cache_groups_from_builder(
builder, vllm_config, merged_kv_cache_specs
)
spec_config = vllm_config.speculative_config
if spec_config is not None and spec_config.use_eagle_block_drop():
draft_layer_names = {
name for name, spec in merged_kv_cache_specs.items() if spec.is_draft
}
for group in global_kv_cache_groups:
group.is_eagle_group = not draft_layer_names.isdisjoint(group.layer_names)
_warn_if_unannotated_eagle_mamba(spec_config, global_kv_cache_groups)
# If original_max_model_len was -1, automatically
# determine the maximum model length that fits in available GPU memory.
+14 -9
View File
@@ -1732,14 +1732,14 @@ class Scheduler(SchedulerInterface):
Discards the last sampled output token from the prior input chunk.
"""
# Current streaming input behaviour: Keep only computed output tokens
# (discard final sampled output token).
num_computed_tokens = session.num_computed_tokens
# Keep all output tokens but the final sampled one.
keep_end = session.num_tokens - 1
assert keep_end >= session.num_computed_tokens
kept_output_tokens = session._all_token_ids[
session.num_prompt_tokens : num_computed_tokens
session.num_prompt_tokens : keep_end
]
del session._all_token_ids[num_computed_tokens:]
del session.block_hashes[num_computed_tokens // self.hash_block_size :]
del session._all_token_ids[keep_end:]
del session.block_hashes[keep_end // self.hash_block_size :]
session._output_token_ids.clear()
assert session.prompt_token_ids is not None
# Extend prompt with kept output tokens.
@@ -2155,6 +2155,8 @@ class Scheduler(SchedulerInterface):
# to avoid expensive operations inside the loop.
stopped_running_reqs: set[Request] = set()
stopped_preempted_reqs: set[Request] = set()
# Stopped resumable requests to re-enqueue, in output order.
continuing_reqs: list[Request] = []
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
assert num_tokens_scheduled > 0
request = self.requests.get(req_id)
@@ -2320,6 +2322,8 @@ class Scheduler(SchedulerInterface):
kv_transfer_params["remote_prefill_cached_tokens"] = (
prefill_stats.num_cached_tokens
)
else:
continuing_reqs.append(request)
if status_before_stop == RequestStatus.RUNNING:
stopped_running_reqs.add(request)
@@ -2371,14 +2375,16 @@ class Scheduler(SchedulerInterface):
assert not prompt_logprobs_tensors
assert prompt_token_id_logprobs is None
# Remove the stopped requests from the running and waiting queues.
# Remove stopped requests from their queues, then enqueue the ones
# that continue, so each ends up in exactly one queue.
if stopped_running_reqs:
self.running = remove_all(self.running, stopped_running_reqs)
if stopped_preempted_reqs:
# This is a rare case and unlikely to impact performance.
self.waiting.remove_requests(stopped_preempted_reqs)
self.kv_holding_waiting.remove_requests(stopped_preempted_reqs)
self.deferred_waiting.difference_update(stopped_preempted_reqs)
for request in continuing_reqs:
self._enqueue_waiting_request(request)
error_req_ids = set(self.grammar_compile_error_reqs)
self.grammar_compile_error_reqs.clear()
@@ -2568,7 +2574,6 @@ class Scheduler(SchedulerInterface):
request.status = RequestStatus.WAITING_FOR_STREAMING_REQ
self.num_waiting_for_streaming_input += 1
self._enqueue_waiting_request(request)
return False
def _update_request_with_output(
+8
View File
@@ -599,6 +599,7 @@ class AsyncLLM(EngineClient):
session_id: str | None = None,
kv_hints: KvHintsEnvelope | None = None,
) -> RequestOutputCollector:
self._validate_resumable_request()
self._validate_streaming_input_sampling_params(sampling_params)
inputs = dict(
@@ -693,6 +694,13 @@ class AsyncLLM(EngineClient):
queue._input_stream_task = asyncio.create_task(handle_inputs())
return queue
def _validate_resumable_request(self) -> None:
if self.vllm_config.is_mm_encoder_only or self.model_config.is_diffusion:
raise VLLMValidationError(
"Resumable requests are not supported for encoder-only or "
"diffusion models."
)
@staticmethod
def _validate_streaming_input_sampling_params(
params: SamplingParams | PoolingParams,
+11 -1
View File
@@ -175,6 +175,10 @@ class KVCacheSpec:
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
blocks of up to ``n`` tokens."""
is_draft: bool = field(default=False, kw_only=True, compare=False)
"""Whether a draft model owns this layer; excluded from equality so it never
affects grouping."""
def __post_init__(self):
alignment = self.block_stride_alignment
if isinstance(alignment, int) and alignment <= 0:
@@ -660,6 +664,8 @@ class FullAttentionSpec(AttentionSpec):
)
for spec in specs:
for f in fields(AttentionSpec):
if not f.compare:
continue
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
"All attention layers in the same KV cache group must have "
"the same attention spec."
@@ -747,6 +753,8 @@ class MLAAttentionSpec(FullAttentionSpec):
)
for spec in specs:
for f in fields(AttentionSpec):
if not f.compare:
continue
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
"All attention layers in the same KV cache group must have "
"the same attention spec."
@@ -1208,6 +1216,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
)
for spec in specs:
for f in fields(AttentionSpec):
if not f.compare:
continue
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
"All attention layers in the same KV cache group must have "
"the same attention spec."
@@ -1445,7 +1455,7 @@ class KVCacheGroupSpec:
layer_names: list[str]
# The KV cache spec of this manager layer
kv_cache_spec: KVCacheSpec
# Whether this group contains EAGLE/MTP draft attention layers.
# Whether this group holds draft layers and takes the EAGLE last-block drop.
is_eagle_group: bool = False
# Host groups use the dedicated HiSparse pool; others share the device pool.
host_resident: bool = False
+4 -1
View File
@@ -1330,10 +1330,13 @@ class SpecDecodeBaseProposer:
self.vllm_config,
AttentionLayerBase, # type: ignore[type-abstract]
)
draft_attn_layer_names = set(all_attn_layers) - target_attn_layer_names
for name in draft_attn_layer_names:
all_attn_layers[name].is_draft_layer = True
# Filter to only layers that have KV cache specs.
self._draft_attn_layer_names = {
name
for name in (set(all_attn_layers.keys()) - target_attn_layer_names)
for name in draft_attn_layer_names
if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None
}
+3 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from dataclasses import dataclass, replace
from typing import TYPE_CHECKING, Any, cast
import numpy as np
@@ -148,6 +148,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
if spec := attn_module.get_kv_cache_spec(vllm_config):
if isinstance(spec, AttentionSpec):
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
if attn_module.is_draft_layer:
spec = replace(spec, is_draft=True)
kv_cache_spec[layer_name] = spec
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
+13 -6
View File
@@ -74,6 +74,7 @@ class BlockTables:
self.input_block_tables: list[torch.Tensor] = [
torch.zeros_like(b.gpu) for b in self.block_tables
]
self._num_dirty_rows = 0
self.slot_mappings = torch.zeros(
self.num_kv_cache_groups,
@@ -151,15 +152,18 @@ class BlockTables:
) -> tuple[torch.Tensor, ...]:
if self.num_kv_cache_groups == 0:
return ()
num_reqs = idx_mapping.shape[0]
num_rows = num_reqs_padded
if out is None:
out = tuple(self.input_block_tables)
out_ptrs = self.input_block_table_ptrs
num_rows = max(num_rows, self._num_dirty_rows)
self._num_dirty_rows = num_reqs
else:
assert out_ptrs is not None
assert len(out) == self.num_kv_cache_groups
num_reqs = idx_mapping.shape[0]
# Launch kernel with num_reqs_padded to fuse zeroing of padded rows.
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_reqs_padded)](
# Also clear rows left by a larger batch in the shared buffer.
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_rows)](
idx_mapping,
self.block_table_ptrs,
out_ptrs,
@@ -180,9 +184,12 @@ class BlockTables:
# Zero the rows so dummy runs write mamba state to the reserved null
# block rather than through the previous real step's (stale) block
# ids, which may point at blocks since freed and reallocated.
return tuple(
block_table[:num_reqs].zero_() for block_table in self.input_block_tables
)
num_rows = max(num_reqs, self._num_dirty_rows)
for block_table in self.input_block_tables:
block_table[:num_rows].zero_()
# Profiling may populate the returned rows with valid block IDs.
self._num_dirty_rows = num_reqs
return tuple(block_table[:num_reqs] for block_table in self.input_block_tables)
def compute_slot_mappings(
self,
+19 -4
View File
@@ -41,9 +41,14 @@ class KVConnector:
def finish_forward(self) -> None:
pass
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
def post_forward(
self, finished_req_ids: set[str], defer_finalize: bool = False
) -> KVConnectorOutput | None:
return None
def finalize_step(self) -> None:
pass
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
return EMPTY_MODEL_RUNNER_OUTPUT
@@ -128,7 +133,9 @@ class ActiveKVConnector(KVConnector):
def reset_capture_state(self) -> None:
self.kv_connector.reset_capture_state()
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
def post_forward(
self, finished_req_ids: set[str], defer_finalize: bool = False
) -> KVConnectorOutput | None:
if self._disabled:
return None
@@ -136,7 +143,8 @@ class ActiveKVConnector(KVConnector):
self._start_load_kv()
output = KVConnectorOutput()
self.kv_connector.wait_for_save()
if not defer_finalize:
self.kv_connector.wait_for_save()
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
output.finished_sending = transfer_results.finished_sending or None
output.finished_recving = transfer_results.finished_recving or None
@@ -147,9 +155,16 @@ class ActiveKVConnector(KVConnector):
output.kv_connector_worker_meta = (
self.kv_connector.build_connector_worker_meta()
)
self.kv_connector.clear_connector_metadata()
if not defer_finalize:
self.kv_connector.clear_connector_metadata()
return output
def finalize_step(self) -> None:
if self._disabled:
return
self.kv_connector.wait_for_save()
self.kv_connector.clear_connector_metadata()
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
if self._disabled:
return EMPTY_MODEL_RUNNER_OUTPUT
+8 -3
View File
@@ -2207,6 +2207,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
pending_aux_output=pending_aux_output,
)
# Collect KV transfer results now; finalize saves after the drafter runs.
defer_kv_finalize = self.speculator is not None
model_runner_output.kv_connector_output = self.kv_connector.post_forward(
finished_req_ids, defer_finalize=defer_kv_finalize
)
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
# The encoder runner exists only on the first PP rank, so later ranks
# have no cached embeddings to gather.
@@ -2288,9 +2294,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.draft_tokens, input_batch
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
model_runner_output.kv_connector_output = kv_connector_output
if defer_kv_finalize:
self.kv_connector.finalize_step()
model_runner_output.ec_connector_output = ec_connector_output
return async_output
+2 -1
View File
@@ -238,7 +238,8 @@ def warmup_kernels(
rejection_sampler.enable_adaptive_verification = False
try:
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
if rejection_sampler is not None:
# All PP ranks must run the same forwards to match sends and receives.
if model_runner.speculative_config is not None:
# Greedy batches skip logits processing, so rejection kernels
# also see model-dtype logits instead of FP32.
_warmup_kernels(
+2
View File
@@ -7140,6 +7140,8 @@ class GPUModelRunner(
spec = customize_attention_spec(
attn_module.get_attn_backend(), spec
)
if attn_module.is_draft_layer:
spec = replace(spec, is_draft=True)
kv_cache_spec[layer_name] = spec
return kv_cache_spec
+3 -3
View File
@@ -61,9 +61,9 @@ def _torch_cuda_wrapper():
torch.cuda.stream = partial(torch.xpu.stream)
torch.cuda.set_stream = partial(torch.xpu.set_stream)
# torch.xpu.Event does not accept the ``blocking`` kwarg that
# torch.cuda.Event supports, so drop it here.
def _xpu_event(*args, blocking=None, **kwargs):
# torch.xpu.Event does not accept the ``blocking`` or ``external`` kwargs
# that torch.cuda.Event supports, so drop them here.
def _xpu_event(*args, blocking=None, external=None, **kwargs):
return torch.xpu.Event(*args, **kwargs)
torch.cuda.Event = _xpu_event