Merge branch 'main' into patches
This commit is contained in:
+16
-6
@@ -2124,7 +2124,6 @@ class rocm_aiter_ops:
|
||||
VLLM_ROCM_USE_AITER_MHA: Controls MHA ops including flash_attn_varlen.
|
||||
VLLM_ROCM_USE_AITER_UNIFIED_ATTENTION: Controls Triton unified attention.
|
||||
VLLM_ROCM_USE_AITER_FP8BMM: Controls FP8 batched matrix multiply.
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: Controls FP4 assembly GEMM.
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: Controls Triton rotary embeddings.
|
||||
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: Controls Triton sparse MLA (gfx950).
|
||||
VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS: Controls shared expert fusion.
|
||||
@@ -2196,8 +2195,6 @@ class rocm_aiter_ops:
|
||||
_FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM
|
||||
_FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM
|
||||
_LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
|
||||
# TODO: Consolidate under _LINEAR_ENABLED
|
||||
_FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
# TODO: Consolidate under VLLM_ROCM_USE_AITER_ROPE
|
||||
_TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
_TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA
|
||||
@@ -2228,7 +2225,6 @@ class rocm_aiter_ops:
|
||||
cls._FP8BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP8BMM
|
||||
cls._FP4BMM_ENABLED = envs.VLLM_ROCM_USE_AITER_FP4BMM
|
||||
cls._LINEAR_HIPBMM_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR_HIPBMM
|
||||
cls._FP4_GEMM_DYNAMIC_QUANT_ASM = envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM
|
||||
cls._TRITON_ROTARY_EMBED = envs.VLLM_ROCM_USE_AITER_TRITON_ROPE
|
||||
cls._TRITON_SPARSE_MLA = envs.VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA
|
||||
cls._MOE_SHARED_EXPERTS_ENABLED = envs.VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS
|
||||
@@ -2565,10 +2561,10 @@ class rocm_aiter_ops:
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_asm_fp4_gemm_dynamic_quant_enabled(cls) -> bool:
|
||||
def is_asm_fp4_gemm_supported(cls) -> bool:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return cls._AITER_ENABLED and cls._FP4_GEMM_DYNAMIC_QUANT_ASM and on_gfx950()
|
||||
return cls._AITER_ENABLED and on_gfx950()
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
@@ -4360,6 +4356,20 @@ class rocm_aiter_ops:
|
||||
except (AssertionError, ImportError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
@functools.cache
|
||||
def is_asm_fp4_gemm_tuned(n: int, k_bytes: int) -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
try:
|
||||
from aiter.ops.gemm_op_a4w4 import get_GEMM_config
|
||||
|
||||
return all(
|
||||
get_GEMM_config(m, n, 2 * k_bytes) is not None for m in (1024, 8192)
|
||||
)
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
@functools.cache
|
||||
def is_shuffled_per_token_w8a8_gemm_tuned(
|
||||
|
||||
@@ -339,6 +339,17 @@ def fused_add_rms_norm(
|
||||
torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon)
|
||||
|
||||
|
||||
def muse_spark_mul_tanh_soft_cap(
|
||||
x: torch.Tensor,
|
||||
multiplier: float,
|
||||
soft_cap_temp: float,
|
||||
) -> torch.Tensor:
|
||||
"""Cast to FP32 and apply ``temp * tanh(x * multiplier / temp)`` in one pass."""
|
||||
out = torch.empty_like(x, dtype=torch.float32)
|
||||
torch.ops._C.muse_spark_mul_tanh_soft_cap(out, x, multiplier, soft_cap_temp)
|
||||
return out
|
||||
|
||||
|
||||
def fused_qk_norm_rope(
|
||||
qkv: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
@@ -2598,6 +2609,105 @@ def reshape_and_cache_flash(
|
||||
)
|
||||
|
||||
|
||||
def fused_qknorm_rope_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rotary_dim: int,
|
||||
eps: float,
|
||||
norm_weight_bias: float = 0.0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Fuse per-head RMSNorm, partial NeoX RoPE and KV insertion.
|
||||
|
||||
Norm weights may be FP32 independently of QKV. A zero rotary dimension
|
||||
skips RoPE; norm outputs are rounded to QKV dtype before rotation.
|
||||
"""
|
||||
if rotary_dim < 0 or rotary_dim > head_dim or rotary_dim % 8:
|
||||
raise ValueError("rotary_dim must be a multiple of 8 in [0, head_dim]")
|
||||
q = qkv.new_empty((qkv.shape[0], num_q_heads * head_dim))
|
||||
k = qkv.new_empty((qkv.shape[0], num_kv_heads * head_dim))
|
||||
v = torch.empty_like(k)
|
||||
cos, sin = cos_sin_cache.chunk(2, dim=-1)
|
||||
torch.ops._C.fused_qknorm_rope_cache(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos,
|
||||
sin,
|
||||
positions,
|
||||
slot_mapping,
|
||||
key_cache,
|
||||
value_cache,
|
||||
head_dim,
|
||||
num_q_heads,
|
||||
num_kv_heads,
|
||||
rotary_dim // 2,
|
||||
eps,
|
||||
norm_weight_bias,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
return q, k, v
|
||||
|
||||
|
||||
def fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
cos_sin_cache: torch.Tensor | None = None,
|
||||
rotary_dim: int = 0,
|
||||
slot_mapping: torch.Tensor | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
block_size: int = 0,
|
||||
kv_cache_dtype: str = "auto",
|
||||
k_scale: torch.Tensor | None = None,
|
||||
v_scale: torch.Tensor | None = None,
|
||||
q_fp8_out: torch.Tensor | None = None,
|
||||
q_scale: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
"""In-place weightless QK RMSNorm + GPT-J RoPE, optionally writing k/v to cache.
|
||||
|
||||
When ``kv_cache`` is given, the caller must skip the attention layer's own KV write.
|
||||
With ``q_fp8_out``, write scaled E4M3 Q there instead of rewriting Q in ``qkv``.
|
||||
"""
|
||||
torch.ops._C.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
head_dim,
|
||||
eps,
|
||||
cos_sin_cache,
|
||||
rotary_dim,
|
||||
slot_mapping,
|
||||
kv_cache,
|
||||
block_size,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
q_fp8_out,
|
||||
q_scale,
|
||||
)
|
||||
|
||||
|
||||
def fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv: torch.Tensor,
|
||||
q_norm_weight: torch.Tensor,
|
||||
|
||||
@@ -55,6 +55,8 @@ class ForwardBench:
|
||||
iters: int,
|
||||
samples: int,
|
||||
state_views: list[torch.Tensor],
|
||||
name: str,
|
||||
profile_prefix: str | None = None,
|
||||
):
|
||||
self.runner = runner
|
||||
self.batch = batch
|
||||
@@ -62,6 +64,8 @@ class ForwardBench:
|
||||
self.iters = iters
|
||||
self.samples = samples
|
||||
self.state_views = state_views
|
||||
self.name = name
|
||||
self.profile_prefix = profile_prefix
|
||||
self.phase = "warmup"
|
||||
self.result: dict[str, Any] = {}
|
||||
|
||||
@@ -193,6 +197,20 @@ class ForwardBench:
|
||||
),
|
||||
)
|
||||
self.result["latency_us"] = self._time(graph)
|
||||
from vllm.benchmarks.forward.profile import profile_graph
|
||||
|
||||
# Each replay restores the state views, then (multi-rank) rendezvouses,
|
||||
# before the timed forward.
|
||||
skip = len(self.state_views) + (world.world_size > 1)
|
||||
summary = profile_graph(graph, self.name, self.profile_prefix, skip)
|
||||
if summary is not None:
|
||||
self.result["profile"] = {
|
||||
"summary": f"{self.profile_prefix}.kernels.json",
|
||||
"span_us": summary["span_us"],
|
||||
"busy_us": summary["busy_us"],
|
||||
"activities": summary["activities"],
|
||||
"top_kernels": summary["kernels"][:10],
|
||||
}
|
||||
del graph
|
||||
return output
|
||||
|
||||
|
||||
@@ -74,6 +74,13 @@ def add_cli_args(parser: FlexibleArgumentParser) -> None:
|
||||
parser.add_argument("--iters", type=int, default=20, help="Replays per sample.")
|
||||
parser.add_argument("--samples", type=int, default=5)
|
||||
parser.add_argument("--output-json", help="Write results to this file.")
|
||||
parser.add_argument(
|
||||
"--profile-dir",
|
||||
help="After timing, trace a few replays of each case and write "
|
||||
"`<case>.rank<r>.trace.json.gz` (Perfetto) and `<case>.rank<r>.kernels.json` "
|
||||
"(GPU time per kernel per forward) here. Summarize or compare with "
|
||||
"`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.",
|
||||
)
|
||||
EngineArgs.add_cli_args(parser)
|
||||
parser.set_defaults(load_format="dummy", enable_prefix_caching=False)
|
||||
|
||||
@@ -146,6 +153,13 @@ def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any
|
||||
dcp_size,
|
||||
)
|
||||
zero_kv_blocks(context, kv_config.kv_cache_groups, batch, kv_config.num_blocks)
|
||||
profile_prefix = None
|
||||
if args.profile_dir:
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
|
||||
os.makedirs(args.profile_dir, exist_ok=True)
|
||||
rank = get_world_group().rank
|
||||
profile_prefix = os.path.join(args.profile_dir, f"{case.name}.rank{rank}")
|
||||
bench = ForwardBench(
|
||||
runner,
|
||||
batch,
|
||||
@@ -154,6 +168,8 @@ def run_case(runner, case: BenchCase, args: argparse.Namespace) -> dict[str, Any
|
||||
touched_kv_views(
|
||||
context, kv_config.kv_cache_groups, batch, kv_config.num_blocks, dcp_size
|
||||
),
|
||||
case.name,
|
||||
profile_prefix,
|
||||
)
|
||||
routing, balanced = install_routing(
|
||||
runner, args.routing, args.load_format == "dummy", batch.num_tokens, args.seed
|
||||
@@ -221,6 +237,14 @@ def main(args: argparse.Namespace) -> None:
|
||||
f"eager rel rms {rank0['graph_vs_eager']['relative_rms']:.1e}",
|
||||
flush=True,
|
||||
)
|
||||
if profile := rank0.get("profile"):
|
||||
from vllm.benchmarks.forward.profile import format_summary
|
||||
|
||||
print(
|
||||
f"[forward] {case.name} rank 0 kernels ({profile['summary']}):\n"
|
||||
+ format_summary({**profile, "kernels": profile["top_kernels"]}),
|
||||
flush=True,
|
||||
)
|
||||
if latency["sample_spread_pct"] > 0.5:
|
||||
logger.warning(
|
||||
"%s: unstable timing (sample medians spread %.2f%%).",
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GPU kernel profiles of `vllm bench forward` cases.
|
||||
|
||||
With `--profile-dir`, each case's graph is replayed a few more times under
|
||||
`torch.profiler` after timing. Each rank writes a Perfetto trace and a kernel
|
||||
summary (`<case>.rank<r>.kernels.json`): per-kernel time per forward, sorted.
|
||||
|
||||
Summarize one file or compare two (e.g. before/after a change):
|
||||
`python -m vllm.benchmarks.forward.profile A.kernels.json [B.kernels.json]`.
|
||||
|
||||
The same replays run between `cudaProfilerStart`/`Stop` in an NVTX range per
|
||||
case (with or without `--profile-dir`), so Nsight Systems can record just them:
|
||||
`nsys profile --capture-range=cudaProfilerApi --capture-range-end=repeat
|
||||
--cuda-graph-trace=node vllm bench forward ...`.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
_GPU_CATEGORIES = ("kernel", "gpu_memcpy", "gpu_memset")
|
||||
# Kernels named after a generic launcher are identified by their first
|
||||
# template argument instead.
|
||||
_GENERIC = re.compile(r"(^|::)(device_kernel|Kernel|kernel|KernelWrapper)$")
|
||||
# CuTe DSL kernel names embed per-shape layouts after these markers.
|
||||
_CUTE_DSL_SUFFIX = re.compile(r"_object_at_|_tensorptr")
|
||||
|
||||
|
||||
def _match_back(text: str, end: int, open_ch: str, close_ch: str) -> int:
|
||||
"""Index of the bracket opening the one closing at `end`, skipping
|
||||
brackets of the other kind nested inside parentheses."""
|
||||
depth, parens = 0, 0
|
||||
for i in range(end, -1, -1):
|
||||
ch = text[i]
|
||||
if open_ch == "<" and ch in "()":
|
||||
parens += 1 if ch == ")" else -1
|
||||
continue
|
||||
if parens:
|
||||
continue
|
||||
if ch == close_ch:
|
||||
depth += 1
|
||||
elif ch == open_ch:
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return i
|
||||
return -1
|
||||
|
||||
|
||||
def short_name(name: str) -> str:
|
||||
"""`void ns::foo<T, 4>(args)` -> `ns::foo`: return type, template and
|
||||
function arguments dropped. Generic launchers (e.g.
|
||||
`cutlass::device_kernel<Op<...>>`) keep their first template argument."""
|
||||
if name.startswith("kernel_cutlass_"):
|
||||
return _CUTE_DSL_SUFFIX.split(name, maxsplit=1)[0]
|
||||
head = name.strip()
|
||||
if head.endswith(")"):
|
||||
head = head[: max(_match_back(head, len(head) - 1, "(", ")"), 0)].rstrip()
|
||||
template = ""
|
||||
if head.endswith(">"):
|
||||
start = _match_back(head, len(head) - 1, "<", ">")
|
||||
if start > 0:
|
||||
head, template = head[:start], head[start + 1 : -1]
|
||||
base = head.split()[-1] if head.split() else name
|
||||
if _GENERIC.search(base) and template:
|
||||
arg = re.split(r"[<>(,]", template, maxsplit=1)[0].strip()
|
||||
if arg:
|
||||
return f"{base}<{arg.removeprefix('void ')}>"
|
||||
return base
|
||||
|
||||
|
||||
def summarize(events: list[dict], replays: int, skip: int) -> dict[str, Any]:
|
||||
"""Per-forward kernel times from the GPU activities of `replays` replays.
|
||||
|
||||
Each replay first runs `skip` activities that aren't part of the timed
|
||||
forward (the KV state restore and the rank rendezvous); they are reported
|
||||
separately as `pre_forward_us`.
|
||||
"""
|
||||
acts = sorted(
|
||||
(e for e in events if e.get("cat") in _GPU_CATEGORIES and "dur" in e),
|
||||
key=lambda e: e["ts"],
|
||||
)
|
||||
if not acts:
|
||||
raise ValueError(
|
||||
"No GPU activity recorded. Another profiler (e.g. nsys) may hold "
|
||||
"CUPTI; don't combine --profile-dir with it."
|
||||
)
|
||||
if len(acts) % replays:
|
||||
raise ValueError(
|
||||
f"{len(acts)} GPU activities don't split into {replays} replays"
|
||||
)
|
||||
per_replay = len(acts) // replays
|
||||
if per_replay <= skip:
|
||||
raise ValueError(f"{per_replay} activities per replay, {skip} to skip")
|
||||
kernels: dict[str, dict[str, Any]] = defaultdict(
|
||||
lambda: {"total_us": 0.0, "calls": 0, "example": ""}
|
||||
)
|
||||
spans, busy, pre = [], [], []
|
||||
for r in range(replays):
|
||||
chunk = acts[r * per_replay : (r + 1) * per_replay]
|
||||
pre.append(sum(e["dur"] for e in chunk[:skip]))
|
||||
forward = chunk[skip:]
|
||||
spans.append(
|
||||
max(e["ts"] + e["dur"] for e in forward) - min(e["ts"] for e in forward)
|
||||
)
|
||||
busy.append(sum(e["dur"] for e in forward))
|
||||
for e in forward:
|
||||
name = short_name(e["name"]) if e["cat"] == "kernel" else e["name"]
|
||||
k = kernels[name]
|
||||
k["total_us"] += e["dur"]
|
||||
k["calls"] += 1
|
||||
k["example"] = k["example"] or e["name"][:300]
|
||||
total = sum(k["total_us"] for k in kernels.values())
|
||||
rows = [
|
||||
{
|
||||
"name": name,
|
||||
"us": round(k["total_us"] / replays, 2),
|
||||
"pct": round(100 * k["total_us"] / total, 2),
|
||||
"calls": k["calls"] // replays,
|
||||
"example": k["example"],
|
||||
}
|
||||
for name, k in kernels.items()
|
||||
]
|
||||
rows.sort(key=lambda row: -row["us"])
|
||||
return {
|
||||
"replays": replays,
|
||||
# First to last forward activity; busy can exceed it when streams overlap.
|
||||
"span_us": round(statistics.median(spans), 2),
|
||||
"busy_us": round(statistics.median(busy), 2),
|
||||
"pre_forward_us": round(statistics.median(pre), 2),
|
||||
"activities": per_replay - skip,
|
||||
"kernels": rows,
|
||||
}
|
||||
|
||||
|
||||
def profile_graph(
|
||||
graph: torch.cuda.CUDAGraph,
|
||||
name: str,
|
||||
prefix: str | None,
|
||||
skip: int,
|
||||
replays: int = 3,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Replay `replays` times between `cudaProfilerStart`/`Stop`, in an NVTX
|
||||
range named after the case, for `nsys --capture-range=cudaProfilerApi`.
|
||||
|
||||
With `prefix`, also trace them with `torch.profiler` and write
|
||||
`prefix.trace.json.gz` and `prefix.kernels.json`. Every rank must call
|
||||
this (replays may contain collectives).
|
||||
"""
|
||||
|
||||
def run() -> None:
|
||||
torch.cuda.profiler.start()
|
||||
with torch.cuda.nvtx.range(f"forward/{name}"):
|
||||
for _ in range(replays):
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.cuda.profiler.stop()
|
||||
|
||||
if prefix is None:
|
||||
run()
|
||||
return None
|
||||
activities = [torch.profiler.ProfilerActivity.CUDA]
|
||||
with torch.profiler.profile(activities=activities) as prof:
|
||||
run()
|
||||
trace = f"{prefix}.trace.json.gz"
|
||||
prof.export_chrome_trace(trace)
|
||||
with gzip.open(trace, "rt") as f:
|
||||
events = json.load(f)["traceEvents"]
|
||||
summary = {"trace": trace, **summarize(events, replays, skip)}
|
||||
with open(f"{prefix}.kernels.json", "w") as f:
|
||||
json.dump(summary, f, indent=1)
|
||||
return summary
|
||||
|
||||
|
||||
def format_summary(summary: dict[str, Any], top: int = 20) -> str:
|
||||
lines = [
|
||||
f"span {summary['span_us']:.1f} us, busy {summary['busy_us']:.1f} us, "
|
||||
f"{summary['activities']} activities/forward",
|
||||
f"{'us':>10} {'%':>6} {'calls':>6} kernel",
|
||||
]
|
||||
for row in summary["kernels"][:top]:
|
||||
lines.append(
|
||||
f"{row['us']:10.1f} {row['pct']:6.2f} {row['calls']:6d} {row['name']}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_diff(a: dict[str, Any], b: dict[str, Any], top: int = 20) -> str:
|
||||
rows_a = {row["name"]: row for row in a["kernels"]}
|
||||
rows_b = {row["name"]: row for row in b["kernels"]}
|
||||
deltas = []
|
||||
for name in rows_a.keys() | rows_b.keys():
|
||||
ra, rb = rows_a.get(name), rows_b.get(name)
|
||||
us_a, us_b = ra["us"] if ra else 0.0, rb["us"] if rb else 0.0
|
||||
calls_a, calls_b = ra["calls"] if ra else 0, rb["calls"] if rb else 0
|
||||
deltas.append((us_b - us_a, us_a, us_b, calls_a, calls_b, name))
|
||||
deltas.sort(key=lambda d: -abs(d[0]))
|
||||
lines = [
|
||||
f"span {a['span_us']:.1f} -> {b['span_us']:.1f} us "
|
||||
f"({b['span_us'] - a['span_us']:+.1f}), busy {a['busy_us']:.1f} -> "
|
||||
f"{b['busy_us']:.1f} us ({b['busy_us'] - a['busy_us']:+.1f})",
|
||||
f"{'delta us':>10} {'A us':>10} {'B us':>10} {'A n':>5} {'B n':>5} kernel",
|
||||
]
|
||||
for delta, us_a, us_b, calls_a, calls_b, name in deltas[:top]:
|
||||
lines.append(
|
||||
f"{delta:+10.1f} {us_a:10.1f} {us_b:10.1f} {calls_a:5d} {calls_b:5d} "
|
||||
f"{name}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("summaries", nargs="+", help="One or two *.kernels.json.")
|
||||
parser.add_argument("--top", type=int, default=20)
|
||||
args = parser.parse_args()
|
||||
if len(args.summaries) > 2:
|
||||
parser.error("Pass one summary, or two to compare.")
|
||||
loaded = []
|
||||
for path in args.summaries:
|
||||
with open(path) as f:
|
||||
loaded.append(json.load(f))
|
||||
if len(loaded) == 1:
|
||||
print(format_summary(loaded[0], args.top))
|
||||
else:
|
||||
print(format_diff(loaded[0], loaded[1], args.top))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -122,8 +122,22 @@ async def _align_prompts_to_server_tokenizer(
|
||||
|
||||
try:
|
||||
first_tokens = await _tokenize(input_requests[0].prompt)
|
||||
except Exception:
|
||||
print("WARNING: /tokenize unavailable, skipping alignment.")
|
||||
except aiohttp.ClientConnectionError as e:
|
||||
print(f"WARNING: {base_url} unreachable ({e!r}), skipping alignment.")
|
||||
return input_requests
|
||||
except aiohttp.ClientResponseError as e:
|
||||
if e.status == 404:
|
||||
hint = (
|
||||
"404 Not Found: either this server has no /tokenize route,"
|
||||
f" or it does not serve a model named `{model_id}`"
|
||||
" (its served names are listed by GET /v1/models)"
|
||||
)
|
||||
else:
|
||||
hint = f"HTTP {e.status}"
|
||||
print(f"WARNING: /tokenize unavailable ({hint}), skipping alignment.")
|
||||
return input_requests
|
||||
except Exception as e:
|
||||
print(f"WARNING: /tokenize probe failed ({e!r}), skipping alignment.")
|
||||
return input_requests
|
||||
|
||||
expected = input_requests[0].prompt_len
|
||||
|
||||
@@ -1063,6 +1063,28 @@ class VllmConfig:
|
||||
)
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
|
||||
|
||||
def _maybe_disable_dynamic_sd_for_v1(self) -> None:
|
||||
speculative_config = self.speculative_config
|
||||
# V2 truncates drafts centrally. V1 requires proposer support.
|
||||
if (
|
||||
speculative_config is None
|
||||
or not speculative_config.uses_dynamic_speculative_decoding()
|
||||
or self.use_v2_model_runner
|
||||
or speculative_config.method
|
||||
in ("ngram", "eagle", "eagle3", "mtp", "draft_model", "dflash")
|
||||
):
|
||||
return
|
||||
|
||||
logger.warning_once(
|
||||
"Dynamic speculative decoding is not supported with the '%s' "
|
||||
"speculative method on Model Runner V1. "
|
||||
"Disabling num_speculative_tokens_per_batch_size "
|
||||
"and falling back to static num_speculative_tokens=%d.",
|
||||
speculative_config.method,
|
||||
speculative_config.num_speculative_tokens,
|
||||
)
|
||||
speculative_config.num_speculative_tokens_per_batch_size = None
|
||||
|
||||
def _maybe_disable_dynamic_sd_for_data_parallel(self) -> None:
|
||||
speculative_config = self.speculative_config
|
||||
if (
|
||||
@@ -1836,6 +1858,7 @@ class VllmConfig:
|
||||
)
|
||||
|
||||
self._maybe_disable_dynamic_sd_for_data_parallel()
|
||||
self._maybe_disable_dynamic_sd_for_v1()
|
||||
self._maybe_override_dynamic_sd_cudagraph_mode()
|
||||
|
||||
if (
|
||||
|
||||
@@ -156,10 +156,12 @@ def _resolve_fi_ar_backend(group: ProcessGroup) -> tuple[str | None, bool]:
|
||||
|
||||
node_count = _node_count(group)
|
||||
if node_count == 1:
|
||||
# TRTLLM supports single-node allreduce on Hopper and Blackwell. Do not
|
||||
# create an MNNVL workspace when the process group is node-local.
|
||||
backend = "trtllm"
|
||||
allow_trtllm_fallback = False
|
||||
# MNNVL decodes faster than TRTLLM on single-node NVSwitch systems
|
||||
# (about 3-5% throughput at TP4/TP8 on H200 and B200). Fall back to
|
||||
# TRTLLM where the node lacks NVSwitch multicast and MNNVL cannot
|
||||
# create its workspace.
|
||||
backend = "mnnvl"
|
||||
allow_trtllm_fallback = True
|
||||
elif current_platform.has_device_capability(100):
|
||||
# MNNVL requires Blackwell-class GPUs and is the only supported
|
||||
# FlashInfer backend for a multi-node allreduce process group.
|
||||
|
||||
@@ -354,6 +354,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
# Per-rank expert_id -> physical row; set by set_transfer_context.
|
||||
self._expert_to_src_row: list[dict[int, int]] | None = None
|
||||
self._layer_idx: int | None = None
|
||||
self.set_stream(None)
|
||||
|
||||
nixl_agent_config = nixl_utils.nixl_agent_config
|
||||
config = (
|
||||
@@ -403,7 +404,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
return f"eplb-{self._rank}{pp_suffix}-{uid}"
|
||||
|
||||
def set_stream(self, stream: torch.Stream | None) -> None:
|
||||
pass
|
||||
self._stream = stream
|
||||
|
||||
def add_send(
|
||||
self,
|
||||
@@ -421,6 +422,13 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
f"pending transfers from layer {self._layer_idx}; "
|
||||
f"execute() was not called after previous add_recv() calls"
|
||||
)
|
||||
# NIXL READs bypass device streams; wait for pending copies out of
|
||||
# the receive buffer before add_recv overwrites it.
|
||||
if current_platform.is_rocm():
|
||||
with gpu_sync_allowed():
|
||||
torch.accelerator.current_stream().synchronize()
|
||||
if self._stream is not None:
|
||||
self._stream.synchronize()
|
||||
self._layer_idx = layer_idx
|
||||
n = self._num_local_experts
|
||||
rank_experts = old_indices[: self._world_size * n].reshape(self._world_size, n)
|
||||
|
||||
@@ -748,11 +748,9 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""Drain HB / completion notifs forwarded by the writer thread.
|
||||
|
||||
The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG
|
||||
notifs are handled there. Everything else is forwarded here for
|
||||
existing accounting.
|
||||
notifs are handled there. P learns of finished pushes from its WRITE
|
||||
handles, so this never reports finished sends.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
notified_req_ids: set[str] = set()
|
||||
while True:
|
||||
try:
|
||||
notif = self._pending_completion_notifs.get_nowait()
|
||||
@@ -766,46 +764,27 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
req_id, tp_size = msg.rsplit(":", 1)
|
||||
|
||||
# Not tracked as a P-side send/process for this notif.
|
||||
if req_id not in self._reqs_to_send and req_id not in self._reqs_to_process:
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None:
|
||||
# Consumer waits for one notif per producer rank writing
|
||||
# here: pp_size stages * producers-per-consumer (>1 when
|
||||
# producer TP > consumer TP; tp_size is the producer TP).
|
||||
producers_per_consumer = max(1, int(tp_size) // self.world_size)
|
||||
expected_notifs = meta.pp_size * producers_per_consumer
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
notifs = self.consumer_notification_counts_by_req[req_id]
|
||||
if notifs < expected_notifs:
|
||||
continue
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
# P drove the transfer (we own no NIXL handle), so
|
||||
# materialise an empty ``_recving_transfers`` entry for
|
||||
# ``_pop_done_transfers`` to report done.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
else:
|
||||
# Not tracked on either side (lease may have expired
|
||||
# before the notif arrived). Log and skip.
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Unrecognized request %s notif (may have expired).",
|
||||
req_id,
|
||||
)
|
||||
if (meta := self._recving_metadata.get(req_id)) is None:
|
||||
# The lease may have expired before the notif arrived.
|
||||
self.xfer_stats.record_notification_after_expiry()
|
||||
logger.error(
|
||||
"Unrecognized request %s notif (may have expired).", req_id
|
||||
)
|
||||
continue
|
||||
|
||||
n_consumers = int(tp_size)
|
||||
tp_ratio = self.transfer_topo.tp_ratio(n_consumers)
|
||||
consumers_per_producer = -tp_ratio if n_consumers > self.world_size else 1
|
||||
# Consumer waits for one notif per producer rank writing
|
||||
# here: pp_size stages * producers-per-consumer (>1 when
|
||||
# producer TP > consumer TP; tp_size is the producer TP).
|
||||
producers_per_consumer = max(1, int(tp_size) // self.world_size)
|
||||
expected_notifs = meta.pp_size * producers_per_consumer
|
||||
self.consumer_notification_counts_by_req[req_id] += 1
|
||||
if (
|
||||
self.consumer_notification_counts_by_req[req_id]
|
||||
== consumers_per_producer
|
||||
):
|
||||
notified_req_ids.add(req_id)
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
self._reqs_to_process.remove(req_id)
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
return notified_req_ids
|
||||
if self.consumer_notification_counts_by_req[req_id] < expected_notifs:
|
||||
continue
|
||||
del self.consumer_notification_counts_by_req[req_id]
|
||||
# P drove the transfer (we own no NIXL handle), so
|
||||
# materialise an empty ``_recving_transfers`` entry for
|
||||
# ``_pop_done_transfers`` to report done.
|
||||
self._recving_transfers.setdefault(req_id, [])
|
||||
return set()
|
||||
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
# Engine main thread asking for completions: also wake the writer
|
||||
@@ -835,7 +814,6 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
for req_id in done_pushing:
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
self._reqs_to_process.discard(req_id)
|
||||
self.consumer_notification_counts_by_req.pop(req_id, None)
|
||||
done_sending.add(req_id)
|
||||
|
||||
# Tell the writer to drop any state it still holds for any
|
||||
|
||||
@@ -236,10 +236,9 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
)
|
||||
if use_eagle_block_drop and not eagle_groups:
|
||||
# No group is annotated as holding drafter layers. This happens
|
||||
# for shared-group MTP models (e.g. Qwen3.5-style), whose drafter
|
||||
# is a regular decoder layer merged into the target's
|
||||
# full-attention group, and for separate-draft models that miss
|
||||
# annotation. Treating every group as a drafter group here makes
|
||||
# when the drafter owns no KV cache (e.g. KV-sharing drafters) or
|
||||
# the model runner does not tag draft layers' KV cache specs.
|
||||
# Treating every group as a drafter group here makes
|
||||
# the volatile-tail pop apply to ALL groups, which can zero the
|
||||
# whole request's offload hit whenever any group's servable
|
||||
# window is a single chunk (issue #52735). Drafter KV served one
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
import math
|
||||
from collections.abc import Callable, Sequence
|
||||
|
||||
import torch
|
||||
@@ -104,14 +105,6 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
structured_output_backend: StructuredOutputBackend | None = None
|
||||
structured_output_key = None
|
||||
structured_output_bitmask = None
|
||||
if params.structured_outputs is not None:
|
||||
(
|
||||
structured_output_backend,
|
||||
structured_output_key,
|
||||
structured_output_bitmask,
|
||||
) = self._init_beam_search_structured_output(
|
||||
params.structured_outputs, tokenizer
|
||||
)
|
||||
|
||||
# generate 2 * beam_width candidates at each step
|
||||
# following the huggingface transformers implementation
|
||||
@@ -124,6 +117,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
detokenize=False,
|
||||
skip_clone=True, # Internal beam search, safe to skip clone
|
||||
)
|
||||
params.configure_logprobs(
|
||||
base_sampling_params, self.llm_engine.input_processor.vllm_config
|
||||
)
|
||||
if params.structured_outputs is not None:
|
||||
(
|
||||
structured_output_backend,
|
||||
structured_output_key,
|
||||
structured_output_bitmask,
|
||||
) = self._init_beam_search_structured_output(
|
||||
params.structured_outputs, tokenizer
|
||||
)
|
||||
|
||||
instances: list[BeamSearchInstance] = []
|
||||
|
||||
for lora_req, prompt in zip(lora_requests, engine_inputs):
|
||||
@@ -287,7 +292,12 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
# tokens outside the grammar's allowed set. This filtering is also
|
||||
# the only grammar enforcement for beams whose allowed set exceeds
|
||||
# the engine-side allowed_token_ids cap.
|
||||
allowed_sets: list[set[int] | None] = [None] * len(all_beams)
|
||||
request_allowed = (
|
||||
set(base_sampling_params.logprob_token_ids)
|
||||
if base_sampling_params.logprob_token_ids is not None
|
||||
else None
|
||||
)
|
||||
allowed_sets: list[set[int] | None] = [request_allowed] * len(all_beams)
|
||||
if structured_output_backend is not None:
|
||||
for i, entry in enumerate(beam_entries):
|
||||
if entry is not None:
|
||||
@@ -316,6 +326,18 @@ class BeamSearchOfflineMixin(OfflineInferenceMixin):
|
||||
|
||||
if result.outputs[0].logprobs is not None:
|
||||
logprobs = result.outputs[0].logprobs[0]
|
||||
if request_allowed is not None:
|
||||
logprobs = {
|
||||
token: score
|
||||
for token, score in sorted(
|
||||
logprobs.items(),
|
||||
key=lambda item: item[1].logprob,
|
||||
reverse=True,
|
||||
)
|
||||
if token in request_allowed and math.isfinite(score.logprob)
|
||||
}
|
||||
for score in logprobs.values():
|
||||
score.rank = None
|
||||
allowed = allowed_sets[i]
|
||||
for token_id, logprob_obj in logprobs.items():
|
||||
if allowed is not None and token_id not in allowed:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
from abc import ABC
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
|
||||
@@ -16,7 +17,10 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import collect_from_async_generator
|
||||
|
||||
from .utils import BeamSearchSequence, create_sort_beams_key_function
|
||||
from .utils import (
|
||||
BeamSearchSequence,
|
||||
create_sort_beams_key_function,
|
||||
)
|
||||
|
||||
|
||||
class BeamSearchOnlineMixin(ABC):
|
||||
@@ -66,6 +70,10 @@ class BeamSearchOnlineMixin(ABC):
|
||||
watermarking=False,
|
||||
detokenize=False,
|
||||
)
|
||||
params.configure_logprobs(
|
||||
sampling_params, self.engine_client.input_processor.vllm_config
|
||||
)
|
||||
allowed = set(params.allowed_token_ids) if params.allowed_token_ids else None
|
||||
all_beams = [
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
@@ -135,6 +143,18 @@ class BeamSearchOnlineMixin(ABC):
|
||||
|
||||
if result.outputs[0].logprobs is not None:
|
||||
logprobs = result.outputs[0].logprobs[0]
|
||||
if allowed is not None:
|
||||
logprobs = {
|
||||
token: score
|
||||
for token, score in sorted(
|
||||
logprobs.items(),
|
||||
key=lambda item: item[1].logprob,
|
||||
reverse=True,
|
||||
)
|
||||
if token in allowed and math.isfinite(score.logprob)
|
||||
}
|
||||
for score in logprobs.values():
|
||||
score.rank = None
|
||||
for token_id, logprob_obj in logprobs.items():
|
||||
candidate_logprob = (
|
||||
current_beam.cum_logprob + logprob_obj.logprob
|
||||
|
||||
@@ -710,6 +710,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
|
||||
return BeamSearchParams(
|
||||
beam_width=n,
|
||||
allowed_token_ids=self.allowed_token_ids,
|
||||
max_tokens=max_tokens,
|
||||
ignore_eos=self.ignore_eos,
|
||||
temperature=temperature,
|
||||
|
||||
@@ -334,6 +334,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
|
||||
return BeamSearchParams(
|
||||
beam_width=n,
|
||||
allowed_token_ids=self.allowed_token_ids,
|
||||
max_tokens=max_tokens,
|
||||
ignore_eos=self.ignore_eos,
|
||||
temperature=temperature,
|
||||
|
||||
@@ -146,7 +146,6 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_AITER_MLA_ASM_PADDING: Literal["auto", "gluon", "asm"] = "auto"
|
||||
VLLM_ROCM_AITER_MLA_DCP_VERIFY: Literal["auto", "asm", "segmented"] = "auto"
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
VLLM_ROCM_USE_AITER_FP4_ASM_GEMM: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA: bool = False
|
||||
VLLM_ROCM_USE_AITER_TRITON_ROPE: bool = False
|
||||
VLLM_ROCM_USE_AITER_FP8BMM: bool = False
|
||||
@@ -1346,11 +1345,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MHA": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MHA", "True").lower() in ("true", "1")
|
||||
),
|
||||
# Whether to use aiter fp4 gemm asm.
|
||||
# By default is disabled.
|
||||
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_FP4_ASM_GEMM", "False").lower() in ("true", "1")
|
||||
),
|
||||
# Whether sparse MLA prefill and decode run on aiter's Triton kernel, which
|
||||
# reads the KV cache as stored (bf16 or fp8, paged or flat). Used by the
|
||||
# ROCM_AITER_MLA_SPARSE backend (DeepSeek V3.2, GLM-5.x) and DeepSeek
|
||||
|
||||
@@ -1424,6 +1424,9 @@ def override_envs_for_invariance():
|
||||
|
||||
# torch.compile settings
|
||||
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
|
||||
import torch._inductor.config as inductor_config
|
||||
|
||||
inductor_config.batch_invariant = True
|
||||
|
||||
|
||||
def init_batch_invariance():
|
||||
|
||||
@@ -159,7 +159,8 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
|
||||
def __init__(self, config: MxFp4LinearLayerConfig) -> None:
|
||||
super().__init__(config)
|
||||
self.use_asm_gemm = rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled()
|
||||
self.asm_gemm_supported = rocm_aiter_ops.is_asm_fp4_gemm_supported()
|
||||
self.use_asm_gemm = False
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
|
||||
@classmethod
|
||||
@@ -199,6 +200,13 @@ class AiterMxfp4LinearKernel(MxFp4LinearKernel):
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Use preshuffled ASM (P) and Triton (D) GEMM when tuned in aiter
|
||||
if self.asm_gemm_supported:
|
||||
n, k_bytes = layer.weight.shape
|
||||
self.use_asm_gemm = rocm_aiter_ops.is_asm_fp4_gemm_tuned(
|
||||
n, k_bytes
|
||||
) and rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(n, k_bytes)
|
||||
|
||||
if self.use_asm_gemm and not _asm_fp4_scale_swizzle_supported(
|
||||
layer.weight_scale.data
|
||||
):
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16).
|
||||
"""LoRA-aware FlashInfer TRT-LLM MoE experts (BF16, MXFP4 x BF16).
|
||||
|
||||
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153:
|
||||
Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153
|
||||
(BF16) and PR #3987 (FP4 block-scale):
|
||||
|
||||
* The W13 (gate_up) LoRA delta is passed directly as ``gemm1_lora_delta`` to
|
||||
the routed kernel, which fuses it into FC1 before SwiGLU (BiasType::Mn).
|
||||
@@ -13,11 +14,12 @@ Reuses the routed API + ``gemm1_lora_delta`` path from FlashInfer PR #3153:
|
||||
|
||||
Constraints (matching the PR support matrix; final gating lives in the oracle):
|
||||
* SM100+ (Blackwell), gated SwiGLU, shuffled weights only;
|
||||
* BF16 only;
|
||||
* BF16 weights, or MXFP4 weights with BF16 activations;
|
||||
* routing must be computed outside the MoE (the Modular path satisfies this).
|
||||
|
||||
"""
|
||||
|
||||
import functools
|
||||
from abc import abstractmethod
|
||||
|
||||
import torch
|
||||
@@ -37,9 +39,17 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
view_as_block_major_k,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
||||
TrtLlmMxfp4ExpertsBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
@@ -224,6 +234,15 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def prepare_gemm1_lora_delta(
|
||||
self,
|
||||
gemm1_lora_delta: torch.Tensor,
|
||||
lora_context: MoELoRAContext,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Hook to rewrite the W13 delta before it is handed to the kernel."""
|
||||
return gemm1_lora_delta
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
@@ -247,7 +266,8 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
num_tokens = hidden_states.size(0)
|
||||
top_k = self.topk
|
||||
intermediate_size = self.intermediate_size_per_partition
|
||||
K = output.size(1)
|
||||
# LoRA buffers use the padded hidden size; `output` may be narrower.
|
||||
K = self.hidden_dim
|
||||
|
||||
# ---- Base-model fast path ----
|
||||
# When no token in the batch selects a LoRA adapter, skip the LoRA machinery
|
||||
@@ -288,8 +308,7 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
w13_meta = (None, None, None, None)
|
||||
|
||||
# zeros (not empty): under EP the punica expand kernel only writes
|
||||
# slots whose expert is local to this rank; non-local (token, top_k)
|
||||
# slots must stay 0 so they contribute no bias when fed to flashinfer.
|
||||
# slots whose expert is local to this rank; the kernel skips the rest.
|
||||
gemm1_lora_delta = torch.zeros(
|
||||
num_tokens,
|
||||
top_k,
|
||||
@@ -326,6 +345,9 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
add_inputs=False,
|
||||
swap_w13_slices=True,
|
||||
)
|
||||
gemm1_lora_delta = self.prepare_gemm1_lora_delta(
|
||||
gemm1_lora_delta, lora_context, topk_ids
|
||||
)
|
||||
|
||||
# ---- 2) Call the routed flashinfer kernel ----
|
||||
ret = self.invoke_routed_moe(
|
||||
@@ -466,7 +488,9 @@ class _TrtLlmLoRAExpertsBase(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
(``expert_weights`` in expanded order, ``idx_map < 0`` dropped), scale by
|
||||
``scale``, and add the already-weighted ``w2_delta`` reduced over top_k.
|
||||
"""
|
||||
K = gemm2_permuted.size(1)
|
||||
# Inputs may carry hidden padding that `output` does not.
|
||||
K = output.size(1)
|
||||
assert gemm2_permuted.size(1) >= K and w2_delta.size(2) >= K
|
||||
BLOCK_K = 512
|
||||
grid = (num_tokens, triton.cdiv(K, BLOCK_K))
|
||||
_finalize_lora_kernel[grid](
|
||||
@@ -550,3 +574,158 @@ class TrtLlmBf16LoRAExperts(_TrtLlmLoRAExpertsBase):
|
||||
return list(ret)
|
||||
# do_finalize=True finalized directly into `output`.
|
||||
return [output]
|
||||
|
||||
|
||||
# MXFP4 weights x BF16 activations trtllm MoE + LoRA
|
||||
class TrtLlmMxfp4LoRAExperts(_TrtLlmLoRAExpertsBase, TrtLlmMxfp4ExpertsBase):
|
||||
"""MXFP4-weight, BF16-activation trtllm MoE + LoRA. Not MXFP8 activations:
|
||||
the kernel then returns the FC1 activation quantized, without its scales."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
_TrtLlmLoRAExpertsBase.__init__(self, moe_config, quant_config)
|
||||
# gemm1_alpha / gemm1_beta / gemm1_clamp_limit, hidden_dim_unpadded.
|
||||
TrtLlmMxfp4ExpertsBase.__init__(self, moe_config, quant_config)
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return weight_key == kMxfp4Static and activation_key is None
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in (MoEActivation.SWIGLUOAI, MoEActivation.SILU)
|
||||
|
||||
@staticmethod
|
||||
def _supports_routing_method(routing_method, weight_key, activation_key) -> bool:
|
||||
return True # routing is done outside the kernel
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
# DP gathers tokens, but the no-LoRA fast path reads the local batch.
|
||||
p = moe_parallel_config
|
||||
return p.dp_size == 1 and not p.use_all2all_kernels and not p.enable_eplb
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
topk,
|
||||
global_num_experts,
|
||||
local_num_experts,
|
||||
expert_tokens_meta,
|
||||
activation,
|
||||
):
|
||||
# flashinfer manages its own workspace and drops the hidden padding.
|
||||
return (0,), (0,), (M, self.hidden_dim_unpadded)
|
||||
|
||||
def output_hidden_dim(self, hidden_dim: int) -> int:
|
||||
# FusedMoEExpertsModular precedes TrtLlmMxfp4ExpertsBase in the MRO.
|
||||
return self.hidden_dim_unpadded
|
||||
|
||||
@functools.cached_property
|
||||
def _w13_bias_up_gate_index(self) -> torch.Tensor:
|
||||
"""Columns of the kernel-format FC1 bias in [up | gate] order."""
|
||||
from flashinfer.fused_moe.core import get_w2_permute_indices_with_cache
|
||||
|
||||
# Kernel format is the (up_i, gate_i) bias permuted for the tile.
|
||||
assert self.w1_bias is not None
|
||||
bias = self.w1_bias.reshape(self.local_num_experts, -1)[0]
|
||||
permute = get_w2_permute_indices_with_cache({}, bias.reshape(-1, 1), 128)
|
||||
interleaved = torch.argsort(permute.to(bias.device))
|
||||
return torch.cat([interleaved[0::2], interleaved[1::2]])
|
||||
|
||||
def prepare_gemm1_lora_delta(
|
||||
self,
|
||||
gemm1_lora_delta: torch.Tensor,
|
||||
lora_context: MoELoRAContext,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Bring the W13 delta to [up | gate] halves and fold in the FC1 bias."""
|
||||
intermediate_size = gemm1_lora_delta.size(-1) // 2
|
||||
# View as (I, 2) pairs of the first and second w13 half. SWIGLUOAI
|
||||
# weights, and the deltas that follow them, are interleaved.
|
||||
if self.moe_config.activation == MoEActivation.SWIGLUOAI:
|
||||
pairs = gemm1_lora_delta.unflatten(-1, (intermediate_size, 2))
|
||||
else:
|
||||
pairs = gemm1_lora_delta.unflatten(-1, (2, intermediate_size)).mT
|
||||
# Per-expert deltas arrive as (up, gate); fused 3D ones follow the
|
||||
# checkpoint (gate, up) order.
|
||||
if lora_context.w13_num_slices != 2:
|
||||
pairs = pairs.flip(-1)
|
||||
delta = pairs.mT.flatten(-2) # [up | gate]
|
||||
|
||||
# The kernel has a single FC1 bias slot, shared by the static bias and
|
||||
# the per-token delta, so the static bias rides along in the delta.
|
||||
if self.w1_bias is not None:
|
||||
bias = self.w1_bias.reshape(self.local_num_experts, -1)
|
||||
bias = bias[:, self._w13_bias_up_gate_index].float()
|
||||
local_ids = topk_ids.long() - self.ep_rank * self.local_num_experts
|
||||
local_ids = local_ids.clamp_(0, self.local_num_experts - 1)
|
||||
delta = (delta.float() + bias[local_ids]).to(torch.bfloat16)
|
||||
return delta.contiguous()
|
||||
|
||||
def invoke_routed_moe(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_ids_and_weights: tuple[torch.Tensor, torch.Tensor],
|
||||
gemm1_lora_delta: torch.Tensor | None,
|
||||
global_num_experts: int,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
output: torch.Tensor,
|
||||
) -> list[torch.Tensor]:
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
assert a1q_scale is None and hidden_states.dtype == torch.bfloat16
|
||||
topk_ids, topk_weights = topk_ids_and_weights
|
||||
# See TrtLlmBf16LoRAExperts.invoke_routed_moe for the return contract.
|
||||
do_finalize = gemm1_lora_delta is None
|
||||
ret = trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=(topk_ids.to(torch.int32), topk_weights),
|
||||
routing_bias=None,
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_scale=None,
|
||||
gemm1_weights=w1,
|
||||
gemm1_weights_scale=self.w1_scale,
|
||||
# With a delta the bias is folded into it (prepare_gemm1_lora_delta).
|
||||
gemm1_bias=self.w1_bias if do_finalize else None,
|
||||
gemm1_alpha=self.gemm1_alpha,
|
||||
gemm1_beta=self.gemm1_beta,
|
||||
gemm1_clamp_limit=self.gemm1_clamp_limit,
|
||||
gemm2_weights=w2,
|
||||
gemm2_weights_scale=self.w2_scale,
|
||||
gemm2_bias=self.w2_bias,
|
||||
output1_scale_scalar=None,
|
||||
output1_scale_gate_scalar=None,
|
||||
output2_scale_scalar=None,
|
||||
num_experts=global_num_experts,
|
||||
top_k=self.topk,
|
||||
n_group=None,
|
||||
topk_group=None,
|
||||
intermediate_size=self.intermediate_size_per_partition,
|
||||
local_expert_offset=self.ep_rank * self.local_num_experts,
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
# Routing is already done; Renormalize is a safe default.
|
||||
routing_method_type=RoutingMethodType.Renormalize,
|
||||
do_finalize=do_finalize,
|
||||
enable_pdl=True,
|
||||
activation_type=self._flashinfer_activation_type(
|
||||
self.moe_config.activation
|
||||
),
|
||||
output=output if do_finalize else None,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
gemm1_lora_delta=gemm1_lora_delta,
|
||||
)
|
||||
return [output] if do_finalize else list(ret)
|
||||
|
||||
@@ -122,6 +122,9 @@ class TrtLlmMxfp4ExpertsBase:
|
||||
p = current_platform
|
||||
return p.is_cuda() and p.is_device_capability_family(100) and has_flashinfer()
|
||||
|
||||
def output_hidden_dim(self, hidden_dim: int) -> int:
|
||||
return self.hidden_dim_unpadded
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@@ -843,6 +843,11 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
"""Workspace type: The dtype to use for the workspace tensors."""
|
||||
return act_dtype
|
||||
|
||||
def output_hidden_dim(self, hidden_dim: int) -> int:
|
||||
"""Hidden size of the finalized output. Experts that drop the hidden
|
||||
padding return the unpadded size."""
|
||||
return hidden_dim
|
||||
|
||||
@abstractmethod
|
||||
def workspace_shapes(
|
||||
self,
|
||||
@@ -1321,7 +1326,8 @@ class FusedMoEKernelModularImpl:
|
||||
# low-latency kernels are always batched and can never run into
|
||||
# the tensor.numel() == 0 case.
|
||||
if M_full == 0:
|
||||
return torch.empty_like(a1q, dtype=in_dtype)
|
||||
out_dim = self.fused_experts.output_hidden_dim(a1q.size(-1))
|
||||
return a1q.new_empty((*a1q.shape[:-1], out_dim), dtype=in_dtype)
|
||||
|
||||
workspace13, workspace2, fused_out, fused_out_is_specialized = (
|
||||
self._allocate_buffers(
|
||||
@@ -1505,7 +1511,12 @@ class FusedMoEKernelModularImpl:
|
||||
a fused consumer.
|
||||
|
||||
"""
|
||||
output = torch.empty_like(hidden_states)
|
||||
output = hidden_states.new_empty(
|
||||
(
|
||||
*hidden_states.shape[:-1],
|
||||
self.fused_experts.output_hidden_dim(hidden_states.size(-1)),
|
||||
)
|
||||
)
|
||||
|
||||
local_num_experts = w1.shape[0]
|
||||
if global_num_experts == -1:
|
||||
|
||||
@@ -535,6 +535,30 @@ def _make_log_unsupported(backend: Mxfp4MoeBackend, reason: str | None) -> str:
|
||||
return f"{base} since {reason}." if reason else f"{base}."
|
||||
|
||||
|
||||
def _requested_kernel_cls(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
) -> list[type[mk.FusedMoEExperts]]:
|
||||
"""Kernel classes for an explicitly requested backend. The LoRA-aware
|
||||
TRT-LLM experts are opt-in: "auto" keeps its current LoRA selection."""
|
||||
candidates = backend_to_kernel_cls(backend)
|
||||
if not config.is_lora_enabled or backend not in TRTLLM_BACKENDS:
|
||||
return candidates
|
||||
if backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8:
|
||||
raise ValueError(
|
||||
_make_log_unsupported(
|
||||
backend,
|
||||
"expert LoRA is not supported with MXFP8 activations; use "
|
||||
"moe_backend='flashinfer_trtllm'",
|
||||
)
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_lora_moe import (
|
||||
TrtLlmMxfp4LoRAExperts,
|
||||
)
|
||||
|
||||
return [TrtLlmMxfp4LoRAExperts, *candidates]
|
||||
|
||||
|
||||
def _return_or_raise(
|
||||
backend: Mxfp4MoeBackend,
|
||||
config: FusedMoEConfig,
|
||||
@@ -544,7 +568,7 @@ def _return_or_raise(
|
||||
scope: Literal["process", "global", "local"] = "local",
|
||||
) -> tuple[Mxfp4MoeBackend, type[mk.FusedMoEExperts]]:
|
||||
reason: str | None = None
|
||||
for k_cls in backend_to_kernel_cls(backend):
|
||||
for k_cls in _requested_kernel_cls(backend, config):
|
||||
supported, reason = k_cls.is_supported_config(
|
||||
k_cls, config, weight_key, activation_key, activation_format
|
||||
)
|
||||
|
||||
@@ -1663,6 +1663,13 @@ class Gemma4ForConditionalGeneration(
|
||||
batch_size, num_patches = src.shape[0], src.shape[1]
|
||||
dst[:batch_size, :num_patches].copy_(src)
|
||||
|
||||
configured_max_tokens = self.config.vision_config.default_output_length
|
||||
# One graph per row width (soft tokens of one image or video frame),
|
||||
# so a budget's rows are not all padded to the widest image.
|
||||
row_widths = tuple(
|
||||
t for t in _SUPPORTED_SOFT_TOKENS if t <= configured_max_tokens
|
||||
)
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image", "video"],
|
||||
buffer_keys=[
|
||||
@@ -1676,6 +1683,7 @@ class Gemma4ForConditionalGeneration(
|
||||
"pixel_values": pad_pixel_values,
|
||||
"pixel_position_ids": pad_pixel_position_ids,
|
||||
},
|
||||
capture_axes=(row_widths,),
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
@@ -1712,10 +1720,15 @@ class Gemma4ForConditionalGeneration(
|
||||
if modality == "image":
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
if isinstance(pixel_values, list):
|
||||
# A batch pads its images to the widest one, so pack by that.
|
||||
row_tokens = (
|
||||
max((pv.shape[0] for pv in pixel_values), default=0) // pool_ratio
|
||||
)
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=pv.shape[0],
|
||||
output_tokens=pv.shape[0] // pool_ratio,
|
||||
path_output_tokens={"default": row_tokens},
|
||||
)
|
||||
for pv in pixel_values
|
||||
]
|
||||
@@ -1750,10 +1763,25 @@ class Gemma4ForConditionalGeneration(
|
||||
mm_kwargs: dict[str, Any],
|
||||
indices: list[int],
|
||||
) -> dict[str, Any]:
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG,
|
||||
)
|
||||
|
||||
vision_cfg = self.vision_tower.config
|
||||
pool_ratio = getattr(vision_cfg, "pooling_kernel_size", 2) ** 2
|
||||
|
||||
modality = self.get_input_modality(mm_kwargs)
|
||||
if modality == "image":
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
pixel_position_ids = mm_kwargs["pixel_position_ids"]
|
||||
if isinstance(pixel_values, list):
|
||||
row_patches = max(
|
||||
(pixel_values[i].shape[0] for i in indices), default=0
|
||||
)
|
||||
else:
|
||||
row_patches = pixel_values.shape[1]
|
||||
axis_keys = (row_patches // pool_ratio,)
|
||||
|
||||
if len(indices) == 0:
|
||||
is_pv_list = isinstance(pixel_values, list)
|
||||
is_pp_list = isinstance(pixel_position_ids, list)
|
||||
@@ -1762,20 +1790,24 @@ class Gemma4ForConditionalGeneration(
|
||||
"pixel_position_ids": (
|
||||
[] if is_pp_list else pixel_position_ids[:0]
|
||||
),
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
|
||||
}
|
||||
if isinstance(pixel_values, list):
|
||||
return {
|
||||
"pixel_values": [pixel_values[i] for i in indices],
|
||||
"pixel_position_ids": [pixel_position_ids[i] for i in indices],
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
|
||||
}
|
||||
return {
|
||||
"pixel_values": pixel_values[indices],
|
||||
"pixel_position_ids": pixel_position_ids[indices],
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
|
||||
}
|
||||
elif modality == "video":
|
||||
pixel_values_videos = mm_kwargs["pixel_values_videos"]
|
||||
pixel_position_ids_videos = mm_kwargs["pixel_position_ids_videos"]
|
||||
video_frame_counts = mm_kwargs["video_frame_counts"]
|
||||
axis_keys = (pixel_values_videos.shape[1] // pool_ratio,)
|
||||
|
||||
if len(indices) == 0:
|
||||
is_fc_tensor = isinstance(video_frame_counts, torch.Tensor)
|
||||
@@ -1785,6 +1817,7 @@ class Gemma4ForConditionalGeneration(
|
||||
"video_frame_counts": (
|
||||
video_frame_counts[:0] if is_fc_tensor else []
|
||||
),
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
|
||||
}
|
||||
|
||||
fc_list = (
|
||||
@@ -1819,6 +1852,7 @@ class Gemma4ForConditionalGeneration(
|
||||
"pixel_values_videos": selected_pv,
|
||||
"pixel_position_ids_videos": selected_pp,
|
||||
"video_frame_counts": selected_fc,
|
||||
ENCODER_CUDAGRAPH_AXIS_KEYS_KWARG: axis_keys,
|
||||
}
|
||||
raise ValueError(f"Unknown modality: {modality}")
|
||||
|
||||
@@ -1839,20 +1873,21 @@ class Gemma4ForConditionalGeneration(
|
||||
|
||||
dtype = dtype or torch.float32
|
||||
|
||||
max_size = max(max_batch_size, max_frames_per_batch)
|
||||
assert axis_keys is not None
|
||||
(per_item_output,) = axis_keys
|
||||
assert isinstance(per_item_output, int)
|
||||
|
||||
# Every row of a batch replayed on this graph is an image or video
|
||||
# frame of per_item_output tokens, so a batch that fits token_budget
|
||||
# never needs more rows than this. Capture needs at least one row.
|
||||
max_size = min(
|
||||
max(max_batch_size, max_frames_per_batch),
|
||||
max(1, token_budget // per_item_output),
|
||||
)
|
||||
|
||||
vision_cfg = self.vision_tower.config
|
||||
pool_ratio = getattr(vision_cfg, "pooling_kernel_size", 2) ** 2
|
||||
|
||||
# Retrieve the model's actual configured maximum tokens:
|
||||
configured_max_tokens = getattr(
|
||||
self.config.vision_config,
|
||||
"num_soft_tokens",
|
||||
_SUPPORTED_SOFT_TOKENS[2],
|
||||
)
|
||||
# Dynamically compute the slot capacity per item bounded by both the
|
||||
# current graph budget and the user's maximum config:
|
||||
per_item_output = min(token_budget, configured_max_tokens)
|
||||
# Satisfy k^2 * per_item_output = per_item_patches
|
||||
per_item_patches = per_item_output * pool_ratio
|
||||
|
||||
|
||||
@@ -79,6 +79,24 @@ def _make_deepseek_v4_vl_weights_mapper(
|
||||
)
|
||||
|
||||
|
||||
def stream_language_model_first(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Yield ``language_model.*`` weights as they arrive, then the rest.
|
||||
|
||||
Keeps the language model's weights in one contiguous group without
|
||||
materializing the stream. Deferred weights are cloned so they do not keep
|
||||
the loader's per-shard storage alive until the end of the load.
|
||||
"""
|
||||
rest = []
|
||||
for name, tensor in weights:
|
||||
if name.startswith("language_model."):
|
||||
yield name, tensor
|
||||
else:
|
||||
rest.append((name, tensor.clone()))
|
||||
yield from rest
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
DeepseekV4VLMultiModalProcessor,
|
||||
info=DeepseekV4VLProcessingInfo,
|
||||
@@ -334,7 +352,7 @@ class DeepseekV4ForConditionalGeneration(
|
||||
if child_finalizes:
|
||||
# A child which finalizes inside load_weights must see all of its
|
||||
# weights in one contiguous delegation from AutoWeightsLoader.
|
||||
mapped = iter(sorted(mapped, key=lambda x: x[0]))
|
||||
mapped = stream_language_model_first(mapped)
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
self._weights_finalized = child_finalizes
|
||||
|
||||
@@ -41,6 +41,7 @@ from vllm.models.deepseek_v4.common.vision import (
|
||||
DeepseekV4ViT,
|
||||
run_dp_sharded_vision_tower,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.vl_model import stream_language_model_first
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
@@ -317,12 +318,9 @@ class DeepseekV41ForCausalLM(
|
||||
return self.language_model.get_mtp_target_hidden_states()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# Map HF names into this wrapper's namespace up front and sort, so
|
||||
# the "language_model." group reaches the child loader as one
|
||||
# contiguous block (AutoWeightsLoader delegates per contiguous group,
|
||||
# and the child's load_weights finalizes fused expert weights, which
|
||||
# must not run on a partially loaded model).
|
||||
mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0])
|
||||
# The child finalizes in load_weights, so its weights must reach it as
|
||||
# one contiguous group.
|
||||
mapped = stream_language_model_first(self.hf_to_vllm_mapper.apply(weights))
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded_params = loader.load_weights(mapped)
|
||||
# The child's load_weights already ran its post-load finalization.
|
||||
|
||||
@@ -75,7 +75,7 @@ from vllm.model_executor.layers.quantization import (
|
||||
resolve_quant_method,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.model_executor.utils import is_weights_pre_processed, replace_parameter
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.kimi_k3.nvidia.low_latency_gemm import try_low_latency_gemm
|
||||
from vllm.models.kimi_k3.nvidia.ops.fused_mla_key_concat_kv_cache import (
|
||||
@@ -440,18 +440,21 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
|
||||
projected into latent space by ``W_UK_T`` and the attention output is
|
||||
projected back to ``v`` by ``W_UV`` -- avoiding materializing full K/V.
|
||||
"""
|
||||
W_UK, W_UV = split_kv_b_proj(
|
||||
self.kv_b_proj,
|
||||
act_dtype,
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads,
|
||||
self.qk_nope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
# (L, N, V) -> (N, L, V)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# (L, N, P) -> (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
# The weight cache daemon exports the absorbed weights. Repeating the
|
||||
# absorption would mutate its shared CUDA allocation.
|
||||
if not (is_weights_pre_processed() and getattr(self, "W_UV", None) is not None):
|
||||
W_UK, W_UV = split_kv_b_proj(
|
||||
self.kv_b_proj,
|
||||
act_dtype,
|
||||
self.kv_lora_rank,
|
||||
self.num_local_heads,
|
||||
self.qk_nope_head_dim,
|
||||
self.v_head_dim,
|
||||
)
|
||||
# (L, N, V) -> (N, L, V)
|
||||
replace_parameter(self, "W_UV", W_UV.transpose(0, 1), prefer_copy=True)
|
||||
# (L, N, P) -> (N, P, L)
|
||||
replace_parameter(self, "W_UK_T", W_UK.permute(1, 2, 0), prefer_copy=True)
|
||||
|
||||
quant_method = (
|
||||
resolve_quant_method(self.quant_config, self, prefix=self.layer_name)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiMo's compact XML parameters contain verbatim string values."""
|
||||
|
||||
import functools
|
||||
import json
|
||||
from dataclasses import replace
|
||||
|
||||
@@ -10,6 +11,7 @@ from vllm.parser.qwen3 import (
|
||||
_PARTIAL_PARAM_RE,
|
||||
CHATML_TURN_BOUNDARIES,
|
||||
Qwen3Parser,
|
||||
_param_value_extender,
|
||||
qwen3_config,
|
||||
)
|
||||
|
||||
@@ -37,6 +39,9 @@ class MiMoParser(Qwen3Parser):
|
||||
turn_boundary_tokens=CHATML_TURN_BOUNDARIES,
|
||||
),
|
||||
arg_converter=_mimo_arg_converter,
|
||||
arg_value_extender=functools.partial(
|
||||
_param_value_extender, trim_newlines=False
|
||||
),
|
||||
),
|
||||
)
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
|
||||
@@ -25,6 +25,7 @@ import regex as re
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
@@ -86,6 +87,64 @@ def _qwen3_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
return json.dumps(params, ensure_ascii=False)
|
||||
|
||||
|
||||
# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener.
|
||||
_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s*=")
|
||||
|
||||
|
||||
def _param_value_end_window(text: str) -> str:
|
||||
"""Return the suffix of *text* that may still grow into a value end."""
|
||||
start = text.rfind("<")
|
||||
if start == -1:
|
||||
return ""
|
||||
tail = text[start:]
|
||||
return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else ""
|
||||
|
||||
|
||||
def _param_value_extender(
|
||||
raw_args: str, trim_newlines: bool
|
||||
) -> ArgValueExtender | None:
|
||||
"""Extend the open trailing ``<parameter=...>`` value of *raw_args*.
|
||||
|
||||
The extender streams exactly what the converter would for the same text.
|
||||
With *trim_newlines* it follows ``_trim_wrapping_newlines``: it drops a
|
||||
leading newline of the value and holds back a trailing one.
|
||||
"""
|
||||
match = _PARTIAL_PARAM_RE.search(_PARAM_RE.sub("", raw_args))
|
||||
if match is None:
|
||||
return None
|
||||
name, value = match.group(1), match.group(2)
|
||||
# A repeated key keeps its first position, so its value is not last.
|
||||
if any(m.group(1) == name for m in _PARAM_RE.finditer(raw_args)):
|
||||
return None
|
||||
|
||||
window = _param_value_end_window(value)
|
||||
started = bool(value)
|
||||
held = "\n" if trim_newlines and value.endswith("\n") and value != "\n" else ""
|
||||
|
||||
def extend(delta: str) -> str | None:
|
||||
nonlocal window, started, held
|
||||
text = window + delta
|
||||
if _PARAM_VALUE_END_RE.search(text):
|
||||
return None
|
||||
window = _param_value_end_window(text)
|
||||
if trim_newlines:
|
||||
text = held + delta
|
||||
if not started and text:
|
||||
started = True
|
||||
text = text.removeprefix("\n")
|
||||
held = "\n" if text.endswith("\n") else ""
|
||||
text = text[: len(text) - len(held)]
|
||||
else:
|
||||
text = delta
|
||||
return json.dumps(text, ensure_ascii=False)[1:-1]
|
||||
|
||||
return extend
|
||||
|
||||
|
||||
def _qwen3_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
|
||||
return _param_value_extender(raw_args, trim_newlines=True)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def qwen3_config(
|
||||
thinking: bool = True,
|
||||
@@ -223,6 +282,7 @@ def qwen3_config(
|
||||
),
|
||||
},
|
||||
arg_converter=_qwen3_arg_converter,
|
||||
arg_value_extender=_qwen3_arg_value_extender,
|
||||
stream_arg_deltas=True,
|
||||
strip_trailing_reasoning_whitespace=False,
|
||||
tool_args_json=False,
|
||||
|
||||
@@ -405,6 +405,12 @@ if (
|
||||
):
|
||||
os.environ["HIP_ONLINE_TUNING"] = "1"
|
||||
|
||||
# RCCL 2.30's RAS thread can abort the process at exit ("terminate called
|
||||
# without an active exception") when its shutdown notification fails.
|
||||
# https://github.com/ROCm/rocm-systems/issues/13572
|
||||
if torch.version.hip:
|
||||
os.environ.setdefault("NCCL_RAS_ENABLE", "0")
|
||||
|
||||
|
||||
@cache
|
||||
def use_rocm_custom_paged_attention(
|
||||
|
||||
@@ -834,6 +834,9 @@ class OnlineDerenderer:
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
top_limit = _completion_top_logprobs_limit(completion_request)
|
||||
return_token_ids = (
|
||||
completion_request is not None and completion_request.return_token_ids
|
||||
)
|
||||
|
||||
for gen, pt, seed_ids_override in zip(
|
||||
generate_responses, prompt_tokens_list, prompt_token_ids_list
|
||||
@@ -877,6 +880,7 @@ class OnlineDerenderer:
|
||||
finish_reason=choice.finish_reason,
|
||||
stop_reason=choice.stop_reason,
|
||||
logprobs=completion_logprobs,
|
||||
token_ids=choice.token_ids if return_token_ids else None,
|
||||
)
|
||||
)
|
||||
total_completion_tokens += len(choice.token_ids)
|
||||
@@ -942,6 +946,9 @@ class OnlineDerenderer:
|
||||
updated_state = state
|
||||
|
||||
top_limit = _completion_top_logprobs_limit(completion_request)
|
||||
return_token_ids = (
|
||||
completion_request is not None and completion_request.return_token_ids
|
||||
)
|
||||
for choice in generate_chunk.choices:
|
||||
delta_tids = choice.token_ids or []
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
@@ -982,6 +989,7 @@ class OnlineDerenderer:
|
||||
logprobs=completion_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
stop_reason=choice.stop_reason,
|
||||
token_ids=choice.token_ids if return_token_ids else None,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
+33
-1
@@ -8,7 +8,7 @@ import math
|
||||
from dataclasses import field
|
||||
from enum import Enum, IntEnum
|
||||
from functools import cached_property
|
||||
from typing import Annotated, Any
|
||||
from typing import TYPE_CHECKING, Annotated, Any
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -24,6 +24,9 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.utils.torch_utils import make_ndarray_with_pad
|
||||
from vllm.v1.serial_utils import PydanticMsgspecMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_SAMPLING_EPS = 1e-5
|
||||
@@ -1507,6 +1510,35 @@ class BeamSearchParams(
|
||||
structured_outputs: StructuredOutputsParams | None = None
|
||||
skip_special_tokens: bool = True
|
||||
watermarking: bool | None = None
|
||||
allowed_token_ids: list[int] | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_verify_num_sequences(self.beam_width, "beam_width")
|
||||
|
||||
def configure_logprobs(
|
||||
self, sampling_params: SamplingParams, config: "VllmConfig"
|
||||
) -> None:
|
||||
ids = self.allowed_token_ids
|
||||
if ids is None:
|
||||
return
|
||||
if not ids:
|
||||
raise ValueError("Beam search allowed_token_ids must not be empty")
|
||||
if self.structured_outputs is not None:
|
||||
# Grammar candidates can exceed the exact-ID limit of 128 tokens.
|
||||
raise ValueError(
|
||||
"Beam allowlists with structured outputs are not supported"
|
||||
)
|
||||
if config.speculative_config is not None:
|
||||
# Rejection sampling does not yet fully support exact-ID logprobs.
|
||||
raise ValueError(
|
||||
"Beam allowlists with speculative decoding are not supported"
|
||||
)
|
||||
if config.model_config.is_diffusion:
|
||||
# The diffusion sampler does not consume logprob_token_ids.
|
||||
raise ValueError("Beam allowlists with diffusion models are not supported")
|
||||
if not config.use_v2_model_runner:
|
||||
# V1 exact-ID logprobs overwrite other rows in a mixed batch.
|
||||
raise ValueError("Beam allowlists require the V2 model runner")
|
||||
# Score the original distribution without applying a sampling mask.
|
||||
sampling_params.logprobs = None
|
||||
sampling_params.logprob_token_ids = ids
|
||||
|
||||
@@ -9,7 +9,7 @@ from dataclasses import replace
|
||||
from functools import partial, reduce
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config import SpeculativeConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
@@ -1038,120 +1038,20 @@ def _get_packed_kv_cache_groups(
|
||||
assert group_spec is not None
|
||||
groups.append(KVCacheGroupSpec(group_layer_names, group_spec))
|
||||
|
||||
_annotate_eagle_groups(
|
||||
vllm_config,
|
||||
kv_cache_spec,
|
||||
groups,
|
||||
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
|
||||
)
|
||||
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
|
||||
return groups
|
||||
|
||||
|
||||
def _uses_trailing_mtp_layers(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the drafter's KV layers can be located positionally.
|
||||
|
||||
MTP drafters register their KV layers after the target layers, without a
|
||||
distinguishing spec marker. DeepseekV4/V4.1 do this with ``dspark`` too.
|
||||
The annotator separately checks that the groups partition the layers exactly.
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return False
|
||||
if spec_config.method == "mtp":
|
||||
return True
|
||||
model_config = vllm_config.model_config
|
||||
return model_config is not None and model_config.hf_config.model_type in (
|
||||
"deepseek_v4",
|
||||
"deepseek_v41",
|
||||
)
|
||||
|
||||
|
||||
def _groups_partition_layers_exactly(
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> bool:
|
||||
"""Whether the groups cover every layer of ``kv_cache_spec`` exactly once.
|
||||
|
||||
The trailing-layer draft fallback is only meaningful when the last
|
||||
registered layer is guaranteed to appear in exactly one group.
|
||||
"""
|
||||
names = [name for group in kv_cache_groups for name in group.layer_names]
|
||||
return len(names) == len(kv_cache_spec) and set(names) == set(kv_cache_spec)
|
||||
|
||||
|
||||
def _annotate_eagle_groups(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: dict[str, KVCacheSpec],
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
use_trailing_layer_fallback: bool = False,
|
||||
) -> None:
|
||||
"""Flag the KV cache groups that hold drafter attention layers.
|
||||
|
||||
Two detection rules, in order of preference:
|
||||
|
||||
1. Spec-driven. ``non_causal_multi_token_decode`` is declared on
|
||||
MLAAttentionSpec and set by drafter attention layers that run a
|
||||
non-causal multi-token decode (today only Kimi-K3 DSpark). It survives
|
||||
MLAAttentionSpec.merge, so it still identifies a group after per-group
|
||||
spec merging, wherever grouping happens to land. It is sufficient but
|
||||
not necessary: a drafter whose spec is indistinguishable from the
|
||||
target's cannot be found this way.
|
||||
2. Positional fallback for MTP drafters (including DeepseekV4/V4.1 DSpark), whose
|
||||
MTP block reuses the target's own decoder layer and so carries no spec
|
||||
marker. Their draft attention layers always register after every
|
||||
target layer, so flag whichever group holds the last registered layer.
|
||||
This rule is only valid where the groups partition exactly the layers
|
||||
of ``kv_cache_spec``, which is re-checked here before applying it.
|
||||
When a drafter's trailing caches span several groups, this rule flags
|
||||
only the group holding the very last layer and must be generalized.
|
||||
FIXME(yifan): avoid/generalize this hacky check.
|
||||
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
kv_cache_spec: The kv cache spec of each attention layer, in layer
|
||||
registration order. Only read by rule 2.
|
||||
kv_cache_groups: Groups to annotate in place.
|
||||
use_trailing_layer_fallback: Enable rule 2. Callers gate this on
|
||||
``_uses_trailing_mtp_layers``.
|
||||
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle_block_drop():
|
||||
return
|
||||
|
||||
for group in kv_cache_groups:
|
||||
if any(
|
||||
getattr(spec, "non_causal_multi_token_decode", False)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
):
|
||||
group.is_eagle_group = True
|
||||
|
||||
if not use_trailing_layer_fallback:
|
||||
return
|
||||
if not _groups_partition_layers_exactly(kv_cache_spec, kv_cache_groups):
|
||||
return
|
||||
last_layer = next(reversed(kv_cache_spec))
|
||||
for group in kv_cache_groups:
|
||||
if last_layer in group.layer_names:
|
||||
group.is_eagle_group = True
|
||||
break
|
||||
|
||||
|
||||
def _warn_if_unannotated_eagle_mamba(
|
||||
vllm_config: VllmConfig,
|
||||
spec_config: SpeculativeConfig,
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> None:
|
||||
"""Warn when no KV cache group could be identified as the draft model's.
|
||||
|
||||
Args:
|
||||
vllm_config: Config supplying the speculative method, if any.
|
||||
spec_config: Config supplying the speculative method.
|
||||
kv_cache_groups: Groups as they will be handed to consumers.
|
||||
|
||||
"""
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is None or not spec_config.use_eagle():
|
||||
return
|
||||
if any(group.is_eagle_group for group in kv_cache_groups):
|
||||
return
|
||||
mamba_groups = [
|
||||
@@ -1297,13 +1197,6 @@ def get_kv_cache_groups(
|
||||
aligned = replace(spec, block_size=new_bs, page_size_padded=common_page)
|
||||
groups.append(KVCacheGroupSpec([name], aligned))
|
||||
|
||||
_annotate_eagle_groups(
|
||||
vllm_config,
|
||||
kv_cache_spec,
|
||||
groups,
|
||||
use_trailing_layer_fallback=_uses_trailing_mtp_layers(vllm_config),
|
||||
)
|
||||
_warn_if_unannotated_eagle_mamba(vllm_config, groups)
|
||||
return groups
|
||||
|
||||
|
||||
@@ -1556,7 +1449,7 @@ def _project_kv_cache_groups_to_worker(
|
||||
KVCacheGroupSpec(
|
||||
worker_layer_names,
|
||||
group_spec,
|
||||
is_eagle_group=group.is_eagle_group and bool(worker_layer_names),
|
||||
is_eagle_group=group.is_eagle_group,
|
||||
host_resident=group.host_resident,
|
||||
enable_kv_transfer=group.enable_kv_transfer,
|
||||
role=group.role,
|
||||
@@ -1636,6 +1529,14 @@ def get_kv_cache_configs(
|
||||
global_kv_cache_groups = get_kv_cache_groups_from_builder(
|
||||
builder, vllm_config, merged_kv_cache_specs
|
||||
)
|
||||
spec_config = vllm_config.speculative_config
|
||||
if spec_config is not None and spec_config.use_eagle_block_drop():
|
||||
draft_layer_names = {
|
||||
name for name, spec in merged_kv_cache_specs.items() if spec.is_draft
|
||||
}
|
||||
for group in global_kv_cache_groups:
|
||||
group.is_eagle_group = not draft_layer_names.isdisjoint(group.layer_names)
|
||||
_warn_if_unannotated_eagle_mamba(spec_config, global_kv_cache_groups)
|
||||
|
||||
# If original_max_model_len was -1, automatically
|
||||
# determine the maximum model length that fits in available GPU memory.
|
||||
|
||||
@@ -1732,14 +1732,14 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
Discards the last sampled output token from the prior input chunk.
|
||||
"""
|
||||
# Current streaming input behaviour: Keep only computed output tokens
|
||||
# (discard final sampled output token).
|
||||
num_computed_tokens = session.num_computed_tokens
|
||||
# Keep all output tokens but the final sampled one.
|
||||
keep_end = session.num_tokens - 1
|
||||
assert keep_end >= session.num_computed_tokens
|
||||
kept_output_tokens = session._all_token_ids[
|
||||
session.num_prompt_tokens : num_computed_tokens
|
||||
session.num_prompt_tokens : keep_end
|
||||
]
|
||||
del session._all_token_ids[num_computed_tokens:]
|
||||
del session.block_hashes[num_computed_tokens // self.hash_block_size :]
|
||||
del session._all_token_ids[keep_end:]
|
||||
del session.block_hashes[keep_end // self.hash_block_size :]
|
||||
session._output_token_ids.clear()
|
||||
assert session.prompt_token_ids is not None
|
||||
# Extend prompt with kept output tokens.
|
||||
@@ -2155,6 +2155,8 @@ class Scheduler(SchedulerInterface):
|
||||
# to avoid expensive operations inside the loop.
|
||||
stopped_running_reqs: set[Request] = set()
|
||||
stopped_preempted_reqs: set[Request] = set()
|
||||
# Stopped resumable requests to re-enqueue, in output order.
|
||||
continuing_reqs: list[Request] = []
|
||||
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
|
||||
assert num_tokens_scheduled > 0
|
||||
request = self.requests.get(req_id)
|
||||
@@ -2320,6 +2322,8 @@ class Scheduler(SchedulerInterface):
|
||||
kv_transfer_params["remote_prefill_cached_tokens"] = (
|
||||
prefill_stats.num_cached_tokens
|
||||
)
|
||||
else:
|
||||
continuing_reqs.append(request)
|
||||
|
||||
if status_before_stop == RequestStatus.RUNNING:
|
||||
stopped_running_reqs.add(request)
|
||||
@@ -2371,14 +2375,16 @@ class Scheduler(SchedulerInterface):
|
||||
assert not prompt_logprobs_tensors
|
||||
assert prompt_token_id_logprobs is None
|
||||
|
||||
# Remove the stopped requests from the running and waiting queues.
|
||||
# Remove stopped requests from their queues, then enqueue the ones
|
||||
# that continue, so each ends up in exactly one queue.
|
||||
if stopped_running_reqs:
|
||||
self.running = remove_all(self.running, stopped_running_reqs)
|
||||
if stopped_preempted_reqs:
|
||||
# This is a rare case and unlikely to impact performance.
|
||||
self.waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.kv_holding_waiting.remove_requests(stopped_preempted_reqs)
|
||||
self.deferred_waiting.difference_update(stopped_preempted_reqs)
|
||||
for request in continuing_reqs:
|
||||
self._enqueue_waiting_request(request)
|
||||
|
||||
error_req_ids = set(self.grammar_compile_error_reqs)
|
||||
self.grammar_compile_error_reqs.clear()
|
||||
@@ -2568,7 +2574,6 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
self.num_waiting_for_streaming_input += 1
|
||||
|
||||
self._enqueue_waiting_request(request)
|
||||
return False
|
||||
|
||||
def _update_request_with_output(
|
||||
|
||||
@@ -599,6 +599,7 @@ class AsyncLLM(EngineClient):
|
||||
session_id: str | None = None,
|
||||
kv_hints: KvHintsEnvelope | None = None,
|
||||
) -> RequestOutputCollector:
|
||||
self._validate_resumable_request()
|
||||
self._validate_streaming_input_sampling_params(sampling_params)
|
||||
|
||||
inputs = dict(
|
||||
@@ -693,6 +694,13 @@ class AsyncLLM(EngineClient):
|
||||
queue._input_stream_task = asyncio.create_task(handle_inputs())
|
||||
return queue
|
||||
|
||||
def _validate_resumable_request(self) -> None:
|
||||
if self.vllm_config.is_mm_encoder_only or self.model_config.is_diffusion:
|
||||
raise VLLMValidationError(
|
||||
"Resumable requests are not supported for encoder-only or "
|
||||
"diffusion models."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_streaming_input_sampling_params(
|
||||
params: SamplingParams | PoolingParams,
|
||||
|
||||
@@ -175,6 +175,10 @@ class KVCacheSpec:
|
||||
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
|
||||
blocks of up to ``n`` tokens."""
|
||||
|
||||
is_draft: bool = field(default=False, kw_only=True, compare=False)
|
||||
"""Whether a draft model owns this layer; excluded from equality so it never
|
||||
affects grouping."""
|
||||
|
||||
def __post_init__(self):
|
||||
alignment = self.block_stride_alignment
|
||||
if isinstance(alignment, int) and alignment <= 0:
|
||||
@@ -660,6 +664,8 @@ class FullAttentionSpec(AttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -747,6 +753,8 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -1208,6 +1216,8 @@ class SinkFullAttentionSpec(FullAttentionSpec):
|
||||
)
|
||||
for spec in specs:
|
||||
for f in fields(AttentionSpec):
|
||||
if not f.compare:
|
||||
continue
|
||||
assert getattr(spec, f.name) == getattr(merged_spec, f.name), (
|
||||
"All attention layers in the same KV cache group must have "
|
||||
"the same attention spec."
|
||||
@@ -1445,7 +1455,7 @@ class KVCacheGroupSpec:
|
||||
layer_names: list[str]
|
||||
# The KV cache spec of this manager layer
|
||||
kv_cache_spec: KVCacheSpec
|
||||
# Whether this group contains EAGLE/MTP draft attention layers.
|
||||
# Whether this group holds draft layers and takes the EAGLE last-block drop.
|
||||
is_eagle_group: bool = False
|
||||
# Host groups use the dedicated HiSparse pool; others share the device pool.
|
||||
host_resident: bool = False
|
||||
|
||||
@@ -1330,10 +1330,13 @@ class SpecDecodeBaseProposer:
|
||||
self.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
)
|
||||
draft_attn_layer_names = set(all_attn_layers) - target_attn_layer_names
|
||||
for name in draft_attn_layer_names:
|
||||
all_attn_layers[name].is_draft_layer = True
|
||||
# Filter to only layers that have KV cache specs.
|
||||
self._draft_attn_layer_names = {
|
||||
name
|
||||
for name in (set(all_attn_layers.keys()) - target_attn_layer_names)
|
||||
for name in draft_attn_layer_names
|
||||
if all_attn_layers[name].get_kv_cache_spec(self.vllm_config) is not None
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -148,6 +148,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
if attn_module.is_draft_layer:
|
||||
spec = replace(spec, is_draft=True)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
|
||||
@@ -74,6 +74,7 @@ class BlockTables:
|
||||
self.input_block_tables: list[torch.Tensor] = [
|
||||
torch.zeros_like(b.gpu) for b in self.block_tables
|
||||
]
|
||||
self._num_dirty_rows = 0
|
||||
|
||||
self.slot_mappings = torch.zeros(
|
||||
self.num_kv_cache_groups,
|
||||
@@ -151,15 +152,18 @@ class BlockTables:
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
if self.num_kv_cache_groups == 0:
|
||||
return ()
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
num_rows = num_reqs_padded
|
||||
if out is None:
|
||||
out = tuple(self.input_block_tables)
|
||||
out_ptrs = self.input_block_table_ptrs
|
||||
num_rows = max(num_rows, self._num_dirty_rows)
|
||||
self._num_dirty_rows = num_reqs
|
||||
else:
|
||||
assert out_ptrs is not None
|
||||
assert len(out) == self.num_kv_cache_groups
|
||||
num_reqs = idx_mapping.shape[0]
|
||||
# Launch kernel with num_reqs_padded to fuse zeroing of padded rows.
|
||||
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_reqs_padded)](
|
||||
# Also clear rows left by a larger batch in the shared buffer.
|
||||
_gather_block_tables_kernel[(self.num_kv_cache_groups, num_rows)](
|
||||
idx_mapping,
|
||||
self.block_table_ptrs,
|
||||
out_ptrs,
|
||||
@@ -180,9 +184,12 @@ class BlockTables:
|
||||
# Zero the rows so dummy runs write mamba state to the reserved null
|
||||
# block rather than through the previous real step's (stale) block
|
||||
# ids, which may point at blocks since freed and reallocated.
|
||||
return tuple(
|
||||
block_table[:num_reqs].zero_() for block_table in self.input_block_tables
|
||||
)
|
||||
num_rows = max(num_reqs, self._num_dirty_rows)
|
||||
for block_table in self.input_block_tables:
|
||||
block_table[:num_rows].zero_()
|
||||
# Profiling may populate the returned rows with valid block IDs.
|
||||
self._num_dirty_rows = num_reqs
|
||||
return tuple(block_table[:num_reqs] for block_table in self.input_block_tables)
|
||||
|
||||
def compute_slot_mappings(
|
||||
self,
|
||||
|
||||
@@ -41,9 +41,14 @@ class KVConnector:
|
||||
def finish_forward(self) -> None:
|
||||
pass
|
||||
|
||||
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
|
||||
def post_forward(
|
||||
self, finished_req_ids: set[str], defer_finalize: bool = False
|
||||
) -> KVConnectorOutput | None:
|
||||
return None
|
||||
|
||||
def finalize_step(self) -> None:
|
||||
pass
|
||||
|
||||
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
@@ -128,7 +133,9 @@ class ActiveKVConnector(KVConnector):
|
||||
def reset_capture_state(self) -> None:
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
def post_forward(self, finished_req_ids: set[str]) -> KVConnectorOutput | None:
|
||||
def post_forward(
|
||||
self, finished_req_ids: set[str], defer_finalize: bool = False
|
||||
) -> KVConnectorOutput | None:
|
||||
if self._disabled:
|
||||
return None
|
||||
|
||||
@@ -136,7 +143,8 @@ class ActiveKVConnector(KVConnector):
|
||||
self._start_load_kv()
|
||||
|
||||
output = KVConnectorOutput()
|
||||
self.kv_connector.wait_for_save()
|
||||
if not defer_finalize:
|
||||
self.kv_connector.wait_for_save()
|
||||
transfer_results = self.kv_connector.get_transfer_results(finished_req_ids)
|
||||
output.finished_sending = transfer_results.finished_sending or None
|
||||
output.finished_recving = transfer_results.finished_recving or None
|
||||
@@ -147,9 +155,16 @@ class ActiveKVConnector(KVConnector):
|
||||
output.kv_connector_worker_meta = (
|
||||
self.kv_connector.build_connector_worker_meta()
|
||||
)
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
if not defer_finalize:
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
return output
|
||||
|
||||
def finalize_step(self) -> None:
|
||||
if self._disabled:
|
||||
return
|
||||
self.kv_connector.wait_for_save()
|
||||
self.kv_connector.clear_connector_metadata()
|
||||
|
||||
def no_forward(self, scheduler_output: "SchedulerOutput") -> ModelRunnerOutput:
|
||||
if self._disabled:
|
||||
return EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
@@ -2207,6 +2207,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
pending_aux_output=pending_aux_output,
|
||||
)
|
||||
|
||||
# Collect KV transfer results now; finalize saves after the drafter runs.
|
||||
defer_kv_finalize = self.speculator is not None
|
||||
model_runner_output.kv_connector_output = self.kv_connector.post_forward(
|
||||
finished_req_ids, defer_finalize=defer_kv_finalize
|
||||
)
|
||||
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
# The encoder runner exists only on the first PP rank, so later ranks
|
||||
# have no cached embeddings to gather.
|
||||
@@ -2288,9 +2294,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.draft_tokens, input_batch
|
||||
)
|
||||
|
||||
# Post-step KV connector related operations.
|
||||
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
|
||||
model_runner_output.kv_connector_output = kv_connector_output
|
||||
if defer_kv_finalize:
|
||||
self.kv_connector.finalize_step()
|
||||
model_runner_output.ec_connector_output = ec_connector_output
|
||||
|
||||
return async_output
|
||||
|
||||
@@ -238,7 +238,8 @@ def warmup_kernels(
|
||||
rejection_sampler.enable_adaptive_verification = False
|
||||
try:
|
||||
_warmup_kernels(model_runner, worker_execute_model, worker_sample_tokens)
|
||||
if rejection_sampler is not None:
|
||||
# All PP ranks must run the same forwards to match sends and receives.
|
||||
if model_runner.speculative_config is not None:
|
||||
# Greedy batches skip logits processing, so rejection kernels
|
||||
# also see model-dtype logits instead of FP32.
|
||||
_warmup_kernels(
|
||||
|
||||
@@ -7140,6 +7140,8 @@ class GPUModelRunner(
|
||||
spec = customize_attention_spec(
|
||||
attn_module.get_attn_backend(), spec
|
||||
)
|
||||
if attn_module.is_draft_layer:
|
||||
spec = replace(spec, is_draft=True)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
@@ -61,9 +61,9 @@ def _torch_cuda_wrapper():
|
||||
torch.cuda.stream = partial(torch.xpu.stream)
|
||||
torch.cuda.set_stream = partial(torch.xpu.set_stream)
|
||||
|
||||
# torch.xpu.Event does not accept the ``blocking`` kwarg that
|
||||
# torch.cuda.Event supports, so drop it here.
|
||||
def _xpu_event(*args, blocking=None, **kwargs):
|
||||
# torch.xpu.Event does not accept the ``blocking`` or ``external`` kwargs
|
||||
# that torch.cuda.Event supports, so drop them here.
|
||||
def _xpu_event(*args, blocking=None, external=None, **kwargs):
|
||||
return torch.xpu.Event(*args, **kwargs)
|
||||
|
||||
torch.cuda.Event = _xpu_event
|
||||
|
||||
Reference in New Issue
Block a user