[Model] Add DiffusionGemma Support (#45163)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Martin Kukla <martin.kukla@cantab.net>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Dipika Sikka <dsikka@redhat.com>
Co-authored-by: NickLucche <nlucches@redhat.com>
Co-authored-by: jiahanc <173873397+jiahanc@users.noreply.github.com>
Co-authored-by: Alec Kohlhoff <134344302+aleckohlhoff@users.noreply.github.com>
Co-authored-by: Porras Huang <20535584+porrashuang@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: scoootscooob <167050519+scoootscooob@users.noreply.github.com>
This commit is contained in:
Lucas Wilkinson
2026-06-11 22:17:35 -07:00
committed by GitHub
co-authored by Martin Kukla Matthew Bonanni Dipika Sikka NickLucche jiahanc Alec Kohlhoff Porras Huang Claude scoootscooob
parent 1ce3cdc5c1
commit eb28452b10
52 changed files with 2695 additions and 232 deletions
+6
View File
@@ -792,6 +792,12 @@ def get_model_params(config):
topk = text_config.num_experts_per_tok
intermediate_size = text_config.moe_intermediate_size
hidden_size = text_config.hidden_size
elif architecture == "DiffusionGemmaForBlockDiffusion":
text_config = config.get_text_config()
E = text_config.num_experts
topk = text_config.top_k_experts
intermediate_size = text_config.moe_intermediate_size
hidden_size = text_config.hidden_size
elif architecture == "HunYuanMoEV1ForCausalLM":
E = config.num_experts
topk = config.moe_topk[0]
@@ -39,7 +39,7 @@ else()
FetchContent_Declare(
vllm-flash-attn
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
GIT_TAG dd62dac706b1cf7895bd99b18c6cb7e7e117ee25
GIT_TAG 803020a8fa15407871341d41eba4919ade2ee1ee
GIT_PROGRESS TRUE
# Don't share the vllm-flash-attn build between build types
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
+1 -1
View File
@@ -180,7 +180,7 @@ Priority is **1 = highest** (tried first).
| `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ✅ | ✅ | ❌ | ❌ | Decoder | N/A |
| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |
| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |
| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | Any |
| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ✅ | ✅ | ❌ | All | Any |
| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
@@ -0,0 +1,318 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for per-request causal/non-causal attention (mixed batches).
Validates that both triton and flash-attention backends correctly handle
batches where some sequences use causal masking and others use non-causal
(bidirectional) masking — needed by DiffusionGemma.
"""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
# Mixed causal/non-causal attention is only validated on a subset of GPUs:
# the Triton path on Hopper (SM90) and B200 (SM100); the FA4 path on Hopper
# (SM90) only.
_device_capability = current_platform.get_device_capability()
_major = _device_capability.major if _device_capability is not None else None
NUM_HEADS = [(4, 4), (8, 2)]
HEAD_SIZES = [128]
BLOCK_SIZES = [16]
DTYPES = [torch.bfloat16]
def ref_paged_attn(
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
query_lens: list[int],
kv_lens: list[int],
block_tables: torch.Tensor,
scale: float,
per_seq_causal: list[bool],
sliding_window: int | None = None,
) -> torch.Tensor:
num_seqs = len(query_lens)
block_tables_np = block_tables.cpu().numpy()
_, block_size, num_kv_heads, head_size = key_cache.shape
outputs: list[torch.Tensor] = []
start_idx = 0
for i in range(num_seqs):
query_len = query_lens[i]
kv_len = kv_lens[i]
q = query[start_idx : start_idx + query_len]
q = q * scale
num_kv_blocks = (kv_len + block_size - 1) // block_size
block_indices = block_tables_np[i, :num_kv_blocks]
k = key_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
v = value_cache[block_indices].view(-1, num_kv_heads, head_size)[:kv_len]
if q.shape[1] != k.shape[1]:
k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
attn = torch.einsum("qhd,khd->hqk", q, k).float()
if per_seq_causal[i]:
mask = torch.triu(
torch.ones(query_len, kv_len, device=attn.device),
diagonal=kv_len - query_len + 1,
).bool()
else:
mask = torch.zeros(query_len, kv_len, device=attn.device).bool()
if sliding_window is not None:
sw_mask = (
torch.triu(
torch.ones(query_len, kv_len, device=attn.device),
diagonal=kv_len - (query_len + sliding_window) + 1,
)
.bool()
.logical_not()
)
mask |= sw_mask
attn.masked_fill_(mask, float("-inf"))
attn = torch.softmax(attn, dim=-1).to(v.dtype)
out = torch.einsum("hqk,khd->qhd", attn, v)
outputs.append(out)
start_idx += query_len
return torch.cat(outputs, dim=0)
# ---- Triton backend test ----
@pytest.mark.skipif(
_major not in (9, 10),
reason="Triton mixed causal attention requires Hopper (SM90) or B200 (SM100).",
)
@pytest.mark.parametrize(
"seq_lens",
[[(1, 128), (5, 64), (1, 256)]],
)
@pytest.mark.parametrize(
"per_seq_causal",
[[True, False, True], [False, True, False], [True, True, False]],
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_triton_mixed_causal(
seq_lens: list[tuple[int, int]],
per_seq_causal: list[bool],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
):
if not current_platform.is_cuda():
pytest.skip("Triton attention requires CUDA")
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
set_random_seed(42)
device = "cuda"
num_query_heads, num_kv_heads = num_heads
assert len(seq_lens) == len(per_seq_causal)
query_lens = [s[0] for s in seq_lens]
kv_lens = [s[1] for s in seq_lens]
num_seqs = len(seq_lens)
num_query_tokens = sum(query_lens)
max_kv_len = max(kv_lens)
max_num_blocks = (max_kv_len + block_size - 1) // block_size
num_blocks = max_num_blocks * num_seqs + 10
scale = head_size**-0.5
query = torch.randn(
num_query_tokens, num_query_heads, head_size, dtype=dtype, device=device
)
key_cache = torch.randn(
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
)
value_cache = torch.randn(
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
)
block_tables_list = []
for i in range(num_seqs):
n_blocks = (kv_lens[i] + block_size - 1) // block_size
blocks = list(range(i * max_num_blocks, i * max_num_blocks + n_blocks))
blocks += [0] * (max_num_blocks - n_blocks)
block_tables_list.append(blocks)
block_tables = torch.tensor(block_tables_list, dtype=torch.int32, device=device)
cu_seqlens_q = torch.zeros(num_seqs + 1, dtype=torch.int32, device=device)
for i, ql in enumerate(query_lens):
cu_seqlens_q[i + 1] = cu_seqlens_q[i] + ql
seqused_k = torch.tensor(kv_lens, dtype=torch.int32, device=device)
max_seqlen_q = max(query_lens)
max_seqlen_k = max(kv_lens)
causal_tensor = torch.tensor(per_seq_causal, dtype=torch.bool, device=device)
output = torch.empty_like(query)
unified_attention(
q=query,
k=key_cache,
v=value_cache,
out=output,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=max_seqlen_q,
seqused_k=seqused_k,
max_seqlen_k=max_seqlen_k,
softmax_scale=scale,
causal=causal_tensor,
window_size=(-1, -1),
block_table=block_tables,
softcap=0.0,
q_descale=None,
k_descale=1.0,
v_descale=1.0,
)
ref_output = ref_paged_attn(
query,
key_cache,
value_cache,
query_lens,
kv_lens,
block_tables,
scale,
per_seq_causal,
)
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
# ---- Flash Attention 4 backend test (native per_seq_causal) ----
@pytest.mark.skipif(
_major != 9,
reason="FA4 mixed causal attention requires Hopper (SM90).",
)
@pytest.mark.parametrize(
"seq_lens",
[[(1, 128), (5, 64), (1, 256)]],
)
@pytest.mark.parametrize(
"per_seq_causal",
[[True, False, True], [False, True, False]],
)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_flash_attn4_mixed_causal(
seq_lens: list[tuple[int, int]],
per_seq_causal: list[bool],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
):
if not current_platform.is_cuda():
pytest.skip("Flash attention requires CUDA")
try:
from vllm.vllm_flash_attn import (
fa_version_unsupported_reason,
flash_attn_varlen_func,
is_fa_version_supported,
)
except ImportError:
pytest.skip("vllm_flash_attn not available")
if not is_fa_version_supported(4):
reason = fa_version_unsupported_reason(4)
pytest.skip(f"FA4 not supported: {reason}")
set_random_seed(42)
device = "cuda"
num_query_heads, num_kv_heads = num_heads
assert len(seq_lens) == len(per_seq_causal)
query_lens = [s[0] for s in seq_lens]
kv_lens = [s[1] for s in seq_lens]
num_seqs = len(seq_lens)
num_query_tokens = sum(query_lens)
max_kv_len = max(kv_lens)
max_num_blocks = (max_kv_len + block_size - 1) // block_size
num_blocks = max_num_blocks * num_seqs + 10
scale = head_size**-0.5
query = torch.randn(
num_query_tokens, num_query_heads, head_size, dtype=dtype, device=device
)
key_cache = torch.randn(
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
)
value_cache = torch.randn(
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype, device=device
)
block_tables_list = []
for i in range(num_seqs):
n_blocks = (kv_lens[i] + block_size - 1) // block_size
blocks = list(range(i * max_num_blocks, i * max_num_blocks + n_blocks))
blocks += [0] * (max_num_blocks - n_blocks)
block_tables_list.append(blocks)
block_tables = torch.tensor(block_tables_list, dtype=torch.int32, device=device)
cu_seqlens_q = torch.zeros(num_seqs + 1, dtype=torch.int32, device=device)
for i, ql in enumerate(query_lens):
cu_seqlens_q[i + 1] = cu_seqlens_q[i] + ql
seqused_k = torch.tensor(kv_lens, dtype=torch.int32, device=device)
per_seq_causal_tensor = torch.tensor(
per_seq_causal, dtype=torch.int32, device=device
)
ref_output = ref_paged_attn(
query,
key_cache,
value_cache,
query_lens,
kv_lens,
block_tables,
scale,
per_seq_causal,
)
output = torch.empty_like(query)
flash_attn_varlen_func(
q=query,
k=key_cache,
v=value_cache,
out=output,
cu_seqlens_q=cu_seqlens_q,
max_seqlen_q=max(query_lens),
seqused_k=seqused_k,
max_seqlen_k=max(kv_lens),
softmax_scale=scale,
# The kernel must be compiled causal for `dynamic_causal` to take effect.
causal=True,
block_table=block_tables,
softcap=0.0,
dynamic_causal=per_seq_causal_tensor,
fa_version=4,
)
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
+4
View File
@@ -898,6 +898,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
"Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
"DiffusionGemmaForBlockDiffusion": _HfExamplesInfo(
"google/diffusiongemma-26B-A4B-it",
trust_remote_code=True,
),
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.5.0",
+3 -1
View File
@@ -486,6 +486,7 @@ def dummy_hf_overrides(
"Gemma3nForConditionalGeneration",
"Gemma4ForCausalLM",
"Gemma4ForConditionalGeneration",
"DiffusionGemmaForBlockDiffusion",
)
else 1
)
@@ -558,7 +559,8 @@ def dummy_hf_overrides(
)
# e.g.: Qwen/Qwen2-Audio-7B-Instruct
if hasattr(hf_config, "audio_config"):
# audio_config may exist but be None (e.g. audio-less Gemma4 variants).
if getattr(hf_config, "audio_config", None) is not None:
hf_config.audio_config.update(
{
"num_layers": 1,
@@ -702,6 +702,88 @@ class TestStreamingExtraction:
' <meta name="viewport" content="width=device-width">\n'
)
def _collect_tool_calls_by_index(self, results):
"""Group streamed tool-call fragments by their ``index``.
Returns ``{index: {"name": str | None, "arguments": str}}`` where
``arguments`` is the concatenation of every streamed argument
fragment for that index (which should form valid JSON once complete).
"""
by_index: dict[int, dict[str, Any]] = {}
for delta, _ in results:
if not (delta and delta.tool_calls):
continue
for tc in delta.tool_calls:
entry = by_index.setdefault(tc.index, {"name": None, "arguments": ""})
func = tc.function
if isinstance(func, dict):
name = func.get("name")
arg = func.get("arguments", "")
else:
name = getattr(func, "name", None)
arg = getattr(func, "arguments", "") or ""
if name:
entry["name"] = name
if arg:
entry["arguments"] += arg
return by_index
def test_streaming_single_chunk_complete_tool_call(self, parser, mock_request):
"""A backend may deliver a whole tool call in one streaming delta.
The start token, ``call:name{...}`` payload and the end token all
arrive in a single chunk. The parser must still emit one
``DeltaToolCall`` with the correct name + complete arguments JSON
(rather than swallowing it and finishing with finish_reason="stop").
"""
chunks = [
'<|tool_call>call:name_a_color{color_hex:<|"|>00ff11<|"|>}<tool_call|>',
]
results = self._simulate_streaming(parser, mock_request, chunks)
# Exactly one delta should carry tool_calls, and it must not be
# emitted as plain content (which would yield finish_reason="stop").
tool_call_deltas = [
delta for delta, _ in results if delta is not None and delta.tool_calls
]
assert len(tool_call_deltas) == 1, (
"Expected exactly one delta carrying the batched tool call"
)
assert all(
delta.content is None for delta, _ in results if delta is not None
), "Complete tool call must not leak as content"
by_index = self._collect_tool_calls_by_index(results)
assert set(by_index) == {0}
assert by_index[0]["name"] == "name_a_color"
assert json.loads(by_index[0]["arguments"]) == {"color_hex": "00ff11"}
def test_streaming_multi_chunk_batched_tool_calls(self, parser, mock_request):
"""A single delta may batch MULTIPLE complete tool calls.
``<|tool_call>...<tool_call|><|tool_call>...<tool_call|>`` arriving in
one chunk must emit BOTH calls (one DeltaToolCall each, with distinct
indices), not just the first.
"""
chunks = [
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}<tool_call|>'
'<|tool_call>call:get_time{timezone:<|"|>GMT<|"|>}<tool_call|>',
]
results = self._simulate_streaming(parser, mock_request, chunks)
by_index = self._collect_tool_calls_by_index(results)
assert set(by_index) == {0, 1}, (
f"Expected two tool calls (indices 0 and 1), got {sorted(by_index)}"
)
assert by_index[0]["name"] == "get_weather"
assert json.loads(by_index[0]["arguments"]) == {"location": "London"}
assert by_index[1]["name"] == "get_time"
assert json.loads(by_index[1]["arguments"]) == {"timezone": "GMT"}
def test_streaming_trailing_bare_bool_not_duplicated(self, parser, mock_request):
"""Trailing bare boolean must not be streamed twice."""
chunks = [
@@ -49,6 +49,7 @@ def _create_vllm_config(
)
mock_config.parallel_config = ParallelConfig()
mock_config.speculative_config = None # No speculative decoding
mock_config.num_speculative_tokens = 0
if not lora_config:
mock_config.lora_config = None
else:
@@ -104,7 +104,7 @@ def _run_engine_core_handshake(
speculative_config=None,
ec_transfer_config=None,
max_concurrent_batches=1,
model_config=SimpleNamespace(runner_type="generate"),
model_config=SimpleNamespace(runner_type="generate", is_diffusion=False),
cache_config=SimpleNamespace(
enable_prefix_caching=False,
prefix_caching_hash_algo="builtin",
@@ -70,6 +70,7 @@ def _make_runner(**overrides: Any) -> Any:
runner.use_aux_hidden_state_outputs = False
runner.speculative_config = None
runner.speculator = None
runner.num_speculative_steps = 0
runner.encoder_cache = None
runner.is_pooling_model = False
runner.is_last_pp_rank = True
@@ -102,18 +103,22 @@ def test_v2_load_model_registers_moe_with_eplb(monkeypatch):
lambda load_config: SimpleNamespace(load_model=lambda **_: model),
)
monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append)
monkeypatch.setattr(mrv2, "init_model_state", lambda *args: "model-state")
monkeypatch.setattr(
mrv2,
"init_model_state",
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
)
monkeypatch.setattr(
eplb,
"is_mixture_of_experts",
lambda loaded_model: getattr(loaded_model, "is_moe", False),
)
runner = _make_runner()
runner = _make_runner(is_last_pp_rank=False)
mrv2.GPUModelRunner.load_model(runner)
assert runner.model is model
assert runner.model_state == "model-state"
assert runner.model_state is not None
assert prepared == [model]
assert runner.eplb_state is not None
assert runner.eplb_state.add_model_calls == [(model, runner.model_config)]
@@ -133,10 +138,14 @@ def test_v2_load_model_with_dummy_weights_skips_eplb_registration(monkeypatch):
lambda load_config: SimpleNamespace(load_model=lambda **_: model),
)
monkeypatch.setattr(mrv2, "prepare_communication_buffer_for_model", prepared.append)
monkeypatch.setattr(mrv2, "init_model_state", lambda *args: "model-state")
monkeypatch.setattr(
mrv2,
"init_model_state",
lambda *args: SimpleNamespace(num_new_sampled_tokens_per_step=1),
)
monkeypatch.setattr(eplb, "is_mixture_of_experts", lambda *_: True)
runner = _make_runner()
runner = _make_runner(is_last_pp_rank=False)
mrv2.GPUModelRunner.load_model(runner, load_dummy_weights=True)
assert runner.load_config.load_format == "dummy"
+117 -1
View File
@@ -248,6 +248,68 @@ async def fetch_spec_decode_metrics(
return None
@dataclass
class DiffusionMetrics:
"""Diffusion (dLLM) decoding metrics from the server's Prometheus endpoint."""
num_denoising_steps: int
num_canvas_positions: int
num_committed_tokens: int
async def fetch_diffusion_metrics(
base_url: str, session: aiohttp.ClientSession
) -> DiffusionMetrics | None:
"""Fetch diffusion decoding metrics from the server's Prometheus endpoint.
Returns None if the model is not a diffusion model or metrics are not
available.
"""
metrics_url = f"{base_url}/metrics"
try:
async with session.get(metrics_url) as response:
if response.status != 200:
return None
text = await response.text()
num_denoising_steps = 0
num_canvas_positions = 0
num_committed_tokens = 0
found_diffusion = False
for line in text.split("\n"):
line = line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("vllm:diffusion"):
# Extract metric name (before labels) to avoid matching
# substrings inside label values.
parts = line.split(None, 1)
metric_name = parts[0].split("{")[0]
if not metric_name.endswith("_total"):
continue
found_diffusion = True
with contextlib.suppress(ValueError):
if "num_denoising_steps" in metric_name:
num_denoising_steps += int(float(parts[-1]))
elif "num_canvas_positions" in metric_name:
num_canvas_positions += int(float(parts[-1]))
elif "num_committed_tokens" in metric_name:
num_committed_tokens += int(float(parts[-1]))
if not found_diffusion:
return None
return DiffusionMetrics(
num_denoising_steps=num_denoising_steps,
num_canvas_positions=num_canvas_positions,
num_committed_tokens=num_committed_tokens,
)
except (aiohttp.ClientError, asyncio.TimeoutError):
return None
class TaskType(Enum):
GENERATION = "generation"
POOLING = "pooling"
@@ -887,6 +949,7 @@ async def benchmark(
print("Self timing is set, using the timestamps from the trace file.")
spec_decode_metrics_before = await fetch_spec_decode_metrics(base_url, session)
diffusion_metrics_before = await fetch_diffusion_metrics(base_url, session)
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
@@ -1016,6 +1079,34 @@ async def benchmark(
"per_position_acceptance_rates": per_pos_rates,
}
diffusion_metrics_after = await fetch_diffusion_metrics(base_url, session)
diffusion_stats: dict[str, Any] | None = None
if diffusion_metrics_before is not None and diffusion_metrics_after is not None:
delta_steps = (
diffusion_metrics_after.num_denoising_steps
- diffusion_metrics_before.num_denoising_steps
)
delta_positions = (
diffusion_metrics_after.num_canvas_positions
- diffusion_metrics_before.num_canvas_positions
)
delta_committed = (
diffusion_metrics_after.num_committed_tokens
- diffusion_metrics_before.num_committed_tokens
)
if delta_steps > 0 and delta_committed > 0:
block_size = delta_positions / delta_steps # canvas length (CL)
num_canvases = delta_committed / block_size # = number of commit steps
denoising_steps = delta_steps - num_canvases # exclude commit steps
diffusion_stats = {
"denoising_steps": denoising_steps,
"canvas_positions": delta_positions,
"committed_tokens": delta_committed,
"committed_throughput": delta_committed / benchmark_duration,
"steps_per_canvas": denoising_steps / num_canvases,
"committed_per_step": delta_committed / denoising_steps,
}
if task_type == TaskType.GENERATION:
metrics, actual_output_lens = calculate_metrics(
input_requests=input_requests,
@@ -1134,6 +1225,16 @@ async def benchmark(
"per_position_acceptance_rates", []
)
if diffusion_stats is not None:
result["diffusion_committed_throughput"] = diffusion_stats[
"committed_throughput"
]
result["diffusion_steps_per_canvas"] = diffusion_stats["steps_per_canvas"]
result["diffusion_committed_per_step"] = diffusion_stats["committed_per_step"]
result["diffusion_committed_tokens"] = int(diffusion_stats["committed_tokens"])
result["diffusion_denoising_steps"] = int(diffusion_stats["denoising_steps"])
result["diffusion_canvas_positions"] = int(diffusion_stats["canvas_positions"])
def process_one_metric(
# E.g., "ttft"
metric_attribute_name: str,
@@ -1179,7 +1280,22 @@ async def benchmark(
process_one_metric("itl", "ITL", "Inter-token Latency")
process_one_metric("e2el", "E2EL", "End-to-end Latency")
if spec_decode_stats is not None:
if diffusion_stats is not None:
print("{s:{c}^{n}}".format(s="Diffusion Decoding", n=50, c="-"))
for label, key, value_fmt in (
("Committed throughput (tok/s):", "committed_throughput", "{:<10.2f}"),
("Denoising steps per canvas:", "steps_per_canvas", "{:<10.2f}"),
("Committed per denoising step:", "committed_per_step", "{:<10.2f}"),
("Committed tokens:", "committed_tokens", "{:<10d}"),
("Denoising steps:", "denoising_steps", "{:<10d}"),
("Canvas positions evaluated:", "canvas_positions", "{:<10d}"),
):
value = diffusion_stats[key]
if value_fmt.endswith("d}"):
value = int(value)
print("{:<40} ".format(label) + value_fmt.format(value))
if spec_decode_stats is not None and diffusion_stats is None:
print("{s:{c}^{n}}".format(s="Speculative Decoding", n=50, c="-"))
print(
"{:<40} {:<10.2f}".format(
+3
View File
@@ -10,6 +10,7 @@ from vllm.config.compilation import (
PassConfig,
)
from vllm.config.device import DeviceConfig
from vllm.config.diffusion import DiffusionConfig
from vllm.config.ec_transfer import ECTransferConfig
from vllm.config.kernel import KernelConfig
from vllm.config.kv_events import KVEventsConfig
@@ -72,6 +73,8 @@ __all__ = [
"PassConfig",
# From vllm.config.device
"DeviceConfig",
# From vllm.config.diffusion
"DiffusionConfig",
# From vllm.config.ec_transfer
"ECTransferConfig",
# From vllm.config.kernel
+26
View File
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for discrete diffusion (dLLM) models."""
from pydantic import Field
from vllm.config.utils import config
@config
class DiffusionConfig:
"""Configuration for discrete diffusion language models (dLLMs).
dLLMs generate tokens via iterative denoising over a fixed-length canvas
rather than left-to-right autoregressive decoding. They reuse the
speculative-decoding data path (draft token ids, scheduled spec decode
tokens) with overloaded semantics for block-based generation.
"""
canvas_length: int = Field(default=None, gt=0) # type: ignore[assignment]
"""Length of the denoising canvas (block). Also determines the number of
speculative tokens scheduled per step."""
max_denoising_steps: int | None = None
"""Maximum number of denoising iterations per canvas block.
If not set, read from the model's generation_config.json."""
+5
View File
@@ -1546,6 +1546,11 @@ class ModelConfig:
"""Extract the HF encoder/decoder model flag."""
return is_encoder_decoder(self.hf_config)
@cached_property
def is_diffusion(self) -> bool:
"""Detect discrete diffusion (dLLM) models from HF config."""
return getattr(self.hf_config, "canvas_length", None) is not None
@property
def uses_alibi(self) -> bool:
cfg = self.hf_text_config
+13 -6
View File
@@ -31,6 +31,7 @@ from .attention import AttentionConfig
from .cache import CacheConfig
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
from .device import DeviceConfig
from .diffusion import DiffusionConfig
from .ec_transfer import ECTransferConfig
from .kernel import KernelConfig
from .kv_events import KVEventsConfig
@@ -323,6 +324,9 @@ class VllmConfig:
"""LoRA configuration."""
speculative_config: SpeculativeConfig | None = None
"""Speculative decoding configuration."""
diffusion_config: DiffusionConfig | None = None
"""Diffusion LLM (dLLM) configuration."""
structured_outputs_config: StructuredOutputsConfig = Field(
default_factory=StructuredOutputsConfig
)
@@ -511,6 +515,11 @@ class VllmConfig:
and self.speculative_config.num_speculative_tokens is not None
):
return self.speculative_config.num_speculative_tokens
if (
self.diffusion_config is not None
and self.diffusion_config.canvas_length is not None
):
return self.diffusion_config.canvas_length
return 0
@property
@@ -519,6 +528,9 @@ class VllmConfig:
if use_v2_model_runner is not None:
return use_v2_model_runner
if self.model_config is not None and self.model_config.is_diffusion:
return True
if not self._is_default_v2_model_runner_model():
return False
@@ -1654,12 +1666,7 @@ class VllmConfig:
self.compilation_config.max_cudagraph_capture_size
)
if max_cudagraph_capture_size is None:
decode_query_len = 1
if (
self.speculative_config
and self.speculative_config.num_speculative_tokens
):
decode_query_len += self.speculative_config.num_speculative_tokens
decode_query_len = 1 + self.num_speculative_tokens
max_cudagraph_capture_size = min(
self.scheduler_config.max_num_seqs * decode_query_len * 2, 512
)
+16
View File
@@ -38,6 +38,7 @@ from vllm.config import (
CompilationConfig,
ConfigType,
DeviceConfig,
DiffusionConfig,
ECTransferConfig,
EPLBConfig,
KernelConfig,
@@ -616,6 +617,7 @@ class EngineArgs:
spec_method: str | None = None
spec_model: str | None = None
spec_tokens: int | None = None
diffusion_config: dict[str, Any] | None = None
show_hidden_metrics_for_version: str | None = (
ObservabilityConfig.show_hidden_metrics_for_version
@@ -1473,6 +1475,10 @@ class EngineArgs:
vllm_group.add_argument(
"--spec-tokens", **speculative_kwargs["num_speculative_tokens"]
)
vllm_kwargs["diffusion_config"]["type"] = optional_type(json.loads)
vllm_group.add_argument(
"--diffusion-config", "-dc", **vllm_kwargs["diffusion_config"]
)
vllm_group.add_argument(
"--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
)
@@ -1702,6 +1708,14 @@ class EngineArgs:
)
return SpeculativeConfig(**self.speculative_config)
def create_diffusion_config(self) -> DiffusionConfig | None:
if self.diffusion_config is None:
return None
cfg = self.diffusion_config
if isinstance(cfg, str):
cfg = json.loads(cfg)
return DiffusionConfig(**cfg)
def create_engine_config(
self,
usage_context: UsageContext | None = None,
@@ -2016,6 +2030,7 @@ class EngineArgs:
target_model_config=model_config,
target_parallel_config=parallel_config,
)
diffusion_config = self.create_diffusion_config()
self._set_default_max_num_seqs_and_batched_tokens_args(
usage_context,
@@ -2243,6 +2258,7 @@ class EngineArgs:
kernel_config=kernel_config,
lora_config=lora_config,
speculative_config=speculative_config,
diffusion_config=diffusion_config,
structured_outputs_config=self.structured_outputs_config,
observability_config=observability_config,
compilation_config=compilation_config,
@@ -188,6 +188,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
def _supports_activation(activation: MoEActivation) -> bool:
return activation in [
MoEActivation.SILU,
MoEActivation.GELU_TANH,
MoEActivation.RELU2_NO_MUL,
MoEActivation.SWIGLUOAI,
]
@@ -267,6 +268,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
activation_str_to_value_map = {
MoEActivation.SILU: ActivationType.Swiglu, # This is the default
MoEActivation.GELU_TANH: ActivationType.Geglu,
MoEActivation.SWIGLUOAI: ActivationType.Swiglu, # gpt-oss alias
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
}
@@ -142,6 +142,7 @@ class TrtLlmNvFp4ExpertsBase:
MoEActivation.SILU,
MoEActivation.RELU2_NO_MUL,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
]
@staticmethod
@@ -34,6 +34,7 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType"
MoEActivation.GELU_NO_MUL: ActivationType.Gelu,
MoEActivation.SILU: ActivationType.Swiglu,
MoEActivation.GELU: ActivationType.Geglu,
MoEActivation.GELU_TANH: ActivationType.Geglu,
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
}
return ACTIVATION_TO_FI_ACTIVATION[activation]
+55
View File
@@ -105,6 +105,60 @@ class Gemma4Config(VerifyAndUpdateConfig):
)
class DiffusionGemmaModelForBlockDiffusionConfig(VerifyAndUpdateConfig):
@classmethod
def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None:
"""Set up the diffusion config and defaults for DiffusionGemma.
Auto-creates DiffusionConfig from the HF config when the user
didn't pass ``--diffusion-config``. Diffusion sampling params are
read straight from generation_config.json at sampler-build time
(see DiffusionGemma's custom_sampler), not injected here.
"""
# Inherit Gemma4's attention backend selection (FA4 on Hopper,
# TRITON_ATTN fallback for heterogeneous head dims).
Gemma4Config.verify_and_update_config(vllm_config)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
attention_config = vllm_config.attention_config
if attention_config.backend == AttentionBackendEnum.FLASHINFER:
raise ValueError(
"FlashInfer does not support DiffusionGemma's mixed "
"causal/bidirectional attention. Use --attention-backend "
"FLASH_ATTN or TRITON_ATTN instead."
)
if attention_config.backend is None and not attention_config.use_non_causal:
attention_config.use_non_causal = True
logger.info(
"DiffusionGemma uses mixed causal/bidirectional attention "
"within a batch; setting use_non_causal=True to exclude "
"FlashInfer from auto-selection."
)
# Auto-create DiffusionConfig from HF config if not provided.
if vllm_config.diffusion_config is None:
from vllm.config.diffusion import DiffusionConfig
hf_config = vllm_config.model_config.hf_config
canvas_length = getattr(hf_config, "canvas_length", 256)
vllm_config.diffusion_config = DiffusionConfig(
canvas_length=canvas_length,
)
# The diffusion sampler materializes [num_seqs, canvas_length, vocab]
# fp32 transients, so concurrency is memory-bound (>8 OOMs a single H200).
# Default to 8 when the user didn't pass --max-num-seqs.
# We can't see the original None here (the engine already filled a generic
# default), so use >= DEFAULT_MAX_NUM_SEQS as a proxy, (the default is much
# larger than any deliberate value for this model)
from vllm.config.scheduler import SchedulerConfig
sc = vllm_config.scheduler_config
if sc is not None and sc.max_num_seqs >= SchedulerConfig.DEFAULT_MAX_NUM_SEQS:
sc.max_num_seqs = 8
class DeepseekV4ForCausalLMConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -591,6 +645,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
"Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLMoeForConditionalGenerationConfig, # noqa: E501
"FalconMambaForCausalLM": MambaModelConfig,
"Gemma3TextModel": Gemma3TextModelConfig,
File diff suppressed because it is too large Load Diff
+1 -3
View File
@@ -725,10 +725,8 @@ class Gemma4DecoderLayer(nn.Module):
if self.enable_moe_block:
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states)
# Router and MoE experts see the residual (pre-MLP state),
# matching the HF transformers forward path
router_logits = self.router(residual)
hidden_states_2 = self.pre_feedforward_layernorm_2(residual)
router_logits = self.router(residual)
hidden_states_2 = self.moe(hidden_states_2, router_logits)
hidden_states_2 = self.post_feedforward_layernorm_2(hidden_states_2)
+4
View File
@@ -400,6 +400,10 @@ _MULTIMODAL_MODELS = {
"gemma3n_mm",
"Gemma3nForConditionalGeneration",
),
"DiffusionGemmaForBlockDiffusion": (
"diffusion_gemma",
"DiffusionGemmaForConditionalGeneration",
),
"Gemma4ForConditionalGeneration": ("gemma4_mm", "Gemma4ForConditionalGeneration"),
"Gemma4UnifiedForConditionalGeneration": (
"gemma4_unified",
+143 -37
View File
@@ -20,9 +20,11 @@ import json
from collections.abc import Sequence
import regex as re
from openai.types.responses import ToolChoiceFunction
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
@@ -343,6 +345,9 @@ class Gemma4ToolParser(ToolParser):
tool parsers.
"""
# Gemma4 emits native special-token tool calls, not generic JSON calls.
supports_required_and_named = False
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
@@ -390,6 +395,23 @@ class Gemma4ToolParser(ToolParser):
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
if request.tools:
tc = request.tool_choice
if tc == "required" or isinstance(
tc,
(ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
):
# Do NOT call super().adjust_request() for required/named tool
# choice. The base implementation injects a JSON-array
# `structured_outputs` schema and forces xgrammar guided
# decoding, which conflicts with Gemma4's native
# `<|tool_call>call:...` (non-JSON) tool syntax and crashes
# EngineCore under MTP spec decode. The streaming/extraction
# parser already handles the native output, so guided decoding
# is skipped here (mirrors the GLM4 precedent).
if request.tool_choice != "none":
request.skip_special_tokens = False
return request
request = super().adjust_request(request)
if request.tools and request.tool_choice != "none":
# Don't skip special tokens — <|tool_call> etc. are needed for
@@ -549,22 +571,40 @@ class Gemma4ToolParser(ToolParser):
return DeltaMessage(content=delta_text)
return None
# Case 2: Starting a new tool call
if start_count > prev_start_count and start_count > end_count:
self.current_tool_id += 1
# Case 2: One or more new tool calls started in this delta.
# A single delta can batch several complete calls, so advance the
# tool id once per newly-seen start token and allocate a tracking
# slot for each.
if start_count > prev_start_count:
num_new = start_count - prev_start_count
for _ in range(num_new):
self.current_tool_id += 1
self.streamed_args_for_tool.append("")
self.prev_tool_call_arr.append({})
self.current_tool_name_sent = False
self.streamed_args_for_tool.append("")
self.prev_tool_call_arr.append({})
logger.debug("Starting new tool call %d", self.current_tool_id)
# Don't return yet — fall through to try parsing if there's
# content after <|tool_call> in this same delta
# (but usually it's just the token itself, so return None)
if len(delta_text) <= len(self.tool_call_start_token):
logger.debug(
"Started %d new tool call(s); current_tool_id=%d",
num_new,
self.current_tool_id,
)
# Don't return yet if this delta also contains call payload or
# the end marker; backends can batch one or more complete tool
# calls into a single streaming chunk. Only wait for more text
# when the delta is just the start token itself.
if start_count > end_count and len(delta_text) <= len(
self.tool_call_start_token
):
return None
# Case 3: Tool call just ended
# Case 3: One or more tool calls just ended (possibly several in a
# single batched delta) — drain every newly-completed call.
if end_count > prev_end_count:
return self._handle_tool_call_end(current_text)
return self._handle_tool_call_end(
current_text,
prev_end_count=prev_end_count,
end_count=end_count,
start_count=start_count,
)
# Case 4: In the middle of a tool call — parse partial content
if start_count > end_count:
@@ -652,45 +692,111 @@ class Gemma4ToolParser(ToolParser):
return None
def _handle_tool_call_end(self, current_text: str) -> DeltaMessage | None:
"""Handle streaming when a tool call has just completed.
def _handle_tool_call_end(
self,
current_text: str,
prev_end_count: int,
end_count: int,
start_count: int,
) -> DeltaMessage | None:
"""Handle streaming when one or more tool calls have just completed.
Performs a final parse of the complete tool call and flushes
any remaining un-streamed argument fragments.
A single streaming delta can batch several complete tool calls
(``<|tool_call>...<tool_call|><|tool_call>...<tool_call|>``). Every
call whose ``<tool_call|>`` end marker arrived in this delta — i.e.
those with index in ``[prev_end_count, end_count)`` — is drained and
emitted, with one ``DeltaToolCall`` per call in a single
``DeltaMessage`` (this matches the OpenAI streaming wire format, and
the serving layer iterates over ``delta.tool_calls``).
Per call:
* If the function name was already streamed incrementally (the
token-by-token path), only the remaining argument fragment is
flushed as a diff.
* If the call is seen complete for the first time in this delta (the
batched-complete path), the id + name + full arguments JSON are
emitted exactly once.
"""
if self.current_tool_id < 0 or self.current_tool_id >= len(
self.prev_tool_call_arr
):
logger.debug(
"Tool call end detected but no active tool call (current_tool_id=%d)",
self.current_tool_id,
)
# Parse the complete tool calls using regex for accuracy.
all_matches = self.tool_call_regex.findall(current_text)
if not all_matches:
logger.debug("Tool call end detected but no complete tool call parsed yet.")
return None
# Parse the complete tool call using regex for accuracy
all_matches = self.tool_call_regex.findall(current_text)
if self.current_tool_id < len(all_matches):
_, args_str = all_matches[self.current_tool_id]
deltas: list[DeltaToolCall] = []
for idx in range(prev_end_count, end_count):
if idx >= len(all_matches):
break
# Ensure the tracking arrays have a slot for this index (defensive;
# Case 2 normally allocates these when the start token arrives).
while len(self.prev_tool_call_arr) <= idx:
self.prev_tool_call_arr.append({})
self.streamed_args_for_tool.append("")
func_name, args_str = all_matches[idx]
final_args = _parse_gemma4_args(args_str)
final_args_json = json.dumps(final_args, ensure_ascii=False)
prev_streamed = self.streamed_args_for_tool[self.current_tool_id]
if len(final_args_json) > len(prev_streamed):
diff = final_args_json[len(prev_streamed) :]
self.streamed_args_for_tool[self.current_tool_id] = final_args_json
self.prev_tool_call_arr[self.current_tool_id]["arguments"] = final_args
# The name is sent exactly once per call. We track that via the
# per-call entry in prev_tool_call_arr (set either by the middle
# path or by the batched-complete branch below), which is robust
# even when several calls are drained in one delta.
name_already_sent = bool(self.prev_tool_call_arr[idx].get("name"))
return DeltaMessage(
tool_calls=[
if not name_already_sent:
# Batched-complete call: emit id + name + full arguments once.
self.streamed_args_for_tool[idx] = final_args_json
self.prev_tool_call_arr[idx] = {
"name": func_name,
"arguments": final_args,
}
deltas.append(
DeltaToolCall(
index=idx,
type="function",
id=make_tool_call_id(),
function=DeltaFunctionCall(
name=func_name, arguments=final_args_json
).model_dump(exclude_none=True),
)
)
else:
# Incrementally-streamed call: flush the remaining argument
# tail that was withheld during the middle phase.
prev_streamed = self.streamed_args_for_tool[idx]
if len(final_args_json) > len(prev_streamed):
diff = final_args_json[len(prev_streamed) :]
self.streamed_args_for_tool[idx] = final_args_json
self.prev_tool_call_arr[idx]["arguments"] = final_args
deltas.append(
DeltaToolCall(
index=self.current_tool_id,
index=idx,
function=DeltaFunctionCall(arguments=diff).model_dump(
exclude_none=True
),
)
]
)
)
# Advance streaming state past the calls completed in this delta. If a
# further tool call is still being accumulated (start without a
# matching end), point current_tool_id at it so the middle path can
# stream its arguments next; otherwise settle on the last completed
# call.
if start_count > end_count:
self.current_tool_id = end_count
while len(self.prev_tool_call_arr) <= self.current_tool_id:
self.prev_tool_call_arr.append({})
self.streamed_args_for_tool.append("")
self.current_tool_name_sent = bool(
self.prev_tool_call_arr[self.current_tool_id].get("name")
)
else:
self.current_tool_id = end_count - 1
self.current_tool_name_sent = True
if deltas:
return DeltaMessage(tool_calls=deltas)
return None
def _emit_argument_diff(self, raw_args_str: str) -> DeltaMessage | None:
+1
View File
@@ -87,6 +87,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
cosmos3_omni="Cosmos3Config",
diffusion_gemma="DiffusionGemmaConfig",
deepseek_vl_v2="DeepseekVLV2Config",
deepseek_v32="DeepseekV3Config",
deepseek_v4="DeepseekV4Config",
@@ -26,6 +26,8 @@ _CLASS_TO_MODULE: dict[str, str] = {
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
"Cosmos3Config": "vllm.transformers_utils.configs.cosmos3",
"DiffusionGemmaConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DiffusionGemmaTextConfig": "vllm.transformers_utils.configs.diffusion_gemma",
"DeepseekVLV2Config": "vllm.transformers_utils.configs.deepseek_vl2",
"DeepseekV4Config": "vllm.transformers_utils.configs.deepseek_v4",
"DotsOCRConfig": "vllm.transformers_utils.configs.dotsocr",
@@ -97,6 +99,8 @@ __all__ = [
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
"Cosmos3Config",
"DiffusionGemmaConfig",
"DiffusionGemmaTextConfig",
"DeepseekVLV2Config",
"DeepseekV3Config",
"DeepseekV4Config",
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from transformers import PretrainedConfig
from transformers.models.gemma4.configuration_gemma4 import Gemma4VisionConfig
def _init_text_config(self: PretrainedConfig, **kwargs: Any) -> None:
PretrainedConfig.__init__(self, **kwargs)
# DiffusionGemma always uses MoE and K=V sharing for full_attention
# layers. The HF reference removed these config fields entirely.
if getattr(self, "num_experts", None):
self.enable_moe_block = True
self.attention_k_eq_v = True
class DiffusionGemmaTextConfig(PretrainedConfig):
model_type = "diffusion_gemma_text"
def __init__(self, **kwargs: Any):
_init_text_config(self, **kwargs)
class DiffusionGemmaConfig(PretrainedConfig):
model_type = "diffusion_gemma"
def __init__(
self,
text_config: dict[str, Any] | None = None,
canvas_length: int = 256,
self_conditioning_size: int | None = None,
**kwargs: Any,
):
self.text_config = DiffusionGemmaTextConfig(**(text_config or {}))
self.canvas_length = canvas_length
self.self_conditioning_size = self_conditioning_size
vision_config = kwargs.pop("vision_config", None)
if isinstance(vision_config, dict):
self.vision_config = Gemma4VisionConfig(**vision_config)
else:
self.vision_config = vision_config
self.audio_config = None
PretrainedConfig.__init__(self, **kwargs)
@@ -582,6 +582,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"cohere_asr": CohereAsrModelArchConfigConvertor,
"dbrx": DbrxModelArchConfigConvertor,
"deepseek_mtp": DeepSeekMTPModelArchConfigConvertor,
"diffusion_gemma_text": Gemma4ModelArchConfigConvertor,
"ernie_mtp": ErnieMTPModelArchConfigConvertor,
"falcon": FalconModelArchConfigConvertor,
"falcon_mamba": MambaModelArchConfigConvertor,
+4 -2
View File
@@ -387,7 +387,7 @@ class CommonAttentionMetadata:
block_table_tensor: torch.Tensor
slot_mapping: torch.Tensor
causal: bool = True
causal: bool | torch.Tensor = True
# Needed by FastPrefillAttentionBuilder
logits_indices_padded: torch.Tensor | None = None
@@ -497,7 +497,9 @@ class CommonAttentionMetadata:
max_seq_len=self.max_seq_len,
block_table_tensor=self.block_table_tensor[:num_actual_reqs],
slot_mapping=self.slot_mapping[:num_actual_tokens],
causal=self.causal,
causal=self.causal[:num_actual_reqs]
if isinstance(self.causal, torch.Tensor)
else self.causal,
logits_indices_padded=self.logits_indices_padded,
num_logits_indices=self.num_logits_indices,
encoder_seq_lens=maybe_slice_reqs(self.encoder_seq_lens),
+6
View File
@@ -131,6 +131,12 @@ def get_flash_attn_version(
and head_size != head_size_v
):
upgrade_reason = "Diff-KV with sinks"
elif (
vllm_config is not None
and vllm_config.model_config is not None
and vllm_config.model_config.is_diffusion
):
upgrade_reason = "Per-sequence causal (dynamic_causal) requires FA4"
if upgrade_reason:
logger.info_once(
"%s: upgrading FlashAttention 3 -> 4",
+35 -3
View File
@@ -267,7 +267,7 @@ class FlashAttentionMetadata:
prefix_scheduler_metadata: torch.Tensor | None = None
max_num_splits: int = 0
causal: bool = True
causal: bool | torch.Tensor = True
# PrefixLM bidirectional ranges for multimodal tokens.
# Shape: (num_seqs, max_ranges, 2) int32, [start, end] per range.
@@ -570,6 +570,9 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad
self.scheduler_metadata[n:] = 0
scheduler_metadata = self.scheduler_metadata[:n]
if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32:
causal = causal.to(torch.int32)
attn_metadata = FlashAttentionMetadata(
num_actual_tokens=num_actual_tokens,
max_query_len=max_query_len,
@@ -824,18 +827,46 @@ class FlashAttentionImpl(AttentionImpl):
if self.sliding_window is not None
else None
)
causal = attn_metadata.causal
is_dynamic_causal = isinstance(causal, torch.Tensor)
# For non-causal (bidirectional) attention, make the
# sliding window symmetric so queries attend in both
# directions.
if (
sliding_window_size is not None
and sliding_window_size[1] == 0
and (is_dynamic_causal or causal is False)
):
sliding_window_size = [
sliding_window_size[0],
sliding_window_size[0],
]
mm_prefix_ranges = attn_metadata.mm_prefix_range_tensor
mm_mask_mod = None
mm_aux = None
if (
mm_prefix_ranges is not None
and attn_metadata.causal
and not is_dynamic_causal
and causal is True
and self.vllm_flash_attn_version == 4
):
max_ranges = mm_prefix_ranges.shape[1]
mm_mask_mod = _make_mm_prefix_mask_mod(max_ranges)
mm_aux = [mm_prefix_ranges]
dynamic_causal = None
if isinstance(causal, torch.Tensor):
if self.vllm_flash_attn_version != 4:
raise NotImplementedError(
"Per-sequence causal requires FA4. Current version: "
f"FA{self.vllm_flash_attn_version}"
)
dynamic_causal = causal
causal = False
flash_attn_varlen_func(
q=query[:num_actual_tokens],
k=key_cache,
@@ -846,7 +877,7 @@ class FlashAttentionImpl(AttentionImpl):
seqused_k=seqused_k,
max_seqlen_k=max_seqlen_k,
softmax_scale=self.scale,
causal=attn_metadata.causal,
causal=causal,
alibi_slopes=self.alibi_slopes,
window_size=sliding_window_size,
block_table=block_table,
@@ -856,6 +887,7 @@ class FlashAttentionImpl(AttentionImpl):
q_descale=q_descale,
k_descale=k_descale,
v_descale=v_descale,
dynamic_causal=dynamic_causal,
num_splits=attn_metadata.max_num_splits,
s_aux=self.sinks,
mask_mod=mm_mask_mod,
+8 -1
View File
@@ -79,6 +79,8 @@ class TritonAttentionMetadata:
softmax_segm_max: torch.Tensor
softmax_segm_expsum: torch.Tensor
causal: bool | torch.Tensor
# For cascade attention.
use_cascade: bool
common_prefix_len: int
@@ -219,6 +221,7 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet
seq_lens=seq_lens,
block_table=block_table_tensor,
slot_mapping=slot_mapping,
causal=common_attn_metadata.causal,
use_cascade=use_cascade,
common_prefix_len=common_prefix_len,
cu_prefix_query_lens=cu_prefix_query_lens,
@@ -271,6 +274,10 @@ class TritonAttentionBackend(AttentionBackend):
forward_includes_kv_cache_update: bool = False
@classmethod
def supports_non_causal(cls) -> bool:
return True
@staticmethod
def get_name() -> str:
return "TRITON_ATTN"
@@ -619,7 +626,7 @@ class TritonAttentionImpl(AttentionImpl):
seqused_k=seqused_k,
max_seqlen_k=max_seqlen_k,
softmax_scale=self.scale,
causal=True,
causal=attn_metadata.causal,
alibi_slopes=self.alibi_slopes,
use_alibi_sqrt=self.use_alibi_sqrt,
window_size=self.sliding_window,
@@ -153,6 +153,8 @@ def compute_tile_loop_bounds(
SLIDING_WINDOW: tl.constexpr,
USE_MM_PREFIX: tl.constexpr,
IS_3D: tl.constexpr,
USE_CAUSAL: tl.constexpr = True,
USE_PER_SEQ_CAUSAL: tl.constexpr = False,
CHUNK_LOOKBACK: tl.constexpr = -1,
CHUNK_SIZE: tl.constexpr = -1,
):
@@ -163,10 +165,11 @@ def compute_tile_loop_bounds(
1. Longest prefix spanned by any query token in this q-block.
Clamped to ``seq_len`` (causal) or extended to it when
mm_prefix is active (bidirectional ranges can reach past the
causal prefix).
mm_prefix is active or non-causal sequences need the full
sequence.
2. Sliding-window pruning: narrows ``[tile_start, tile_end)`` to
only tiles that can contain an allowed key under SWA.
For non-causal sequences, the window extends in both directions.
3. 3D scoping: when ``IS_3D`` is True, further narrows to the
segment's slice via ``(segm_idx * tiles_per_segment,
(segm_idx + 1) * tiles_per_segment)``.
@@ -179,9 +182,10 @@ def compute_tile_loop_bounds(
+ (BLOCK_M - 1) // num_queries_per_kv
+ 1
)
if USE_MM_PREFIX:
# image bidirectional attention ranges require a full range
# including q_block padding to make sure doc mask is correct
if USE_MM_PREFIX or USE_PER_SEQ_CAUSAL or (not USE_CAUSAL):
# Non-causal or mixed batches need the full sequence range.
# Per-element masking in compute_kv_seq_mask handles the
# actual causal/non-causal boundary per sequence.
max_seq_prefix_len = tl.maximum(max_seq_prefix_len, seq_len)
else:
max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len)
@@ -207,12 +211,17 @@ def compute_tile_loop_bounds(
# [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi]
q_abs = context_len + qpos_lo
if CHUNK_LOOKBACK > -1:
# Chunked attention: align lower bound to the start of the
# lookback'th previous chunk.
first_allowed_key = ((q_abs // CHUNK_SIZE) - CHUNK_LOOKBACK) * CHUNK_SIZE
else:
first_allowed_key = q_abs - SLIDING_WINDOW + 1
last_allowed_key = context_len + qpos_hi
if USE_PER_SEQ_CAUSAL or (not USE_CAUSAL):
# Non-causal: keys can be AHEAD of query within the window
last_allowed_key = tl.minimum(
context_len + qpos_hi + SLIDING_WINDOW - 1,
seq_len - 1,
)
else:
last_allowed_key = context_len + qpos_hi
# Convert to tile indices and clamp
tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE)
tile_end = tl.minimum((last_allowed_key // TILE_SIZE) + 1, num_tiles)
@@ -262,10 +271,14 @@ def compute_kv_seq_mask(
query_abs_pos,
seq_offset,
seq_idx,
seq_len,
mm_prefix_range_ptr,
SLIDING_WINDOW: tl.constexpr,
USE_MM_PREFIX: tl.constexpr,
MAX_MM_RANGES: tl.constexpr,
USE_CAUSAL: tl.constexpr = True,
USE_PER_SEQ_CAUSAL: tl.constexpr = False,
per_seq_causal_ptr=None,
CHUNK_LOOKBACK: tl.constexpr = -1,
CHUNK_SIZE: tl.constexpr = -1,
):
@@ -279,9 +292,23 @@ def compute_kv_seq_mask(
Chunked attention takes precedence over sliding window when both
are non-default — the launcher zeros ``CHUNK_LOOKBACK`` whenever
sliding window is disabled.
When ``USE_PER_SEQ_CAUSAL`` is set, each sequence carries its own
causal flag via ``per_seq_causal_ptr``; non-causal sequences use a
simple ``key < seq_len`` bound instead. ``USE_CAUSAL=False``
disables causal masking entirely.
"""
# Compute attention mask: causal by default (key <= query)
seq_mask = seq_offset[None, :] <= query_abs_pos
if USE_PER_SEQ_CAUSAL:
is_causal = tl.load(per_seq_causal_ptr + seq_idx)
seq_mask = tl.where(
is_causal,
seq_offset[None, :] <= query_abs_pos,
seq_offset[None, :] < seq_len,
)
elif USE_CAUSAL:
seq_mask = seq_offset[None, :] <= query_abs_pos
else:
seq_mask = seq_offset[None, :] < seq_len
# Apply sliding window / chunked attention to base mask
# BEFORE mm_prefix OR.
@@ -293,7 +320,15 @@ def compute_kv_seq_mask(
<= CHUNK_LOOKBACK
)
elif SLIDING_WINDOW > 0:
seq_mask = seq_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW)
sw_left = (query_abs_pos - seq_offset) < SLIDING_WINDOW
if USE_PER_SEQ_CAUSAL:
sw_right = (seq_offset[None, :] - query_abs_pos) < SLIDING_WINDOW
seq_mask = seq_mask & tl.where(is_causal, sw_left, sw_left & sw_right)
elif not USE_CAUSAL:
sw_right = (seq_offset[None, :] - query_abs_pos) < SLIDING_WINDOW
seq_mask = seq_mask & sw_left & sw_right
else:
seq_mask = seq_mask & sw_left
# PrefixLM: extend mask with bidirectional ranges for multimodal tokens.
# Applied AFTER sliding window so mm_prefix ranges override SW restriction.
@@ -215,6 +215,9 @@ def kernel_unified_attention(
USE_SOFTCAP: tl.constexpr, # bool
USE_SINKS: tl.constexpr, # bool
SLIDING_WINDOW: tl.constexpr, # int
USE_CAUSAL: tl.constexpr, # bool
USE_PER_SEQ_CAUSAL: tl.constexpr, # bool
per_seq_causal_ptr, # [num_seqs] bool, or None
USE_MM_PREFIX: tl.constexpr, # bool
MAX_MM_RANGES: tl.constexpr, # int
mm_prefix_range_ptr,
@@ -389,6 +392,8 @@ def kernel_unified_attention(
SLIDING_WINDOW,
USE_MM_PREFIX,
IS_3D,
USE_CAUSAL,
USE_PER_SEQ_CAUSAL,
CHUNK_LOOKBACK,
CHUNK_SIZE,
)
@@ -493,10 +498,14 @@ def kernel_unified_attention(
query_abs_pos,
seq_offset,
seq_idx,
seq_len,
mm_prefix_range_ptr,
SLIDING_WINDOW,
USE_MM_PREFIX,
MAX_MM_RANGES,
USE_CAUSAL,
USE_PER_SEQ_CAUSAL,
per_seq_causal_ptr,
CHUNK_LOOKBACK,
CHUNK_SIZE,
)
@@ -532,11 +541,19 @@ def kernel_unified_attention(
if SLIDING_WINDOW:
qpos_lo = q_block_local_idx * BLOCK_Q
V = tl.where(
(context_len + qpos_lo - seq_offset[:, None]) < SLIDING_WINDOW,
V,
0.0,
)
dist = context_len + qpos_lo - seq_offset[:, None]
if USE_PER_SEQ_CAUSAL:
is_causal_seq = tl.load(per_seq_causal_ptr + seq_idx)
sw_mask_v = tl.where(
is_causal_seq,
dist < SLIDING_WINDOW,
(dist < SLIDING_WINDOW) & (dist > -SLIDING_WINDOW),
)
elif USE_CAUSAL:
sw_mask_v = dist < SLIDING_WINDOW
else:
sw_mask_v = (dist < SLIDING_WINDOW) & (dist > -SLIDING_WINDOW)
V = tl.where(sw_mask_v, V, 0.0)
if USE_PER_TOKEN_HEAD_SCALES:
# Per-token-head quant: apply v_scale to P instead of V.
P_v = (P * v_token_head_scales[None, :]).to(V.dtype)
@@ -802,7 +819,11 @@ def unified_attention(
# disabling this flag costs nothing.
use_td: bool = False,
):
assert causal, "Only causal attention is supported"
# Resolve causal: bool or per-seq tensor.
use_per_seq_causal = isinstance(causal, torch.Tensor)
use_causal = bool(causal) if not use_per_seq_causal else True
per_seq_causal_ptr = causal if use_per_seq_causal else None
if sinks is not None:
assert sinks.shape[0] == q.shape[1], "Sinks must be num_query_heads size"
@@ -841,6 +862,26 @@ def unified_attention(
)
BLOCK_Q = BLOCK_M // num_queries_per_kv
# Tuned launch parameters; ``None`` lets Triton pick its defaults.
launch_num_warps: int | None = None
launch_num_stages: int | None = None
# head_size 256 with many query rows per sequence (e.g. diffusion-gemma
# bidirectional canvas passes) is prefill-shaped, but the decode-oriented
# defaults (BLOCK_Q=8, TILE=32, 4 warps) under-tile it. A wider KV tile +
# more query rows per block + 8 warps is ~2x faster on B200.
tuned_large_head = (
head_size == 256
and max_seqlen_q > 1
and num_queries_per_kv <= 16
and current_platform.is_device_capability_family(100)
)
if tuned_large_head:
BLOCK_M = 32
BLOCK_Q = BLOCK_M // num_queries_per_kv
launch_num_warps = 8
launch_num_stages = 2
# Ideally we would launch with kernel with:
# \sum_i[ceil(query_len[i] / BLOCK_Q)] blocks.
# However, it is slow to realize the query_lens on cpu.
@@ -869,6 +910,11 @@ def unified_attention(
head_size, sliding_window_val, q.element_size(), is_prefill=False
)
# Wider KV tile for the tuned large-head path (see above). Only the 2D
# path (used when max_seqlen_q > 1) reads TILE_SIZE_PREFILL.
if tuned_large_head:
TILE_SIZE_PREFILL = 128
# USE_TD requires BLOCK_SIZE % TILE_SIZE == 0 (enforced by a
# ``tl.static_assert`` in the kernel). The default prefill tile
# size (32) is larger than a common ``block_size=16``, so clamp it
@@ -964,6 +1010,12 @@ def unified_attention(
grid = (total_num_q_blocks, num_kv_heads, num_par_softmax_segments)
tile_size = TILE_SIZE_DECODE
launch_kwargs: dict[str, int] = {}
if launch_num_warps is not None:
launch_kwargs["num_warps"] = launch_num_warps
if launch_num_stages is not None:
launch_kwargs["num_stages"] = launch_num_stages
kernel_unified_attention[grid](
output_ptr=out,
segm_output_ptr=segm_output_ptr,
@@ -1002,10 +1054,13 @@ def unified_attention(
USE_QQ_BIAS=use_qq_bias,
USE_SOFTCAP=(softcap > 0),
USE_SINKS=(sinks is not None),
SLIDING_WINDOW=(1 + window_size[0]),
USE_CAUSAL=use_causal,
USE_PER_SEQ_CAUSAL=use_per_seq_causal,
per_seq_causal_ptr=per_seq_causal_ptr,
USE_MM_PREFIX=use_mm_prefix,
MAX_MM_RANGES=max_mm_ranges,
mm_prefix_range_ptr=mm_prefix_range,
SLIDING_WINDOW=(1 + window_size[0]),
stride_k_cache_0=k.stride(0),
stride_k_cache_1=k.stride(1),
stride_k_cache_2=k.stride(2),
@@ -1033,6 +1088,7 @@ def unified_attention(
CHUNK_SIZE=chunk_size,
USE_TD=use_td,
USE_TD_QO=use_td_qo,
**launch_kwargs,
)
if use_3d:
@@ -226,6 +226,7 @@ def kernel_unified_attention_diffkv(
query_abs_pos,
seq_offset,
seq_idx,
seq_len,
None, # mm_prefix_range_ptr
SLIDING_WINDOW,
False, # USE_MM_PREFIX
+7 -3
View File
@@ -27,10 +27,14 @@ class AsyncScheduler(Scheduler):
scheduler_output.pending_structured_output_tokens |= (
request.use_structured_output and request.num_output_placeholders > 0
)
# The request will generate a new token plus num_spec_tokens
# in this scheduling step.
# The request will generate num_sampled_tokens_per_step new tokens
# plus num_spec_tokens in this scheduling step. Diffusion has no AR
# bonus token (num_sampled_tokens_per_step == 0) — only the canvas
# (spec) tokens.
cur_num_spec_tokens = len(spec_decode_tokens.get(req_id, ()))
request.num_output_placeholders += 1 + cur_num_spec_tokens
request.num_output_placeholders += (
self.num_sampled_tokens_per_step + cur_num_spec_tokens
)
# Add placeholders for the new draft/spec tokens.
# We will update the actual spec token ids in the worker process.
request.spec_token_ids = self._spec_token_placeholders
+16 -6
View File
@@ -113,6 +113,10 @@ class Scheduler(SchedulerInterface):
self.kv_events_config is not None
and self.kv_events_config.enable_kv_cache_events
)
# Diffusion models may not sample any tokens for a denoising step.
self.num_sampled_tokens_per_step = (
1 if not vllm_config.model_config.is_diffusion else 0
)
# Create KVConnector for the Scheduler. Note that each Worker
# will have a corresponding KVConnector with Role=WORKER.
@@ -212,9 +216,9 @@ class Scheduler(SchedulerInterface):
speculative_config = vllm_config.speculative_config
self.use_eagle = False
self.num_spec_tokens = self.num_lookahead_tokens = 0
if speculative_config:
self.num_spec_tokens = speculative_config.num_speculative_tokens
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.num_lookahead_tokens = 0
if speculative_config is not None:
if speculative_config.use_eagle():
self.use_eagle = True
self.num_lookahead_tokens = self.num_spec_tokens
@@ -425,7 +429,10 @@ class Scheduler(SchedulerInterface):
# Make sure the input position does not exceed the max model len.
# This is necessary when using spec decoding.
num_new_tokens = min(
num_new_tokens, self.max_model_len - 1 - request.num_computed_tokens
num_new_tokens,
self.max_model_len
- request.num_computed_tokens
- self.num_sampled_tokens_per_step,
)
# Schedule encoder inputs.
@@ -1473,9 +1480,12 @@ class Scheduler(SchedulerInterface):
scheduled_spec_token_ids = (
scheduler_output.scheduled_spec_decode_tokens.get(req_id)
)
if scheduled_spec_token_ids and generated_token_ids:
if scheduled_spec_token_ids and (
generated_token_ids or self.num_sampled_tokens_per_step == 0
):
num_draft_tokens = len(scheduled_spec_token_ids)
num_accepted = len(generated_token_ids) - 1
num_sampled = self.num_sampled_tokens_per_step
num_accepted = max(len(generated_token_ids) - num_sampled, 0)
num_rejected = num_draft_tokens - num_accepted
# num_computed_tokens represents the number of tokens
# processed in the current step, considering scheduled
+1 -5
View File
@@ -34,11 +34,7 @@ class CudagraphDispatcher:
def __init__(self, vllm_config: VllmConfig):
self.vllm_config = vllm_config
self.compilation_config = vllm_config.compilation_config
self.uniform_decode_query_len = (
1
if not self.vllm_config.speculative_config
else 1 + self.vllm_config.speculative_config.num_speculative_tokens
)
self.uniform_decode_query_len = 1 + self.vllm_config.num_speculative_tokens
# Dict to store valid cudagraph dispatching keys.
self.cudagraph_keys: dict[CUDAGraphMode, set[BatchDescriptor]] = {
+14 -13
View File
@@ -156,6 +156,9 @@ class EngineCore:
hash_block_size=hash_block_size,
)
self.use_spec_decode = vllm_config.speculative_config is not None
self.check_for_draft_tokens = (
self.use_spec_decode or vllm_config.model_config.is_diffusion
)
if self.scheduler.connector is not None: # type: ignore
self.model_executor.init_kv_output_aggregator(self.scheduler.connector) # type: ignore
@@ -475,8 +478,7 @@ class EngineCore:
# When using async scheduling we can't get draft token ids in advance,
# so we update draft token ids in the worker process and don't
# need to update draft token ids here.
if not self.async_scheduling and self.use_spec_decode and model_executed:
# Take the draft token ids.
if self.check_for_draft_tokens and not self.async_scheduling and model_executed:
draft_token_ids = self.model_executor.take_draft_token_ids()
if draft_token_ids is not None:
self.scheduler.update_draft_token_ids(draft_token_ids)
@@ -575,18 +577,17 @@ class EngineCore:
# in a field and do it immediately once step_with_batch_queue is
# re-called. The latter slightly favors TTFT over TPOT/throughput.
if deferred_scheduler_output:
# If we are doing speculative decoding with structured output,
# we need to get the draft token ids from the prior step before
# we can compute the grammar bitmask for the deferred request.
if self.use_spec_decode:
# When draft tokens are used with structured output, validate them
# before computing the grammar bitmask for the deferred request.
if self.check_for_draft_tokens:
draft_token_ids = self.model_executor.take_draft_token_ids()
assert draft_token_ids is not None
# Update the draft token ids in the scheduler output to
# filter out the invalid spec tokens, which will be padded
# with -1 and skipped by the grammar bitmask computation.
self.scheduler.update_draft_token_ids_in_output(
draft_token_ids, deferred_scheduler_output
)
if draft_token_ids is not None:
# Update the draft token ids in the scheduler output to
# filter out the invalid spec tokens, which will be padded
# with -1 and skipped by the grammar bitmask computation.
self.scheduler.update_draft_token_ids_in_output(
draft_token_ids, deferred_scheduler_output
)
# We now have the tokens needed to compute the bitmask for the
# deferred request. Get the bitmask and call sample tokens.
grammar_output = self.scheduler.get_grammar_bitmask(
+7 -2
View File
@@ -110,7 +110,9 @@ class LoggingStatLogger(StatLoggerBase):
self.connector_prefix_caching_metrics = CachingMetrics()
self.mm_caching_metrics = CachingMetrics()
self.spec_decoding_logging = SpecDecodingLogging()
model_config = self.vllm_config.model_config
is_diffusion = model_config is not None and model_config.is_diffusion
self.spec_decoding_logging = SpecDecodingLogging(is_diffusion=is_diffusion)
kv_transfer_config = self.vllm_config.kv_transfer_config
self.kv_connector_logging = KVConnectorLogging(kv_transfer_config)
self.cudagraph_logging = None
@@ -436,7 +438,10 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
per_engine_labelvalues = self.per_engine_labelvalues
self.spec_decoding_prom = self._spec_decoding_cls(
vllm_config.speculative_config, labelnames, per_engine_labelvalues
vllm_config.speculative_config,
labelnames,
per_engine_labelvalues,
is_diffusion=vllm_config.model_config.is_diffusion,
)
self.kv_connector_prom = self._kv_connector_cls(
vllm_config, labelnames, per_engine_labelvalues
+106 -44
View File
@@ -53,7 +53,11 @@ class SpecDecodingLogging:
before resetting to zero.
"""
def __init__(self):
def __init__(self, is_diffusion: bool = False):
# Diffusion (dLLM) models reuse the spec-decode data path with
# overloaded semantics, so the raw spec-decode framing (drafts, bonus
# token, per-position vector) is logged with diffusion-native terms.
self.is_diffusion = is_diffusion
self.reset()
def reset(self):
@@ -85,6 +89,17 @@ class SpecDecodingLogging:
draft_throughput = num_draft_tokens / elapsed_time
accepted_throughput = num_accepted_tokens / elapsed_time
if self.is_diffusion:
self._log_diffusion(
log_fn,
num_denoising_steps=num_drafts,
num_canvas_tokens=num_draft_tokens,
num_committed_tokens=num_accepted_tokens,
committed_throughput=accepted_throughput,
)
self.reset()
return
draft_acceptance_rate = (
num_accepted_tokens / num_draft_tokens * 100
if num_draft_tokens > 0
@@ -117,6 +132,43 @@ class SpecDecodingLogging:
)
self.reset()
def _log_diffusion(
self,
log_fn,
num_denoising_steps: int,
num_canvas_tokens: int,
num_committed_tokens: int,
committed_throughput: float,
):
# Each "draft" is one denoising step that re-evaluates the canvas block
# and finalizes some of its positions.
mean_committed_per_step = (
num_committed_tokens / num_denoising_steps
if num_denoising_steps > 0
else float("nan")
)
mean_steps_per_canvas = (
num_canvas_tokens / num_committed_tokens
if num_committed_tokens > 0
else float("nan")
)
log_fn(
"DiffusionDecoding metrics: "
"Committed token throughput: %.2f tokens/s, "
"Mean denoising steps per canvas: %.2f, "
"Mean tokens committed per denoising step: %.2f, "
"Committed: %d tokens, "
"Denoising steps: %d, "
"Canvas positions evaluated: %d",
committed_throughput,
mean_steps_per_canvas,
mean_committed_per_step,
num_committed_tokens,
num_denoising_steps,
num_canvas_tokens,
)
class SpecDecodingProm:
"""Record spec decoding metrics in Prometheus.
@@ -146,56 +198,66 @@ class SpecDecodingProm:
speculative_config: SpeculativeConfig | None,
labelnames: list[str],
per_engine_labelvalues: dict[int, list[object]],
is_diffusion: bool = False,
):
self.spec_decoding_enabled = speculative_config is not None
# Diffusion (dLLM) models reuse the spec-decode counters but expose them
# under diffusion-native names; the per-position acceptance vector does
# not apply, so it is omitted.
self.is_diffusion = is_diffusion
self.spec_decoding_enabled = speculative_config is not None or is_diffusion
if not self.spec_decoding_enabled:
return
counter_drafts = self._counter_cls(
name="vllm:spec_decode_num_drafts",
documentation="Number of spec decoding drafts.",
labelnames=labelnames,
)
self.counter_spec_decode_num_drafts = create_metric_per_engine(
counter_drafts, per_engine_labelvalues
)
if is_diffusion:
counter_specs = [
("vllm:diffusion_num_denoising_steps", "Number of denoising steps."),
(
"vllm:diffusion_num_canvas_positions",
"Number of canvas positions evaluated.",
),
(
"vllm:diffusion_num_committed_tokens",
"Number of committed (finalized) tokens.",
),
]
else:
counter_specs = [
("vllm:spec_decode_num_drafts", "Number of spec decoding drafts."),
("vllm:spec_decode_num_draft_tokens", "Number of draft tokens."),
("vllm:spec_decode_num_accepted_tokens", "Number of accepted tokens."),
]
counter_draft_tokens = self._counter_cls(
name="vllm:spec_decode_num_draft_tokens",
documentation="Number of draft tokens.",
labelnames=labelnames,
)
self.counter_spec_decode_num_draft_tokens = create_metric_per_engine(
counter_draft_tokens, per_engine_labelvalues
)
counters = [
create_metric_per_engine(
self._counter_cls(name=name, documentation=doc, labelnames=labelnames),
per_engine_labelvalues,
)
for name, doc in counter_specs
]
# num_drafts/num_draft_tokens/num_accepted_tokens map onto denoising
# steps/canvas positions/committed tokens in the diffusion path.
self.counter_spec_decode_num_drafts = counters[0]
self.counter_spec_decode_num_draft_tokens = counters[1]
self.counter_spec_decode_num_accepted_tokens = counters[2]
counter_accepted_tokens = self._counter_cls(
name="vllm:spec_decode_num_accepted_tokens",
documentation="Number of accepted tokens.",
labelnames=labelnames,
)
self.counter_spec_decode_num_accepted_tokens = create_metric_per_engine(
counter_accepted_tokens, per_engine_labelvalues
)
assert speculative_config is not None
num_spec_tokens = (
speculative_config.num_speculative_tokens
if self.spec_decoding_enabled
else 0
)
pos_labelnames = labelnames + ["position"]
base_counter = self._counter_cls(
name="vllm:spec_decode_num_accepted_tokens_per_pos",
documentation="Accepted tokens per draft position.",
labelnames=pos_labelnames,
)
self.counter_spec_decode_num_accepted_tokens_per_pos: dict[
int, list[prometheus_client.Counter]
] = {
idx: [base_counter.labels(*lv, str(pos)) for pos in range(num_spec_tokens)]
for idx, lv in per_engine_labelvalues.items()
}
] = {}
if not is_diffusion:
assert speculative_config is not None
num_spec_tokens = speculative_config.num_speculative_tokens
pos_labelnames = labelnames + ["position"]
base_counter = self._counter_cls(
name="vllm:spec_decode_num_accepted_tokens_per_pos",
documentation="Accepted tokens per draft position.",
labelnames=pos_labelnames,
)
self.counter_spec_decode_num_accepted_tokens_per_pos = {
idx: [
base_counter.labels(*lv, str(pos)) for pos in range(num_spec_tokens)
]
for idx, lv in per_engine_labelvalues.items()
}
def observe(self, spec_decoding_stats: SpecDecodingStats, engine_idx: int = 0):
if not self.spec_decoding_enabled:
@@ -210,6 +272,6 @@ class SpecDecodingProm:
spec_decoding_stats.num_accepted_tokens
)
for pos, counter in enumerate(
self.counter_spec_decode_num_accepted_tokens_per_pos[engine_idx]
self.counter_spec_decode_num_accepted_tokens_per_pos.get(engine_idx, [])
):
counter.inc(spec_decoding_stats.num_accepted_tokens_per_pos[pos])
+9 -6
View File
@@ -211,11 +211,8 @@ class StructuredOutputManager:
if not structured_output_request_ids:
return None
max_num_spec_tokens = 0
if self.vllm_config.speculative_config is not None:
max_num_spec_tokens = (
self.vllm_config.speculative_config.num_speculative_tokens
)
# Covers both speculative decoding and diffusion LLMs (canvas_length).
max_num_spec_tokens = self.vllm_config.num_speculative_tokens
if self._grammar_bitmask is None:
assert self.backend is not None
@@ -277,7 +274,13 @@ class StructuredOutputManager:
state_advancements = 0
req_tokens = scheduled_spec_decode_tokens.get(req_id, ())
for token in itertools.chain(req_tokens, (-1,)):
if self.vllm_config.model_config.is_diffusion and req_tokens:
# Diffusion LLMs don't sample a bonus token after the
# scheduled positions, so don't append the -1 placeholder.
token_iter: Iterable[int] = req_tokens
else:
token_iter = itertools.chain(req_tokens, (-1,))
for token in token_iter:
self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),))
if token == -1:
# Stop advancing the grammar once we hit a padding token.
+16 -7
View File
@@ -302,6 +302,7 @@ def _combine_sampled_and_draft_tokens_kernel(
cu_num_logits_ptr,
logits_indices_ptr,
BLOCK_SIZE: tl.constexpr,
NUM_NEW_SAMPLED_TOKENS: tl.constexpr = 1,
):
batch_idx = tl.program_id(0)
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
@@ -310,7 +311,7 @@ def _combine_sampled_and_draft_tokens_kernel(
cu_num_logits_start = tl.load(cu_num_logits_ptr + batch_idx)
cu_num_logits_end = tl.load(cu_num_logits_ptr + batch_idx + 1)
num_logits = cu_num_logits_end - cu_num_logits_start
num_draft_tokens = num_logits - 1
num_draft_tokens = num_logits - NUM_NEW_SAMPLED_TOKENS
# Compute the logits indices.
block = tl.arange(0, BLOCK_SIZE)
@@ -328,9 +329,10 @@ def _combine_sampled_and_draft_tokens_kernel(
# Handling prefill tokens. No sampled or draft tokens.
return
# Write the last sampled token ID to input_ids.
last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx)
tl.store(input_ids_ptr + query_end - num_logits, last_token_id)
if NUM_NEW_SAMPLED_TOKENS > 0:
# Write the last sampled token ID to input_ids.
last_token_id = tl.load(last_sampled_tokens_ptr + req_state_idx)
tl.store(input_ids_ptr + query_end - num_logits, last_token_id)
# Write the draft tokens (if any) to input_ids.
if num_draft_tokens > 0:
@@ -356,7 +358,11 @@ def combine_sampled_and_draft_tokens(
draft_tokens: torch.Tensor,
cu_num_logits: torch.Tensor,
num_logits: int,
num_new_sampled_tokens: int = 1, # excl accepted draft tokens, a.k.a bonus tokens
) -> torch.Tensor:
assert num_new_sampled_tokens in (0, 1), (
f"num_new_sampled_tokens must be 0 or 1, got {num_new_sampled_tokens}"
)
# use idx_mapping.shape[0] for actual request count
num_reqs = idx_mapping.shape[0]
num_speculative_steps = draft_tokens.shape[-1]
@@ -377,9 +383,12 @@ def combine_sampled_and_draft_tokens(
draft_tokens.stride(0),
cu_num_logits,
logits_indices,
# NOTE(woosuk): Add 1 to ensure the block can cover the last sampled token
# in addition to all draft tokens.
BLOCK_SIZE=triton.next_power_of_2(num_speculative_steps + 1),
NUM_NEW_SAMPLED_TOKENS=num_new_sampled_tokens,
# NOTE(woosuk): Add num_new_sampled_tokens to ensure the block covers the
# last sampled token in addition to all draft tokens.
BLOCK_SIZE=triton.next_power_of_2(
num_speculative_steps + num_new_sampled_tokens
),
)
return logits_indices
+56 -51
View File
@@ -78,7 +78,6 @@ from vllm.v1.worker.gpu.input_batch import (
InputBuffers,
combine_sampled_and_draft_tokens,
expand_idx_mapping,
get_num_sampled_and_rejected,
post_update,
post_update_num_computed_tokens,
prepare_pos_seq_lens,
@@ -185,11 +184,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Speculative decoding.
self.speculator = None
self.num_speculative_steps = 0
self.use_aux_hidden_state_outputs = False
self.num_speculative_steps = vllm_config.num_speculative_tokens
if self.speculative_config is not None:
self.num_speculative_steps = self.speculative_config.num_speculative_tokens
if self.is_last_pp_rank:
self.speculator = init_speculator(self.vllm_config, self.device)
@@ -204,7 +201,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Draft tokens propagation - for spec-dec + struct outputs.
self.draft_tokens_handler = DraftTokensHandler(self.device)
self.uniform_decode_query_len = 1 + self.num_speculative_steps
# Pooling models.
self.is_pooling_model = self.model_config.runner_type == "pooling"
@@ -232,38 +228,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
device=self.device,
)
# Samplers and decode_query_len created in load_model() after
# model_state exists (num_new_sampled_tokens_per_step from ModelState).
self.sampler: Sampler | None = None
self.rejection_sampler: RejectionSampler | None = None
self.prompt_logprobs_worker: PromptLogprobsWorker | None = None
self.structured_outputs_worker: StructuredOutputsWorker | None = None
if self.is_last_pp_rank and not self.is_pooling_model:
# Initialize sampling-related workers.
# These components are only set up on the last PP rank and
# for generative (non-pooling) models.
self.sampler = Sampler(
max_num_reqs=self.max_num_reqs,
vocab_size=self.vocab_size,
device=self.device,
req_states=self.req_states,
logprobs_mode=self.model_config.logprobs_mode,
num_speculative_tokens=self.num_speculative_steps + 1,
use_fp64_gumbel=self.model_config.use_fp64_gumbel,
)
if self.speculative_config is not None:
self.rejection_sampler = RejectionSampler(
self.sampler,
self.speculative_config,
self.device,
)
self.prompt_logprobs_worker = PromptLogprobsWorker(self.max_num_reqs)
self.structured_outputs_worker = StructuredOutputsWorker(
max_num_logits=self.max_num_reqs * (self.num_speculative_steps + 1),
vocab_size=self.vocab_size,
device=self.device,
)
# For CUDA graphs, and will init cudagraph_manager after init_attn_backend.
self.decode_query_len = self.num_speculative_steps + 1
self.cudagraph_manager: ModelCudaGraphManager | None = None
# LoRA-related workers.
self.lora_state = LoraState(max_num_reqs=self.max_num_reqs)
@@ -335,6 +305,40 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.model_state = init_model_state(
self.vllm_config, self.model, self.encoder_cache, self.device
)
self.decode_query_len = (
self.num_speculative_steps
+ self.model_state.num_new_sampled_tokens_per_step
)
# Initialize samplers. Model states may override via custom_sampler().
if self.is_last_pp_rank and not self.is_pooling_model:
self.sampler = Sampler(
max_num_reqs=self.max_num_reqs,
vocab_size=self.vocab_size,
device=self.device,
req_states=self.req_states,
logprobs_mode=self.model_config.logprobs_mode,
num_speculative_tokens=self.decode_query_len,
use_fp64_gumbel=self.model_config.use_fp64_gumbel,
)
custom = self.model_state.custom_sampler(self.sampler)
if custom:
self.sampler, self.rejection_sampler = custom
elif self.speculative_config is not None:
self.rejection_sampler = RejectionSampler(
self.sampler,
self.speculative_config,
self.device,
)
self.prompt_logprobs_worker = PromptLogprobsWorker(self.max_num_reqs)
self.structured_outputs_worker = StructuredOutputsWorker(
max_num_logits=self.max_num_reqs * self.decode_query_len,
vocab_size=self.vocab_size,
device=self.device,
)
if self.is_pooling_model and self.is_last_pp_rank:
self.pooling_runner = PoolingRunner(self.model)
eplb_models_added |= self.eplb.maybe_register_model(
@@ -447,7 +451,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes(
attn_cg_support.min_cg_support,
attn_cg_support.min_cg_attn_backend,
self.uniform_decode_query_len,
self.decode_query_len,
self.parallel_config.tensor_parallel_size,
self.kv_cache_config,
self.max_num_reqs,
@@ -710,6 +714,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return cuda_graph_size
def _remove_request(self, req_id: str) -> bool:
# Call model_state.remove_request *before* req_states.remove_request
# so the model_state can still look up the slot index.
self.model_state.remove_request(req_id)
req_idx = self.req_states.remove_request(req_id)
if req_idx is None:
return False
@@ -857,16 +864,16 @@ class GPUModelRunner(LoRAModelRunnerMixin):
dtype=np.int32,
count=num_reqs,
)
num_bonus_tokens = self.model_state.num_new_sampled_tokens_per_step
total_num_draft_tokens = int(num_draft_tokens_per_req.sum())
total_num_logits = num_reqs + total_num_draft_tokens
num_logits = num_draft_tokens_per_req + 1
total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens
num_logits = num_draft_tokens_per_req + num_bonus_tokens
cu_num_logits_np = np.empty(num_reqs + 1, dtype=np.int32)
cu_num_logits_np[0] = 0
np.cumsum(num_logits, out=cu_num_logits_np[1:])
cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device)
max_expand_len = self.num_speculative_steps + 1
max_expand_len = self.decode_query_len
expanded_idx_mapping, expanded_local_pos = expand_idx_mapping(
idx_mapping, total_num_logits, cu_num_logits, max_expand_len
)
@@ -935,6 +942,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.draft_tokens,
cu_num_logits,
total_num_logits,
self.model_state.num_new_sampled_tokens_per_step,
)
# CPU upper bound on seq_lens; padded entries left at zero.
@@ -1027,8 +1035,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
grammar_output.grammar_bitmask,
)
if input_batch.num_draft_tokens == 0:
# No draft tokens (common case).
if input_batch.num_draft_tokens == 0 or self.rejection_sampler is None:
assert self.sampler is not None
sampler_output = self.sampler(logits, input_batch)
else:
@@ -1042,16 +1049,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.speculator.draft_logits,
)
# Get the number of sampled and rejected tokens.
# For chunked prefills, num_sampled and num_rejected are both 0.
num_sampled, num_rejected = get_num_sampled_and_rejected(
sampler_output.num_sampled,
input_batch.seq_lens,
input_batch.cu_num_logits,
input_batch.idx_mapping,
self.req_states.prefill_len.gpu,
)
return sampler_output, num_sampled, num_rejected
return sampler_output, sampler_output.num_sampled, sampler_output.num_rejected
def postprocess_sampled(
self,
@@ -1448,7 +1446,14 @@ class GPUModelRunner(LoRAModelRunnerMixin):
mm_inputs=mm_inputs,
)
self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens
self.draft_tokens_handler.set_draft_tokens(input_batch, draft_tokens)
if self.num_speculative_steps > 0:
# Spec-decode and diffusion LLMs both use draft tokens but the latter does
# not have a speculator (i.e. self.speculator is None)
self.draft_tokens_handler.set_draft_tokens(
input_batch,
self.req_states.draft_tokens[input_batch.idx_mapping],
)
# Post-step KV connector related operations.
kv_connector_output = self.kv_connector.post_forward(finished_req_ids)
@@ -13,6 +13,11 @@ def init_model_state(
encoder_cache: EncoderCache | None,
device: torch.device,
):
# Let the model provide its own ModelState if it defines one.
if hasattr(model, "get_model_state_cls"):
cls = model.get_model_state_cls()
return cls(vllm_config, model, encoder_cache, device)
if (
"WhisperForConditionalGeneration" in vllm_config.model_config.architectures
or "CohereAsrForConditionalGeneration" in vllm_config.model_config.architectures
@@ -53,6 +53,9 @@ class ModelState(ABC):
def add_request(self, req_index: int, new_req_data: NewRequestData) -> None:
return None
def remove_request(self, req_id: str) -> None:
return None
def apply_staged_writes(self) -> None:
return None
@@ -89,3 +92,16 @@ class ModelState(ABC):
for_capture: bool = False,
) -> dict[str, Any]:
raise NotImplementedError
def custom_sampler(self, sampler: Any) -> tuple[Any, Any] | None:
"""Wrap or replace the default sampler.
Called after model loading with the already-constructed base
``Sampler``. Return ``None`` to keep the defaults, or
``(sampler, rejection_sampler | None)`` to override.
"""
return None
num_new_sampled_tokens_per_step: int = 1
"""New tokens sampled on each decode step
(excluding accepted draft tokens, a.k.a num bonus tokens)."""
+1
View File
@@ -13,3 +13,4 @@ class SamplerOutput:
logprobs_tensors: LogprobsTensors | None
num_nans: torch.Tensor | None
num_sampled: torch.Tensor | None
num_rejected: torch.Tensor | None = None
+15 -2
View File
@@ -12,7 +12,7 @@ from vllm.v1.sample.ops.topk_topp_sampler import (
flashinfer_sample,
flashinfer_sampler_supported,
)
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.input_batch import InputBatch, get_num_sampled_and_rejected
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
from vllm.v1.worker.gpu.sample.bad_words import BadWordsState
from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
@@ -44,6 +44,7 @@ class Sampler:
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
self.use_fp64_gumbel = use_fp64_gumbel
self.req_states = req_states
self.sampling_states = SamplingStates(max_num_reqs, vocab_size)
self.penalties_state = PenaltiesState(req_states)
self.logit_bias_state = LogitBiasState(max_num_reqs, device)
@@ -118,6 +119,17 @@ class Sampler:
else:
logprobs_tensors = None
# 1 sampled token per request, except chunked-prefill requests
# (seq_len < prefill_len) which aren't done prefilling and produce no
# output token. num_rejected is always 0 here (one logit per request).
num_sampled, num_rejected = get_num_sampled_and_rejected(
input_batch.seq_lens.new_ones(input_batch.num_reqs),
input_batch.seq_lens,
input_batch.cu_num_logits,
input_batch.idx_mapping,
self.req_states.prefill_len.gpu,
)
# These are GPU tensors.
sampler_output = SamplerOutput(
# The sampled tokens are expanded to 2D tensor with shape
@@ -126,7 +138,8 @@ class Sampler:
sampled_token_ids=sampled.view(-1, 1),
logprobs_tensors=logprobs_tensors,
num_nans=num_nans,
num_sampled=input_batch.seq_lens.new_ones(input_batch.num_reqs),
num_sampled=num_sampled,
num_rejected=num_rejected,
)
return sampler_output
@@ -6,7 +6,10 @@ from vllm.config import SpeculativeConfig
from vllm.triton_utils import tl, triton
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.input_batch import (
InputBatch,
get_num_sampled_and_rejected,
)
from vllm.v1.worker.gpu.metrics.logits import get_num_nans
from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs
from vllm.v1.worker.gpu.sample.output import SamplerOutput
@@ -136,9 +139,18 @@ class RejectionSampler:
else logits,
)
num_sampled, num_rejected = get_num_sampled_and_rejected(
num_sampled,
input_batch.seq_lens,
input_batch.cu_num_logits,
input_batch.idx_mapping,
self.sampler.req_states.prefill_len.gpu,
)
return SamplerOutput(
sampled_token_ids=sampled,
logprobs_tensors=logprobs_tensors,
num_nans=num_nans,
num_sampled=num_sampled,
num_rejected=num_rejected,
)
+4
View File
@@ -35,6 +35,10 @@ class DraftTokensHandler:
self.copy_stream.wait_stream(current_stream)
with torch.cuda.stream(self.copy_stream):
self.draft_tokens_np = async_copy_to_np(draft_tokens)
# draft_tokens is a temporary allocation on the main stream and read here on
# copy_stream; without record_stream, the caching allocator may reuse its
# memory before the async copy executes.
draft_tokens.record_stream(self.copy_stream)
self.copy_event.record()
def get_draft_tokens(self) -> DraftTokenIds | None:
+13 -12
View File
@@ -30,17 +30,18 @@ def warmup_kernels(
pipeline parallel coordination.
The first iteration simulates a prefill with requests of
2 + num_spec_steps prompt tokens each. The second iteration simulates
a decode step with all requests generating 1 + num_spec_steps tokens.
decode_query_len + 1 prompt tokens each. The second iteration simulates
a decode step with all requests generating decode_query_len tokens.
"""
num_spec_steps = model_runner.num_speculative_steps
# Use 1 + num_spec_steps + 1 tokens so the prefill batch's per-request
# query length exceeds decode_query_len (= 1 + num_spec_steps), preventing
# it from being misclassified as a uniform decode batch.
prompt_len = 2 + num_spec_steps
decode_query_len = model_runner.decode_query_len
# Use decode_query_len + 1 tokens so the prefill batch's per-request query
# length exceeds decode_query_len, preventing it from being misclassified as
# a uniform decode batch.
prompt_len = decode_query_len + 1
prompt_token_ids = list(range(prompt_len))
# After prefill, decode generates 1 verified + num_spec_steps draft tokens.
decode_len = prompt_len + 1 + num_spec_steps
# After prefill, decode generates decode_query_len tokens.
decode_len = prompt_len + decode_query_len
kv_cache_groups = model_runner.kv_cache_config.kv_cache_groups
num_kv_cache_groups = len(kv_cache_groups)
@@ -57,7 +58,7 @@ def warmup_kernels(
num_reqs = min(
model_runner.scheduler_config.max_num_seqs,
model_runner.scheduler_config.max_num_batched_tokens
// max(prompt_len, 1 + num_spec_steps),
// max(prompt_len, decode_query_len),
# Reserve block 0 (null block) and ensure we have enough blocks.
max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req),
)
@@ -79,7 +80,7 @@ def warmup_kernels(
nonlocal next_block_id
return list(range(next_block_id, next_block_id := next_block_id + num_blocks))
# Step 1: Prefill all requests with 2 + num_spec_steps prompt tokens each.
# Step 1: Prefill all requests with 1 + decode_query_len prompt tokens each.
new_reqs = [
NewRequestData.from_request(
Request(req_ids[i], prompt_token_ids, sampling_params, pooling_params),
@@ -117,7 +118,7 @@ def warmup_kernels(
worker_sample_tokens(grammar_output)
# Step 2: Decode all requests with 1 + num_spec_steps tokens each.
# Step 2: Decode all requests with decode_query_len tokens each.
cached_req_data = CachedRequestData.make_empty()
cached_req_data.req_ids = list(req_ids)
cached_req_data.num_computed_tokens = [prompt_len] * num_reqs
@@ -131,7 +132,7 @@ def warmup_kernels(
decode_output = SchedulerOutput.make_empty()
decode_output.scheduled_cached_reqs = cached_req_data
decode_output.num_scheduled_tokens = {
req_id: 1 + num_spec_steps for req_id in req_ids
req_id: decode_query_len for req_id in req_ids
}
if num_spec_steps > 0:
decode_output.scheduled_spec_decode_tokens = {
@@ -209,6 +209,7 @@ def flash_attn_varlen_func(
# FA4 only
mask_mod=None,
aux_tensors=None,
dynamic_causal: "torch.Tensor | None" = None,
):
"""dropout_p should be set to 0.0 during evaluation
Supports multi-query and grouped-query attention (MQA/GQA) by passing in K, V with fewer heads
@@ -392,6 +393,7 @@ def flash_attn_varlen_func(
page_table=block_table,
softmax_scale=softmax_scale,
causal=causal,
dynamic_causal=dynamic_causal,
softcap=softcap,
window_size_left=real_window_size[0] if real_window_size[0] >= 0 else None,
window_size_right=real_window_size[1] if real_window_size[1] >= 0 else None,