From 35d6fb3187d0a5a70c43ee5b0c3e923a7cb96226 Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Tue, 29 Sep 2026 14:06:47 +0800 Subject: [PATCH 001/248] [Perf][Spec Decode] Enable fused multi-step draft decode for FlashInfer trtllm-gen (#58371) Signed-off-by: Bowen Wang Co-authored-by: Claude Opus 5.5 --- .../test_trtllm_attention_integration.py | 48 +++++++++++++++++++ vllm/v1/attention/backends/flashinfer.py | 10 ++++ 2 files changed, 58 insertions(+) diff --git a/tests/v1/attention/test_trtllm_attention_integration.py b/tests/v1/attention/test_trtllm_attention_integration.py index 42dd8f9f48..e0af9732a3 100644 --- a/tests/v1/attention/test_trtllm_attention_integration.py +++ b/tests/v1/attention/test_trtllm_attention_integration.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Integration tests for TRTLLM gen-full attention through FlashInfer.""" +import dataclasses import unittest.mock from functools import partial @@ -513,3 +514,50 @@ def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str): kv_cache_dtype="nvfp4", model_name=MODEL_NVFP4, ) + + +@torch.inference_mode() +def test_trtllm_gen_draft_decode_metadata_tracks_in_place_seq_lens( + monkeypatch: pytest.MonkeyPatch, +): + """Fused draft steps reuse the step-1 build after advancing seq_lens in + place, so its decode metadata must match the split path's step-2 rebuild.""" + monkeypatch.setattr( + "vllm.utils.flashinfer.supports_trtllm_attention", + unittest.mock.Mock(return_value=True), + ) + monkeypatch.setattr( + "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", + _mock_get_per_layer_parameters, + ) + vllm_config = create_vllm_config( + model_name=MODEL, block_size=BLOCK_SIZE, num_gpu_blocks=NUM_GPU_BLOCKS + ) + model_config = vllm_config.model_config + kv_cache_spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=model_config.get_num_kv_heads(vllm_config.parallel_config), + head_size=model_config.get_head_size(), + dtype=model_config.dtype, + ) + device = torch.device(f"{DEVICE_TYPE}:0") + with set_current_vllm_config(vllm_config): + builder = FlashInferMetadataBuilder( + kv_cache_spec, ["test_layer_0"], vllm_config, device + ) + assert builder.supports_draft_decode_metadata_update + + step1 = create_common_attn_metadata(BATCH_SPECS["decode_only"], BLOCK_SIZE, device) + reused = builder.build(common_prefix_len=0, common_attn_metadata=step1) + step1.seq_lens.add_(1) + step2 = dataclasses.replace(step1, seq_lens_cpu_upper_bound=step1.seq_lens.cpu()) + rebuilt = builder.build(common_prefix_len=0, common_attn_metadata=step2) + + assert isinstance(reused.decode, FlashInferTrtllmAPIDecode) + for field in dataclasses.fields(reused.decode): + value = getattr(reused.decode, field.name) + expected = getattr(rebuilt.decode, field.name) + if isinstance(value, torch.Tensor): + assert torch.equal(value, expected), field.name + else: + assert value == expected, field.name diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index ecc66a75ca..91ff6da3f0 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -860,6 +860,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode( vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp ) + # Fused draft steps advance seq_lens in place and keep block tables fixed, + # so one trtllm-gen decode build serves every step (not DCP-local seq_lens). + self.supports_draft_decode_metadata_update = ( + self.flashinfer_trtllm_api_decode_kernel + == FlashInferDecodeKernel.TRTLLM_GEN + and not self.use_dcp + ) self._init_reorder_batch_threshold( 1, supports_spec_as_decode=( @@ -1837,6 +1844,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): attn_metadata.decode = FIDecode(wrapper=decode_wrapper) return attn_metadata + def update_draft_decode_metadata(self, _metadata: FlashInferMetadata) -> None: + pass + def use_cascade_attention(self, *args, **kwargs) -> bool: if self.kv_cache_spec.dtype != self.vllm_config.model_config.dtype: # TODO: The cascade wrapper currently does not support setting From e05095c9e1ae37f208c1ad70b08c17f1152878fc Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 29 Sep 2026 14:23:54 +0800 Subject: [PATCH 002/248] [Rust Frontend] Feed parser tests and benchmarks attributed input (#59048) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/src/chat/tests/roundtrip.rs | 34 +++- rust/src/parser/Cargo.toml | 5 + rust/src/parser/benches/gemma4.rs | 52 +++--- rust/src/parser/benches/kimi_k3.rs | 191 +++++++++++++++++++++++ rust/src/parser/benches/utils/adapter.rs | 114 -------------- rust/src/parser/benches/utils/mod.rs | 4 +- rust/src/parser/benches/utils/unified.rs | 76 +++++++++ 7 files changed, 334 insertions(+), 142 deletions(-) create mode 100644 rust/src/parser/benches/kimi_k3.rs delete mode 100644 rust/src/parser/benches/utils/adapter.rs create mode 100644 rust/src/parser/benches/utils/unified.rs diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index a7bc24bd2a..a0ccbe59ab 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -780,10 +780,29 @@ fn decoded_completion_stream( assistant_stop_suffix, text ) })?; + // The engine returns token ids, so attribute the rendered text + // through the production decoder, then split it into small char + // chunks to exercise marker splits. + let token_ids = + tokenizer.encode(body, false).context("failed to encode text completion")?; + let mut rest = vllm_text::DecodedText::default(); + for chunk in incremental_decode_chunks( + tokenizer, + &prompt_token_ids, + &token_ids, + TOKEN_COMPLETION_CHUNK_TOKENS, + )? { + rest.append(chunk.delta); + } + ensure!( + rest.text == body, + "text completion must roundtrip through the tokenizer\nbody: {body:?}\ndecoded: {:?}", + rest.text + ); split_by_chars(body, TEXT_COMPLETION_CHUNK_CHARS) .into_iter() .map(|delta| DecodedCompletionChunk { - delta, + delta: rest.drain_prefix(delta.len()), token_ids: Vec::new(), // unused for text-level roundtrip cases }) .collect() @@ -836,7 +855,7 @@ fn decoded_completion_stream( sampling_mask: None, }); events.push(DecodedTextEvent::TextDelta { - decoded: vllm_text::DecodedText::unattributed(chunk.delta), + decoded: chunk.delta, sampled: vllm_text::SampledDelta { token_ids: chunk.token_ids, logprobs: None, @@ -851,7 +870,8 @@ fn decoded_completion_stream( /// One decoded completion chunk fed into the output processor. struct DecodedCompletionChunk { - delta: String, + /// Attributed for token-id completions, as the production decoder emits it. + delta: vllm_text::DecodedText, token_ids: Vec, } @@ -893,11 +913,11 @@ fn incremental_decode_chunks( let mut decoder = tokenizer.create_decode_stream(prompt_token_ids, false, 0); let mut chunks = Vec::new(); for chunk_token_ids in split_by_count(token_ids, chunk_size) { - let mut delta = String::new(); + let mut delta = vllm_text::DecodedText::default(); for token_id in chunk_token_ids.iter().copied() { decoder.push_token(token_id)?; while let Some(chunk) = decoder.next_chunk() { - delta.push_str(&chunk.text); + delta.append(chunk); } } chunks.push(DecodedCompletionChunk { @@ -909,10 +929,10 @@ fn incremental_decode_chunks( let (last_chunk, _) = decoder.flush(None)?; if let Some(last_chunk) = last_chunk { if let Some(delta) = chunks.last_mut() { - delta.delta.push_str(&last_chunk.text); + delta.delta.append(last_chunk); } else { chunks.push(DecodedCompletionChunk { - delta: last_chunk.text, + delta: last_chunk, token_ids: Vec::new(), }); } diff --git a/rust/src/parser/Cargo.toml b/rust/src/parser/Cargo.toml index 92d71ae227..25546cb1df 100644 --- a/rust/src/parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -81,5 +81,10 @@ name = "granite4" harness = false required-features = ["test-util"] +[[bench]] +name = "kimi_k3" +harness = false +required-features = ["test-util"] + [lints] workspace = true diff --git a/rust/src/parser/benches/gemma4.rs b/rust/src/parser/benches/gemma4.rs index 16aca3c6df..c06daeb63f 100644 --- a/rust/src/parser/benches/gemma4.rs +++ b/rust/src/parser/benches/gemma4.rs @@ -1,15 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::sync::Arc; use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; -use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; -use vllm_parser::tool::{Tool, ToolParser}; +use vllm_parser::tool::Tool; +use vllm_parser::tool::test_utils::test_tools; use vllm_parser::unified::Gemma4UnifiedParser; +use vllm_tokenizer::test_utils::TestTokenizer; mod utils; -use utils::{UnifiedToolParserAdapter, feed_parser}; +use utils::{attributed_chunks, feed_unified_parser}; const CHUNK_CHARS: usize = 7; const LONG_NORMAL_TEXT_REPEATS: usize = 2048; @@ -71,8 +73,17 @@ fn long_tool_argument_fixture() -> String { ) } -fn parser(tools: &[Tool]) -> Box { - UnifiedToolParserAdapter::::create(tools) +fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_special_token("<|tool_call>", 256) + .with_special_token("", 257) + .with_special_token("<|\"|>", 258) + .with_special_token("<|channel>", 259) + .with_special_token("", 260) +} + +fn parser(tools: &[Tool], tokenizer: &TestTokenizer) -> Gemma4UnifiedParser { + Gemma4UnifiedParser::new(tools, Arc::new(tokenizer.clone())) .expect("Gemma4 unified parser should initialize") } @@ -85,7 +96,14 @@ fn run_stream_group( expected_normal_text: &str, expected_calls_len: usize, ) { - let chunks = split_by_chars(text, chunk_chars); + let tokenizer = tokenizer(); + let chunks = attributed_chunks(&tokenizer, text, chunk_chars); + + // Check the fixture once, outside the measurement: a marker lost to + // missing attribution would silently turn the whole stream into content. + let summary = feed_unified_parser(&mut parser(tools, &tokenizer), &[], chunks.clone()); + assert_eq!(summary.normal_text, expected_normal_text, "{name}"); + assert_eq!(summary.calls_len, expected_calls_len, "{name}"); let mut group = c.benchmark_group(name); group.sample_size(50); @@ -94,23 +112,19 @@ fn run_stream_group( group.throughput(Throughput::Bytes(text.len() as u64)); group.bench_function("reuse_parser", |b| { - let mut parser = parser(tools); - b.iter(|| { - let result = feed_parser(&mut *parser, black_box(&chunks)); - debug_assert_eq!(result.0, expected_normal_text); - debug_assert_eq!(result.1, expected_calls_len); - black_box(result); - }) + let mut parser = parser(tools, &tokenizer); + b.iter_batched( + || chunks.clone(), + |chunks| black_box(feed_unified_parser(&mut parser, &[], black_box(chunks))), + BatchSize::SmallInput, + ) }); group.bench_function("create_parser", |b| { b.iter_batched( - || parser(tools), - |mut parser| { - let result = feed_parser(&mut *parser, black_box(&chunks)); - debug_assert_eq!(result.0, expected_normal_text); - debug_assert_eq!(result.1, expected_calls_len); - black_box(result); + || (parser(tools, &tokenizer), chunks.clone()), + |(mut parser, chunks)| { + black_box(feed_unified_parser(&mut parser, &[], black_box(chunks))) }, BatchSize::SmallInput, ) diff --git a/rust/src/parser/benches/kimi_k3.rs b/rust/src/parser/benches/kimi_k3.rs new file mode 100644 index 0000000000..16506a9c33 --- /dev/null +++ b/rust/src/parser/benches/kimi_k3.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::sync::Arc; +use std::time::Duration; + +use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; +use vllm_parser::unified::KimiK3UnifiedParser; +use vllm_tokenizer::Tokenizer; +use vllm_tokenizer::test_utils::TestTokenizer; + +mod utils; +use utils::{attributed_chunks, feed_unified_parser}; + +const CHUNK_CHARS: usize = 7; +const LONG_REASONING_REPEATS: usize = 2048; +const LONG_TOOL_ARGUMENT_REPEATS: usize = 256; + +/// The chat generation prompt for thinking mode ends inside the `think` +/// channel, so the output starts with reasoning. +const THINKING_PROMPT_TAIL: &str = "<|open|>think<|sep|>"; + +fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_special_token("<|open|>", 256) + .with_special_token("<|close|>", 257) + .with_special_token("<|sep|>", 258) + .with_special_token("<|end_of_msg|>", 259) +} + +fn argument(key: &str, arg_type: &str, value: &str) -> String { + format!( + "<|open|>argument key=\"{key}\" type=\"{arg_type}\"<|sep|>{value}<|close|>argument<|sep|>" + ) +} + +fn call(tool: &str, index: usize, body: &str) -> String { + format!("<|open|>call tool=\"{tool}\" index=\"{index}\"<|sep|>{body}<|close|>call<|sep|>") +} + +fn message(reasoning: &str, response: &str, calls: &str) -> String { + format!( + "{reasoning}<|close|>think<|sep|>\ + <|open|>response<|sep|>{response}<|close|>response<|sep|>\ + <|open|>tools<|sep|>{calls}<|close|>tools<|sep|>\ + <|close|>message<|sep|>" + ) +} + +fn mixed_fixture() -> String { + let calls = [ + call( + "convert", + 1, + &[ + argument("whole", "number", "114.514"), + argument("flag", "boolean", "true"), + argument("name", "string", "demo"), + argument("payload", "object", r#"{"count":42,"tags":["red","blue"]}"#), + ] + .concat(), + ), + call("update_record", 2, &argument("id", "integer", "7")), + ] + .concat(); + message( + "I should inspect the data first.", + "I will convert it.", + &calls, + ) +} + +fn long_reasoning_fixture() -> String { + let line = "Ordinary reasoning text with no K3 channel markers at all.\n"; + message(&line.repeat(LONG_REASONING_REPEATS), "Done.", "") +} + +fn long_tool_argument_fixture() -> String { + let line = "

Literal and quotes \" inside content.

\n"; + let content = line.repeat(LONG_TOOL_ARGUMENT_REPEATS); + let body = [ + argument("path", "string", "index.html"), + argument("content", "string", &content), + ] + .concat(); + message( + "Write the file.", + "I will write the file.", + &call("write_file", 1, &body), + ) +} + +fn parser(tokenizer: &TestTokenizer) -> KimiK3UnifiedParser { + KimiK3UnifiedParser::new(&[], Arc::new(tokenizer.clone())) + .expect("Kimi K3 unified parser should initialize") +} + +fn run_stream_group( + c: &mut Criterion, + name: &str, + text: &str, + expected_normal_text: &str, + expected_reasoning_len: usize, + expected_calls_len: usize, +) { + let tokenizer = tokenizer(); + let prompt_token_ids = tokenizer.encode(THINKING_PROMPT_TAIL, false).unwrap(); + let chunks = attributed_chunks(&tokenizer, text, CHUNK_CHARS); + + // Check the fixture once, outside the measurement: a marker lost to + // missing attribution would silently turn the whole stream into content. + let summary = feed_unified_parser(&mut parser(&tokenizer), &prompt_token_ids, chunks.clone()); + assert_eq!(summary.normal_text, expected_normal_text, "{name}"); + assert_eq!( + summary.reasoning_text.len(), + expected_reasoning_len, + "{name}" + ); + assert_eq!(summary.calls_len, expected_calls_len, "{name}"); + + let mut group = c.benchmark_group(name); + group.sample_size(50); + group.warm_up_time(Duration::from_millis(300)); + group.measurement_time(Duration::from_secs(2)); + group.throughput(Throughput::Bytes(text.len() as u64)); + + group.bench_function("reuse_parser", |b| { + let mut parser = parser(&tokenizer); + b.iter_batched( + || chunks.clone(), + |chunks| { + black_box(feed_unified_parser( + &mut parser, + &prompt_token_ids, + black_box(chunks), + )) + }, + BatchSize::SmallInput, + ) + }); + + group.bench_function("create_parser", |b| { + b.iter_batched( + || (parser(&tokenizer), chunks.clone()), + |(mut parser, chunks)| { + black_box(feed_unified_parser( + &mut parser, + &prompt_token_ids, + black_box(chunks), + )) + }, + BatchSize::SmallInput, + ) + }); + + group.finish(); +} + +fn bench_kimi_k3(c: &mut Criterion) { + run_stream_group( + c, + "kimi_k3/mixed_complex_tool_call", + &mixed_fixture(), + "I will convert it.", + "I should inspect the data first.".len(), + 2, + ); + + let long_reasoning = long_reasoning_fixture(); + let reasoning_len = long_reasoning.find("<|close|>think").unwrap(); + run_stream_group( + c, + "kimi_k3/long_reasoning", + &long_reasoning, + "Done.", + reasoning_len, + 0, + ); + + run_stream_group( + c, + "kimi_k3/long_tool_argument", + &long_tool_argument_fixture(), + "I will write the file.", + "Write the file.".len(), + 1, + ); +} + +criterion_group!(benches, bench_kimi_k3); +criterion_main!(benches); diff --git a/rust/src/parser/benches/utils/adapter.rs b/rust/src/parser/benches/utils/adapter.rs deleted file mode 100644 index d482a011c0..0000000000 --- a/rust/src/parser/benches/utils/adapter.rs +++ /dev/null @@ -1,114 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -use std::sync::Arc; - -use vllm_parser::tool::{Result, Tool, ToolParser, ToolParserError, ToolParserOutput}; -use vllm_parser::unified::{ - UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput, -}; -use vllm_tokenizer::{DecodedText, Tokenizer}; - -/// Tokenizer stub used by unified-parser benchmarks. -struct BenchTokenizer; - -impl Tokenizer for BenchTokenizer { - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result> { - Ok(text.chars().map(|_| u32::MAX).collect()) - } - - fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result> { - self.encode(text, false) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok("\u{FFFD}".repeat(token_ids.len())) - } - - fn token_to_id(&self, _token: &str) -> Option { - Some(u32::MAX) - } - - fn id_to_token(&self, _id: u32) -> Option { - Some("\u{FFFD}".to_string()) - } -} - -/// Bench-only adapter that exposes a unified parser through the tool-parser -/// benchmark harness. -/// -/// Returns error if the unified parser produces reasoning events. -pub struct UnifiedToolParserAdapter { - inner: Box, - _marker: std::marker::PhantomData, -} - -fn map_unified_error(error: UnifiedParserError) -> ToolParserError { - ToolParserError::ParsingFailed { - message: format!("unified parser failed: {error}"), - } -} - -fn append_unified_output( - output: UnifiedParserOutput, - tool_output: &mut ToolParserOutput, -) -> Result<()> { - for event in output.events { - match event { - UnifiedParserEvent::Text(text) => tool_output.push_text(text), - UnifiedParserEvent::ToolCall(call) => tool_output.push_call(call), - UnifiedParserEvent::Reasoning(_) => { - return Err(ToolParserError::ParsingFailed { - message: "unified parser emitted reasoning in tool-parser adapter".to_string(), - }); - } - } - } - Ok(()) -} - -impl ToolParser for UnifiedToolParserAdapter { - fn create(tools: &[Tool]) -> Result> - where - Self: Sized + 'static, - { - let inner = T::create(tools, Arc::new(BenchTokenizer)).map_err(map_unified_error)?; - Ok(Box::new(Self { - inner, - _marker: std::marker::PhantomData, - })) - } - - fn preserve_special_tokens(&self) -> bool { - self.inner.preserve_special_tokens() - } - - fn tool_call_id(&self, tool_index: usize) -> Option<&str> { - self.inner.tool_call_id(tool_index) - } - - fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { - let mut unified_output = UnifiedParserOutput::default(); - let result = self - .inner - .parse_into(DecodedText::unattributed(chunk), &mut unified_output) - .map_err(map_unified_error); - append_unified_output(unified_output, output)?; - result - } - - fn finish(&mut self) -> Result { - let unified_output = self.inner.finish().map_err(map_unified_error)?; - let mut output = ToolParserOutput::default(); - append_unified_output(unified_output, &mut output)?; - Ok(output) - } - - fn reset(&mut self) -> String { - self.inner.reset() - } -} diff --git a/rust/src/parser/benches/utils/mod.rs b/rust/src/parser/benches/utils/mod.rs index e67f5fdb02..3ca72ee527 100644 --- a/rust/src/parser/benches/utils/mod.rs +++ b/rust/src/parser/benches/utils/mod.rs @@ -6,12 +6,12 @@ #![allow(dead_code)] #![allow(unused_imports)] -mod adapter; +mod unified; -pub(super) use adapter::UnifiedToolParserAdapter; use futures::FutureExt as _; use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool}; use tool_parser::traits::ToolParser as ExternalToolParser; +pub(super) use unified::{attributed_chunks, feed_unified_parser}; use vllm_parser::tool::test_utils::collect_stream; use vllm_parser::tool::{Tool, ToolParser}; diff --git a/rust/src/parser/benches/utils/unified.rs b/rust/src/parser/benches/utils/unified.rs new file mode 100644 index 0000000000..c3be3d3236 --- /dev/null +++ b/rust/src/parser/benches/utils/unified.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::BTreeSet; + +use vllm_parser::tool::test_utils::split_by_chars; +use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; +use vllm_tokenizer::{DecodedText, Tokenizer}; + +/// Split `text` into token-attributed chunks of at most `chunk_chars` +/// characters, as the frontend streams them to a unified parser. +/// +/// The whole text is encoded with special tokens and run through the +/// incremental decoder once before splitting, so a marker cut by a chunk +/// boundary keeps its special-token anchor. Prepare chunks outside the measured +/// routine. +pub fn attributed_chunks( + tokenizer: &dyn Tokenizer, + text: &str, + chunk_chars: usize, +) -> Vec { + let mut decoder = tokenizer.create_decode_stream(&[], false, 0); + for token_id in tokenizer.encode(text, false).expect("fixture should encode") { + decoder.push_token(token_id).expect("fixture token should decode"); + } + let mut rest = decoder.flush(None).expect("decoder should flush").1; + assert_eq!( + rest.text, text, + "fixture should round-trip through the tokenizer" + ); + + split_by_chars(text, chunk_chars) + .into_iter() + .map(|chunk| rest.drain_prefix(chunk.len())) + .collect() +} + +/// Event totals of one parsed stream, for benchmark sanity checks. +#[derive(Debug, Default)] +pub struct UnifiedStreamSummary { + pub normal_text: String, + pub reasoning_text: String, + pub calls_len: usize, +} + +/// Run one request stream through `parser`: initialize it from the prompt, +/// feed every chunk, and finish. +pub fn feed_unified_parser( + parser: &mut dyn UnifiedParser, + prompt_token_ids: &[u32], + chunks: Vec, +) -> UnifiedStreamSummary { + parser.initialize(prompt_token_ids).expect("parser should initialize"); + + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + parser.parse_into(chunk, &mut output).expect("chunk should parse"); + } + output.append(parser.finish().expect("stream should finish")); + + let mut summary = UnifiedStreamSummary::default(); + let mut tool_indices = BTreeSet::new(); + for event in output.events { + match event { + UnifiedParserEvent::Text(text) => summary.normal_text.push_str(&text), + UnifiedParserEvent::Reasoning(reasoning) => { + summary.reasoning_text.push_str(&reasoning.text); + } + UnifiedParserEvent::ToolCall(call) => { + tool_indices.insert(call.tool_index); + } + } + } + summary.calls_len = tool_indices.len(); + summary +} From 63b9931b7399c9bfa7a8f7cdd60e2991ba87fbf9 Mon Sep 17 00:00:00 2001 From: Mehmet Cagri Date: Tue, 29 Sep 2026 01:37:57 -0500 Subject: [PATCH 003/248] [ROCm][DSv4.1] Paged MXFP4 sparse indexer on aiter's MQA-logits kernel (#58671) Signed-off-by: Mehmet Cagri Kaymak Co-authored-by: Claude Opus 5.5 Co-authored-by: Shanshan Shen <467638484@qq.com> --- .../test_rocm_paged_mxfp4_indexer.py | 599 ++++++++++++++++ .../test_rocm_paged_mxfp4_indexer_plan.py | 162 +++++ vllm/config/attention.py | 5 +- .../layers/rocm_paged_mxfp4_indexer.py | 116 +++ vllm/models/deepseek_v41/amd/rocm.py | 150 +++- vllm/models/deepseek_v41/attention.py | 40 +- vllm/v1/attention/backends/mla/indexer.py | 24 +- .../backends/mla/rocm_paged_mxfp4_indexer.py | 513 +++++++++++++ .../attention/ops/rocm_paged_mxfp4_indexer.py | 676 ++++++++++++++++++ 9 files changed, 2276 insertions(+), 9 deletions(-) create mode 100644 tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py create mode 100644 tests/v1/attention/test_rocm_paged_mxfp4_indexer_plan.py create mode 100644 vllm/model_executor/layers/rocm_paged_mxfp4_indexer.py create mode 100644 vllm/v1/attention/backends/mla/rocm_paged_mxfp4_indexer.py create mode 100644 vllm/v1/attention/ops/rocm_paged_mxfp4_indexer.py diff --git a/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py b/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py new file mode 100644 index 0000000000..311ee89a8d --- /dev/null +++ b/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py @@ -0,0 +1,599 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference. + +The indexer K cache is a view into vLLM's block-major KV pool: its pages are +a block stride apart with other layers' pages in between, so the tests put +it there. The reference reads the keys back from a second cache the same +writer fills in natural order, so it shares none of the preshuffled +addressing it checks. +""" + +import itertools +import types + +import pytest +import torch + +from vllm.platforms import current_platform + +if not current_platform.is_rocm(): + pytest.skip("ROCm-only", allow_module_level=True) + +from vllm.config import CUDAGraphMode +from vllm.models.deepseek_v41.amd.rocm import ( + rocm_mxfp4_indexer_k_store, + rocm_mxfp4_indexer_q_quant, +) +from vllm.platforms.rocm import get_cdna_version +from vllm.utils.math_utils import cdiv +from vllm.utils.torch_utils import set_random_seed +from vllm.v1.attention.backends.mla.indexer import ( + DeepseekV32IndexerPrefillChunkMetadata, +) +from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( + RocmMxfp4IndexerMetadata, + native_decode, + plan_gather_launches, + plan_prefill_chunks, +) +from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops +from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager + +if get_cdna_version() != 4: + pytest.skip("CDNA4-only", allow_module_level=True) + +HEADS, HEAD_DIM = 32, 128 +WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32 +# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled +# down so short test contexts still select. +CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32 +MAX_LEN = 4096 +DEVICE = "cuda" + +_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) +E2M1 = torch.cat([_E2M1, -_E2M1]) + + +def _dequant(packed, e8m0): + """[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32.""" + table = E2M1.to(packed.device) + vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1) + scale = torch.pow(2.0, e8m0.float() - 127) + return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2) + + +@pytest.fixture(autouse=True) +def _workspace(): + init_workspace_manager(torch.device(DEVICE)) + yield + reset_workspace_manager() + + +class _Case: + """Requests with their keys written into a block-major pool (ratio-2 and + ratio-1 pages side by side in every block) and into natural-order caches + the reference reads. The bf16 inputs are kept for the end-to-end test.""" + + def __init__(self, seq_lens, block): + from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import ( + k_norm_rope_mxfp4_cache, + ) + + self.seq_lens = seq_lens + num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3 + perm = torch.randperm(num_blocks) + width = max(cdiv(n, block) for n in seq_lens) + self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32) + used = 0 + for req, n in enumerate(seq_lens): + self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)] + used += cdiv(n, block) + self.block_table = self.block_table.to(DEVICE) + + def align(n): + return cdiv(n, 576) * 576 + + page2, page1 = block // 2 * WIDTH, block * WIDTH + stride = align(page2) + align(page1) + 576 + self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE) + self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool) + for i in range(num_blocks): + base = i * stride + self.page_bytes[base : base + page2] = True + base += align(page2) + self.page_bytes[base : base + page1] = True + self.cache = { + 2: torch.as_strided( + self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0 + ), + 1: torch.as_strided( + self.pool, + (num_blocks, block, WIDTH), + (stride, WIDTH, 1), + align(page2), + ), + } + self.offsets = [0] + for n in seq_lens: + self.offsets.append(self.offsets[-1] + n) + # the same keys in natural order, one entry per page + self.natural = { + r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE) + for r in (1, 2) + } + self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE) + self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5 + self.k_pre = {} + for req, n in enumerate(seq_lens): + if n == 0: + continue + k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16) + self.k_pre[req] = k_pre + pos = torch.arange(n, device=DEVICE) + for ratio, cache in self.cache.items(): + comp = pos // ratio + boundary = (pos + 1) % ratio == 0 + entries = cache.shape[1] + page = self.block_table[req, comp // entries].long() + slot = torch.where(boundary, page * entries + comp % entries, -1) + nat = torch.where(boundary, self.offsets[req] + comp, -1) + rocm_mxfp4_indexer_k_store( + k_pre, + pos, + cos_sin, + norm, + 1e-6, + cache, + slot, + ratio, + True, + num_heads=HEADS, + ) + k_norm_rope_mxfp4_cache( + k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio + ) + + def keys(self, ratio, req, n): + nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0] + return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :]) + + +def _queries(num_rows): + q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8) + scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32)) + q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE) + weights = torch.randn(num_rows, HEADS, device=DEVICE) + # the fused Q kernel returns the scales as one int32 per head + return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales) + + +def _scores(q, w, keys): + return (torch.relu(q @ keys.T) * w[:, None]).sum(0) + + +def _assert_topk(row, scores, k): + """``row`` is a top-k of the entries of ``scores`` above -inf, up to the + fp32 reduce order.""" + valid = scores > float("-inf") + want = min(k, int(valid.sum())) + sel = row[row >= 0].long() + assert sel.numel() == want and sel.unique().numel() == want + if want: + assert bool(valid[sel].all()) + kth = scores[valid].topk(want).values[-1] + finite = scores[torch.isfinite(scores)] + tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0 + assert bool((scores[sel] >= kth - tol).all()) + + +def _block_maxima(scores): + end = scores.numel() + if end == 0: + return scores + padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf")) + padded[:end] = scores.cpu() + out = padded.view(-1, CAND_BLOCK).amax(1) + out[(end - 1) // CAND_BLOCK] = float("inf") + return out.to(scores.device) + + +def _forward_context(monkeypatch, metadata): + context = types.SimpleNamespace( + attn_metadata={"indexer": metadata}, + cudagraph_runtime_mode=CUDAGraphMode.NONE, + ) + monkeypatch.setattr(ops, "get_forward_context", lambda: context) + + +def _decode_metadata(case, rows, ratio, query_lens): + lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32) + lens = lens.to(DEVICE) + block_table = case.block_table[[req for req, _ in rows]].contiguous() + entries = case.cache[ratio].shape[1] + next_n = max(query_lens) + context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE) + native = native_decode(lens, block_table, query_lens, next_n, context_lens) + schedule = torch.empty( + ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n), + dtype=torch.int32, + device=DEVICE, + ) + return RocmMxfp4IndexerMetadata( + seq_lens=None, + max_seq_len=max(case.seq_lens), + slot_mapping=None, + num_decodes=len(case.seq_lens), + num_decode_tokens=len(rows), + num_prefills=0, + num_prefill_tokens=0, + decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]), + decode_row_lens=lens, + decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK, + decode_native=native, + decode_schedule=ops.build_rocm_mxfp4_decode_schedule( + lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native + ), + ) + + +def _prefill_metadata( + case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather +): + chunks = [] + for req_lo, req_hi, t0, t1 in chunk_bounds: + ends = torch.tensor( + [(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32 + ).to(DEVICE) + chunks.append( + DeepseekV32IndexerPrefillChunkMetadata( + block_table=case.block_table[req_lo:req_hi], + cu_seqlen_ks=torch.zeros_like(ends), + cu_seqlen_ke=ends, + cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE), + token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE), + total_seq_lens=1, + token_start=t0, + token_end=t1, + num_reqs=req_hi - req_lo, + ) + ) + seq_lens = torch.tensor(case.seq_lens) + context_lens = (seq_lens // ratio).int().to(DEVICE) + plans = plan_prefill_chunks( + chunks, + query_start_loc, + seq_lens, + context_lens, + ratio, + CAND_BLOCK, + 0.0 if gather else None, + torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE), + ) + return RocmMxfp4IndexerMetadata( + seq_lens=None, + max_seq_len=max(case.seq_lens), + slot_mapping=None, + num_decodes=0, + num_decode_tokens=0, + num_prefills=len(case.seq_lens), + num_prefill_tokens=len(rows), + prefill=types.SimpleNamespace(chunks=chunks), + prefill_plans=plans, + gather_launches=plan_gather_launches(chunks, plans, gather_rows), + ) + + +# The kernel prefers 128-token pages. +BLOCKS = pytest.mark.parametrize("block", [128, 64]) + + +@BLOCKS +def test_k_store_stays_inside_its_pages(block): + """The writer touches only its pages of the block-major pool. What they hold + is checked by the layer tests, which read them back through aiter's kernel.""" + set_random_seed(0) + case = _Case([700, 333], block) + assert int(case.pool[~case.page_bytes].count_nonzero()) == 0 + + +@pytest.mark.parametrize( + "rows, width, k", + # vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048 + [(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)], +) +def test_topk_matches_torch_on_both_kernels(rows, width, k): + set_random_seed(0) + logits = torch.randn(rows, width, device=DEVICE) + lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE) + lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1 + out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE) + ops._topk(logits, lengths, out, k) + for r in range(rows): + n = min(k, int(lengths[r])) + want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values + got = out[r] + assert torch.equal(got[got >= 0].sort().values, want.int()) + assert int((got == -1).sum()) == k - n + + +def _run_layers(monkeypatch, case, rows, metadata): + """Ratio-2 dense layer, the candidate source, and a consumer both ways; + each checked against the torch reference.""" + num_rows = len(rows) + q, q_scale, weights, q_ref = _queries(num_rows) + hidden = torch.empty(num_rows, 1, device=DEVICE) + + def topk_buffer(): + return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE) + + _forward_context(monkeypatch, metadata(2)) + out = topk_buffer() + ops.rocm_mxfp4_sparse_attn_indexer( + hidden, + "indexer", + case.cache[2], + q, + q_scale, + weights, + TOPK, + HEAD_DIM, + MAX_LEN // 2, + out, + compress_ratio=2, + ) + for i, (req, pos) in enumerate(rows): + end = (pos + 1) // 2 + keys = case.keys(2, req, end) + _assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK) + + source = metadata(1) + _forward_context(monkeypatch, source) + pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE) + out = topk_buffer() + ops.rocm_mxfp4_sparse_attn_indexer( + hidden, + "indexer", + case.cache[1], + q, + q_scale, + weights, + TOPK, + HEAD_DIM, + MAX_LEN, + out, + compress_ratio=1, + candidate_blocks=pool, + candidate_block_size=CAND_BLOCK, + candidate_write=True, + ) + allowed = [] + for i, (req, pos) in enumerate(rows): + scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1)) + _assert_topk(out[i], scores, TOPK) + _assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS) + mask = torch.zeros_like(scores, dtype=torch.bool) + for block in pool[i][pool[i] >= 0].tolist(): + mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True + allowed.append(torch.where(mask, scores, float("-inf"))) + + pa = ops._aiter() + build = pa.build_candidate_gather + builds = [] + + def counting_build(*args, **kwargs): + builds.append(1) + return build(*args, **kwargs) + + monkeypatch.setattr(pa, "build_candidate_gather", counting_build) + for gather in (True, False): + consumer = metadata(1, gather) + consumer.decode_use_gather = gather + _forward_context(monkeypatch, consumer) + builds.clear() + # Two consumer layers of one step: the second reuses the first's pools. + for _ in range(2): + out = topk_buffer() + ops.rocm_mxfp4_sparse_mqa_indexer( + hidden, + "indexer", + case.cache[1], + q, + q_scale, + weights, + TOPK, + HEAD_DIM, + MAX_LEN, + out, + 1, + pool, + CAND_BLOCK, + POOL_BLOCKS * CAND_BLOCK, + ) + for i in range(num_rows): + _assert_topk(out[i], allowed[i], TOPK) + if gather: + launches = len(consumer.gather_launches) + (consumer.decode is not None) + assert len(builds) == launches + + +@BLOCKS +@pytest.mark.parametrize( + "query_lens", + # Uniform steps launch the dense layers on next_n-row sequences, also with + # cudagraph padding (query length 0) after them; a ragged step keeps a row + # per token. + [[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]], + ids=["native6", "native2", "padded", "ragged"], +) +def test_decode_layers_match_reference(monkeypatch, block, query_lens): + set_random_seed(0) + next_n = max(query_lens) + case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block) + # a padding request has next_n rows of its own and no context + rows = [ + (req, n - q + j if q else -1) + for req, (n, q) in enumerate(zip(case.seq_lens, query_lens)) + for j in range(q or next_n) + ] + _run_layers( + monkeypatch, + case, + rows, + lambda r, gather=False: _decode_metadata(case, rows, r, query_lens), + ) + + +@pytest.mark.parametrize( + "seq_lens,new_tokens,chunks", + [ + # one request past a cached prefix, then one sliced into two chunks + ([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]), + # three requests in one chunk, the first two with equal rows + ([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]), + # four requests with four different query lengths in one chunk + ([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]), + ], + ids=["sliced", "batched", "ragged"], +) +# The consumers rejoin a request's rows across chunks, then cut them at most +# gather_rows a launch. +@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"]) +@BLOCKS +def test_prefill_layers_match_reference( + monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block +): + set_random_seed(0) + case = _Case(seq_lens, block) + rows = [ + (req, n - q + i) + for req, (n, q) in enumerate(zip(seq_lens, new_tokens)) + for i in range(q) + ] + query_start_loc = [0, *itertools.accumulate(new_tokens)] + _run_layers( + monkeypatch, + case, + rows, + lambda r, gather=True: _prefill_metadata( + case, rows, r, chunks, query_start_loc, gather_rows, gather + ), + ) + + +def _rope_gptj(x, cos_sin, positions): + """GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in + and out.""" + x = x.bfloat16().float() + cos, sin = cos_sin[positions].chunk(2, dim=-1) + if x.dim() == 3: + cos, sin = cos[:, None], sin[:, None] + rot = x[..., -64:] + even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone() + rot[..., 0::2] = even * cos - odd * sin + rot[..., 1::2] = odd * cos + even * sin + return x.bfloat16() + + +def _mxfp4_round_trip(x): + """MXFP4 round trip of x, rounded as the writers do: a scale of + 2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to + even.""" + blocks = x.float().unflatten(-1, (-1, 32)) + amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126) + scale = torch.exp2(torch.ceil(torch.log2(amax / 6))) + y = (blocks / scale).clamp(-6, 6) + # Even codes first: argmin keeps the first of two equal distances. + grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device) + idx = (y.abs()[..., None] - grid).abs().argmin(-1) + return (grid[idx] * y.sign() * scale).flatten(-2) + + +def _torch_keys(case, ratio, req): + """A request's compressed keys in torch: k_norm of each group's last + token, rotated at the group's first position, through MXFP4.""" + k = case.k_pre[req][ratio - 1 :: ratio].float() + k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm + pos = torch.arange(k.shape[0], device=DEVICE) * ratio + return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos)) + + +def _topk_indices(q, weights, keys, ends, k): + """Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end).""" + scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1) + cols = torch.arange(keys.shape[0], device=DEVICE) + return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices + + +def _recall(selected, reference): + """Per row, the fraction of the reference top-k that ``selected`` picked.""" + return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1) + + +@BLOCKS +@pytest.mark.parametrize("ratio", [2, 1]) +@pytest.mark.parametrize("step", ["decode", "prefill"]) +def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block): + """End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the + paged pool, metadata from the planning functions the builder calls, and + the indexer op pick what the same MXFP4 math picks in torch.""" + set_random_seed(0) + if step == "decode": + case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block) + rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)] + metadata = _decode_metadata(case, rows, ratio, [1] * len(rows)) + else: + new_tokens = [300, 200, 250] + case = _Case([1500, 900, 2000], block) + rows = [ + (req, n - q + i) + for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens)) + for i in range(q) + ] + metadata = _prefill_metadata( + case, + rows, + ratio, + [(0, len(new_tokens), 0, len(rows))], + [0, *itertools.accumulate(new_tokens)], + 1 << 20, + gather=False, + ) + num_rows = len(rows) + q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16) + w = torch.randn(num_rows, HEADS, device=DEVICE) + pos = torch.tensor([p for _, p in rows], device=DEVICE) + scales = (HEAD_DIM**-0.5, HEADS**-0.5) + + (q, q_scale), weights = rocm_mxfp4_indexer_q_quant( + pos, q_bf16, case.cos_sin, w, *scales + ) + _forward_context(monkeypatch, metadata) + out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE) + ops.rocm_mxfp4_sparse_attn_indexer( + torch.empty(num_rows, 1, device=DEVICE), + "indexer", + case.cache[ratio], + q, + q_scale, + weights, + TOPK, + HEAD_DIM, + MAX_LEN // ratio, + out, + compress_ratio=ratio, + ) + ends = (pos + 1) // ratio + assert bool(((out >= 0) & (out < ends[:, None])).all()) + assert all(row.unique().numel() == TOPK for row in out) + + q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos)) + w_ref = w * scales[0] * scales[1] + agree = [] + for req in range(len(case.seq_lens)): + mine = torch.tensor([r == req for r, _ in rows], device=DEVICE) + ref = _topk_indices( + q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK + ) + agree.append(_recall(out[mine], ref)) + assert torch.cat(agree).mean().item() >= 0.99 diff --git a/tests/v1/attention/test_rocm_paged_mxfp4_indexer_plan.py b/tests/v1/attention/test_rocm_paged_mxfp4_indexer_plan.py new file mode 100644 index 0000000000..0283ed99ee --- /dev/null +++ b/tests/v1/attention/test_rocm_paged_mxfp4_indexer_plan.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The ROCm MXFP4 indexer reads the paged cache in place, so each prefill +chunk is split back into its requests' rows, which one launch finds through +the chunk's query_start_loc. A chunk is whole requests or one query slice of a +long request, and both have to come back with the right rows and the right +block-table rows. +""" + +import pytest +import torch + +from vllm.platforms import current_platform + +if not current_platform.is_rocm(): + pytest.skip("ROCm-only", allow_module_level=True) + +from vllm.v1.attention.backends.mla.indexer import ( + DeepseekV32IndexerPrefillChunkMetadata, +) +from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( + native_decode, + plan_gather_launches, + plan_prefill_chunks, + split_prefill_chunks, +) +from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import rocm_mxfp4_consumer_rows + + +def _chunk(block_table, token_start, token_end, row_ends): + ends = torch.tensor(row_ends, dtype=torch.int32) + return DeepseekV32IndexerPrefillChunkMetadata( + block_table=block_table, + cu_seqlen_ks=torch.full_like(ends, 7), + cu_seqlen_ke=ends + 7, + cu_seq_lens=torch.zeros(1, dtype=torch.int32), + token_to_seq=torch.zeros(1, dtype=torch.int32), + total_seq_lens=1, + token_start=token_start, + token_end=token_end, + num_reqs=block_table.shape[0], + ) + + +# A decode (1 token), then prefills of 3, 3, 5 and 6 query tokens; the last +# one is sliced into two chunks. +QUERY_START_LOC = [0, 1, 4, 7, 12, 18] +SEQ_LENS = torch.tensor([40, 10, 12, 21, 64]) +BLOCK_TABLE = torch.arange(20, dtype=torch.int32).view(5, 4) +CHUNKS = [ + _chunk(BLOCK_TABLE[1:4], 1, 12, list(range(11))), + _chunk(BLOCK_TABLE[4:5], 12, 15, [29, 30, 30]), + _chunk(BLOCK_TABLE[4:5], 15, 18, [31, 31, 32]), +] + + +def _plans(min_gather_width, block=8): + return plan_prefill_chunks( + CHUNKS, + QUERY_START_LOC, + SEQ_LENS, + SEQ_LENS.int() // 2, + 2, + block, + min_gather_width, + torch.tensor(QUERY_START_LOC, dtype=torch.int32), + ) + + +def test_chunks_split_into_requests(): + plans = _plans(20.0) + + assert [p.requests for p in plans] == [ + [(0, 3, 0), (3, 6, 1), (6, 11, 2)], + [(0, 3, 0)], + [(0, 3, 0)], + ] + # every chunk launches once, with offsets local to the chunk + assert [p.query_start_loc.tolist() for p in plans] == [ + [0, 3, 6, 11], + [0, 3], + [0, 3], + ] + # the logits are as wide as the longest compressed context in the chunk + assert [p.width for p in plans] == [10, 32, 32] + assert [p.context_lens.tolist() for p in plans] == [[5, 6, 10], [32], [32]] + torch.testing.assert_close(plans[2].row_ends, torch.tensor([31, 31, 32]).int()) + torch.testing.assert_close(plans[2].block_ends, torch.tensor([4, 4, 4]).int()) + assert [p.use_gather for p in plans] == [False, True, True] + assert [p.first_request for p in plans] == [1, 4, 4] + # no candidate blocks, as in the model-neutral builder + for plan in _plans(None, block=0): + assert plan.block_ends is None and not plan.use_gather + + +def test_launches_follow_the_logits_budget(monkeypatch): + """Each launch's fp32 logits fit in VLLM_SPARSE_INDEXER_MAX_LOGITS_MB. A dense + chunk's logits are [rows, longest context in the chunk]: K is read in place, + not gathered, so requests share a launch until their rows no longer fit, and + a request that does not fit alone is split on its query rows. A consumer's + logits are [rows, pool].""" + budget = 4 * 1000 * 30 + contexts, queries = torch.tensor([1000, 600, 200]), torch.tensor([10, 10, 10]) + assert split_prefill_chunks(contexts, queries, budget, 2) == [ + (slice(2, 5), slice(0, 30)) + ] + # the third request would take the chunk past the budget + queries = torch.tensor([10, 10, 11]) + assert split_prefill_chunks(contexts, queries, budget) == [ + (slice(0, 2), slice(0, 20)), + (slice(2, 3), slice(0, 11)), + ] + assert split_prefill_chunks(torch.tensor([1000]), torch.tensor([100]), budget) == [ + (slice(0, 1), slice(lo, min(lo + 30, 100))) for lo in range(0, 100, 30) + ] + # the default 512 MiB holds 128 rows of a 1M context, or 8192 of a 16K pool + assert split_prefill_chunks( + torch.tensor([1 << 20]), torch.tensor([200]), 512 << 20 + ) == [(slice(0, 1), slice(0, 128)), (slice(0, 1), slice(128, 200))] + monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512") + assert rocm_mxfp4_consumer_rows(16384) == 8192 + + +def test_gather_launches_rejoin_a_requests_rows(): + """The consumers' logits are [rows, pool] whatever the context, so a + request's rows go back together across the chunks that cut them, and a + launch never mixes requests.""" + # the first chunk is too narrow to gather, the last request's slices rejoin + (joined,) = plan_gather_launches(CHUNKS, _plans(20.0), 100) + assert (joined.token_start, joined.token_end) == (12, 18) + # every chunk gathers: the first chunk's three requests launch apart + launches = plan_gather_launches(CHUNKS, _plans(0.0), 100) + assert [(g.token_start, g.token_end) for g in launches] == [ + (1, 4), + (4, 7), + (7, 12), + (12, 18), + ] + assert plan_gather_launches(CHUNKS, _plans(None), 100) == [] + + +def test_decode_launches_native_on_uniform_steps(): + """A decode step goes to the kernel as next_n-row sequences only when its + shape says so: each request has next_n rows, and cudagraph padding (query + length 0) comes only after them. A FULL graph captured on a uniform batch + then replays the same launch on a padded one.""" + rows = torch.zeros(8, dtype=torch.int32) + row_block_table = torch.arange(32, dtype=torch.int32).view(8, 4) + context_lens = torch.tensor([9, 4, 7, 0, 5], dtype=torch.int32) + + native = native_decode(rows, row_block_table, [2, 2, 2, 0], 2, context_lens) + assert native is not None and native.next_n == 2 + assert native.context_lens.tolist() == [9, 4, 7, 0] + # every request's first flattened row, as a view + assert native.block_table.tolist() == row_block_table[::2].tolist() + assert native.block_table.data_ptr() == row_block_table.data_ptr() + + assert native_decode(rows, row_block_table, [2, 0, 2, 2], 2, context_lens) is None + # a ragged step, rows padded past the requests, and no speculation + assert native_decode(rows[:5], row_block_table, [2, 1, 2], 2, context_lens) is None + assert native_decode(rows, row_block_table, [2, 2, 2], 2, context_lens) is None + assert native_decode(rows[:4], row_block_table, [1] * 4, 1, context_lens) is None diff --git a/vllm/config/attention.py b/vllm/config/attention.py index a6602ed549..8a9be6336c 100644 --- a/vllm/config/attention.py +++ b/vllm/config/attention.py @@ -93,7 +93,10 @@ class AttentionConfig: an SM100-class GPU, DeepGEMM >= 2.8 and the DeepSelect top-k extension (the top-k runs on the kernels' bf16 logits). The sparse path costs O(candidate blocks) per query regardless of context length, so it pays off - for long contexts (roughly 32K tokens and beyond) and is slower below.""" + for long contexts (roughly 32K tokens and beyond) and is slower below. + On ROCm gfx950 it runs aiter's paged MXFP4 MQA-logits kernel over the + candidate pool instead, with fp32 logits, and keeps the masked dense walk + for steps whose contexts are too short for the pool to pay.""" hisparse_config: HiSparseConfig | None = None """HiSparse host-resident KV configuration. Setting this enables experimental diff --git a/vllm/model_executor/layers/rocm_paged_mxfp4_indexer.py b/vllm/model_executor/layers/rocm_paged_mxfp4_indexer.py new file mode 100644 index 0000000000..b42f63ac94 --- /dev/null +++ b/vllm/model_executor/layers/rocm_paged_mxfp4_indexer.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm. + +With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K +cache in the order aiter's MQA-logits kernel reads (see +`rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place. +Requires `RocmMxfp4IndexerMetadataBuilder` metadata. +""" + +import torch +from torch import nn + +from vllm.config import get_current_vllm_config +from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer +from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import ( + rocm_mxfp4_sparse_attn_indexer, + rocm_mxfp4_sparse_mqa_indexer, +) + + +class RocmSparseAttnIndexer(SparseAttnIndexer): + """`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with + aiter's kernel. A two-level indexer's source layer also publishes its + candidate pool here.""" + + def forward_hip( + self, + hidden_states: torch.Tensor, + q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], + k: torch.Tensor | None, + weights: torch.Tensor, + ): + assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, ( + "the ROCm MXFP4 indexer takes (values, scales) Q and a K cache " + "the model already wrote" + ) + q_values, q_scale = q_quant + return rocm_mxfp4_sparse_attn_indexer( + hidden_states, + self.k_cache.prefix, + self.k_cache.kv_cache, + q_values, + q_scale, + weights, + self.topk_tokens, + self.head_dim, + self.max_model_len, + self.topk_indices_buffer, + compress_ratio=self.compress_ratio, + candidate_blocks=self.candidate_blocks, + candidate_block_size=self.candidate_block_size, + candidate_write=self.candidate_write, + ) + + +class RocmSparseMQAIndexer(nn.Module): + """DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4 + MQA-logits kernel: the consumers score only the candidate pool the source + layer published, read straight from the paged cache. + + Built and called as `SparseMQAIndexer` is, so the model can take either. + """ + + weights_dtype = torch.float32 + """The scorer's weights and logits stay fp32: gfx950 has no packed bf16 + arithmetic, so its head reduce costs the same in either dtype.""" + + def __init__( + self, + k_cache, + topk_tokens: int, + head_dim: int, + max_total_seq_len: int, + topk_indices_buffer: torch.Tensor, + candidate_blocks: torch.Tensor, + candidate_block_size: int, + ): + super().__init__() + self.k_cache = k_cache + self.topk_tokens = topk_tokens + self.head_dim = head_dim + # aiter's kernel bounds a row by the model length in compressed + # positions, not by the prefill buffer SparseMQAIndexer sizes. + model_config = get_current_vllm_config().model_config + self.max_model_len = model_config.max_model_len // k_cache.compress_ratio + self.topk_indices_buffer = topk_indices_buffer + self.candidate_blocks = candidate_blocks + self.candidate_block_size = candidate_block_size + self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size + + def forward( + self, + hidden_states: torch.Tensor, + q_quant: tuple[torch.Tensor, torch.Tensor], + k: torch.Tensor | None, + weights: torch.Tensor, + ) -> torch.Tensor: + assert k is None, "the model writes the indexer K cache" + q_values, q_scale = q_quant + return rocm_mxfp4_sparse_mqa_indexer( + hidden_states, + self.k_cache.prefix, + self.k_cache.kv_cache, + q_values, + q_scale, + weights, + self.topk_tokens, + self.head_dim, + self.max_model_len, + self.topk_indices_buffer, + self.k_cache.compress_ratio, + self.candidate_blocks, + self.candidate_block_size, + self.num_candidate_cols, + ) diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index 897b0d12db..fe020f4061 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -2,8 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import functools +from collections.abc import Callable from dataclasses import dataclass -from typing import cast +from typing import Any, cast import torch @@ -15,8 +16,14 @@ from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic +from vllm.model_executor.layers.rocm_paged_mxfp4_indexer import ( + RocmSparseAttnIndexer, + RocmSparseMQAIndexer, +) from vllm.models.deepseek_v41.attention import ( DeepseekV4Attention, + DeepseekV4Indexer, + DeepseekV4IndexerCache, _replace_layer_index, ) from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache @@ -32,6 +39,7 @@ from vllm.triton_utils import tl, triton from vllm.utils.multi_stream_utils import execute_in_parallel from vllm.v1.attention.backend import ( CommonAttentionMetadata, + MultipleOf, ) from vllm.v1.attention.backends.mla.sparse_swa import ( DeepseekSparseSWABackend, @@ -47,6 +55,9 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( rocm_sparse_attn_decode, rocm_sparse_attn_prefill, ) +from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import ( + rocm_paged_mxfp4_cache_layout, +) from vllm.v1.worker.workspace import current_workspace_manager logger = init_logger(__name__) @@ -517,11 +528,141 @@ class DeepseekV4ROCMAiterSparseSWAMetadataBuilder(DeepseekV41SparseSWAMetadataBu ) +@functools.cache +def _aiter_indexer_cache_ops() -> tuple[Callable[..., None], Callable[..., tuple]]: + """The aiter indexer key writer and query quantizer.""" + from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import ( + k_norm_rope_mxfp4_cache, + ) + from aiter.ops.triton.rope.q_rope_mxfp4_quant import q_rope_mxfp4_quant + + return k_norm_rope_mxfp4_cache, q_rope_mxfp4_quant + + +def rocm_mxfp4_indexer_k_store( + k_pre: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + rms_norm_weight: torch.Tensor, + rms_norm_eps: float, + k_cache: torch.Tensor, + kv_slot_mapping: torch.Tensor, + compress_ratio: int, + use_fp4_cache: bool, + *, + num_heads: int, +) -> None: + """`indexer_k_norm_rope_store` for the ROCm MXFP4 cache: aiter's cache op + writes the key in the order its MQA-logits kernel reads with ``num_heads`` + query heads.""" + assert use_fp4_cache, "the ROCm indexer cache op writes MXFP4 only" + layout = rocm_paged_mxfp4_cache_layout(num_heads, k_pre.shape[1], k_cache.shape[1]) + k_norm_rope_mxfp4_cache, _ = _aiter_indexer_cache_ops() + k_norm_rope_mxfp4_cache( + k_pre, + positions, + cos_sin_cache, + rms_norm_weight, + rms_norm_eps, + k_cache, + kv_slot_mapping, + compress_ratio, + shuffle=layout, + ) + + +def rocm_mxfp4_indexer_q_quant( + positions: torch.Tensor, + index_q: torch.Tensor, + index_q_cos_sin_cache: torch.Tensor, + index_weights: torch.Tensor, + index_weights_softmax_scale: float, + index_weights_head_scale: float, + use_fp4: bool = True, + weights_out_dtype: torch.dtype = torch.float32, +) -> tuple[tuple[torch.Tensor, torch.Tensor], torch.Tensor]: + """`fused_indexer_q_rope_quant` for the ROCm MXFP4 indexer, on aiter's op: + ((packed [T, H, D // 2], e8m0 as one int32 per head [T, H]), fp32 + weights).""" + assert use_fp4 and weights_out_dtype == torch.float32, ( + "the ROCm indexer quantizes Q to MXFP4 and scores with fp32 weights" + ) + _, q_rope_mxfp4_quant = _aiter_indexer_cache_ops() + q_packed, q_scale, weights_out = q_rope_mxfp4_quant( + index_q, + positions, + index_q_cos_sin_cache, + index_weights, + index_weights_softmax_scale * index_weights_head_scale, + ) + return (q_packed, q_scale.view(torch.int32).squeeze(-1)), weights_out + + +class DeepseekV41RocmMxfp4Indexer(DeepseekV4Indexer): + """The indexer on aiter's paged MXFP4 cache: its K store and Q quant write + in the order aiter's MQA-logits kernel reads, and its layers score with it. + ``_produce_k`` and ``forward_q`` mirror ``DeepseekV4Indexer``'s except for + those two calls.""" + + mqa_cls = RocmSparseMQAIndexer + attn_cls = RocmSparseAttnIndexer + + def _produce_k( + self, + latent: torch.Tensor | None, + positions: torch.Tensor, + rotary_emb: torch.nn.Module, + ) -> None: + attn_metadata = get_forward_context().attn_metadata + if not isinstance(attn_metadata, dict) or latent is None: + return + assert self.owns_k + indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix]) + k_pre, _ = self.wk(latent) + rocm_mxfp4_indexer_k_store( + k_pre, + positions, + rotary_emb.cos_sin_cache, + self.k_norm.weight, + self.k_norm.variance_epsilon, + self.k_cache.kv_cache, + indexer_metadata.slot_mapping, + self.compress_ratio, + self.use_fp4_kv, + num_heads=self.n_head, + ) + + def forward_q( + self, + qr: torch.Tensor | QuantizedActivation, + qr_scale: torch.Tensor | None, + indexer_weights: torch.Tensor, + positions: torch.Tensor, + rotary_emb: torch.nn.Module, + ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: + q = self._wq_b_proj(qr, qr_scale).view(-1, self.n_head, self.head_dim) + (q, q_scale), weights = rocm_mxfp4_indexer_q_quant( + positions, + q, + rotary_emb.cos_sin_cache, + indexer_weights, + self.softmax_scale, + self.n_head**-0.5, + use_fp4=self.use_fp4_kv, + weights_out_dtype=self.indexer_weights_dtype, + ) + return q, q_scale, weights + + class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend): @staticmethod def get_name() -> str: return "ROCM_FLASHMLA_SPARSE_DSV4" + @staticmethod + def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: + return [64, 128] + @staticmethod def get_builder_cls() -> type[DeepseekV4SparseMLAMetadataBuilder]: return DeepseekV4SparseMLAMetadataBuilder @@ -540,6 +681,13 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): swa_backend_cls = DeepseekV41ROCMAiterSparseSWABackend _use_aiter_sparse_mla = False + def _indexer_cls( + self, k_cache: DeepseekV4IndexerCache | None + ) -> type[DeepseekV4Indexer]: + if k_cache is not None and k_cache.rocm_mxfp4: + return DeepseekV41RocmMxfp4Indexer + return super()._indexer_cls(k_cache) + def __init__(self, *args, **kwargs): vllm_config = args[0] if args else kwargs["vllm_config"] super().__init__(*args, **kwargs) diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index c27a8d9baf..73d9c0c723 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -35,6 +35,9 @@ from vllm.models.deepseek_v41.common.ops import ( if TYPE_CHECKING: from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import GemmRsAr + from vllm.model_executor.layers.rocm_paged_mxfp4_indexer import ( + RocmSparseMQAIndexer, + ) from vllm.v1.attention.backends.mla.sparse_swa import ( DeepseekSparseSWAMetadata, ) @@ -261,6 +264,13 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): """Return whether this instance stores fp8 KV in fp8_ds_mla layout.""" return self.use_fp8_ds_mla_layout + def _indexer_cls( + self, k_cache: "DeepseekV4IndexerCache | None" + ) -> type["DeepseekV4Indexer"]: + """The indexer for this layer's index K cache. A platform subclass + returns one that writes and scores its own cache layout.""" + return DeepseekV4Indexer + def __init__( self, vllm_config: VllmConfig, @@ -449,7 +459,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): ) is_candidate_source = layer_id == self.candidate_source_layer uses_candidates = 0 <= self.candidate_source_layer < layer_id - self.indexer = DeepseekV4Indexer( + self.indexer = self._indexer_cls(index_k_cache)( vllm_config, config=config, hidden_size=self.hidden_size, @@ -1152,6 +1162,10 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase): self.compress_ratio = compress_ratio vllm_config = get_current_vllm_config() self.sparse_logits = vllm_config.attention_config.indexer_sparse_logits + # aiter's paged MXFP4 kernels write this cache and read it in place. + self.rocm_mxfp4 = current_platform.is_rocm() and dsa_indexer_uses_fp4( + vllm_config + ) compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") @@ -1190,13 +1204,21 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase): # it 512B-aligned; the main KV record read from the same blocks # needs its own TMA stride, so keep both. block_stride_alignment=( - math.lcm(512, page_alignment) if self.sparse_logits else None + math.lcm(512, page_alignment) + if self.sparse_logits and not self.rocm_mxfp4 + else None ), ) def forward(self): ... def get_attn_backend(self) -> type[AttentionBackend]: + if self.rocm_mxfp4: + from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( + DeepseekV41RocmMxfp4IndexerBackend, + ) + + return DeepseekV41RocmMxfp4IndexerBackend if self.sparse_logits: return DeepseekV41SparseIndexerBackend return DeepseekV41IndexerBackend @@ -1218,6 +1240,12 @@ class DeepseekV4Indexer(nn.Module): later indexers mask their scores to those blocks before their own top-k. """ + # The scoring layers; a platform subclass swaps in its own. + mqa_cls: ClassVar[type["SparseMQAIndexer | RocmSparseMQAIndexer"]] = ( + SparseMQAIndexer + ) + attn_cls: ClassVar[type[SparseAttnIndexer]] = SparseAttnIndexer + def __init__( self, vllm_config: VllmConfig, @@ -1305,7 +1333,7 @@ class DeepseekV4Indexer(nn.Module): and candidate_block_buffer is not None and not candidate_write ) - self.indexer_op: SparseAttnIndexer | SparseMQAIndexer + self.indexer_op: SparseAttnIndexer | SparseMQAIndexer | RocmSparseMQAIndexer if use_sparse_logits: if not self.use_fp4_kv: raise ValueError( @@ -1314,7 +1342,7 @@ class DeepseekV4Indexer(nn.Module): ) assert candidate_block_buffer is not None assert topk_indices_buffer is not None - self.indexer_op = SparseMQAIndexer( + self.indexer_op = self.mqa_cls( self.k_cache, self.topk_tokens, self.head_dim, @@ -1324,7 +1352,7 @@ class DeepseekV4Indexer(nn.Module): candidate_block_size, ) else: - self.indexer_op = SparseAttnIndexer( + self.indexer_op = self.attn_cls( self.k_cache, self.quant_block_size, self.scale_fmt, @@ -1343,7 +1371,7 @@ class DeepseekV4Indexer(nn.Module): # The fused Q kernel writes the per-head weights in the dtype the # scoring kernels take, so no cast runs per step. self.indexer_weights_dtype = ( - SparseMQAIndexer.weights_dtype if use_sparse_logits else torch.float32 + self.mqa_cls.weights_dtype if use_sparse_logits else torch.float32 ) def _produce_k( diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index e7eb67b61f..f2f2c51925 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -51,7 +51,7 @@ from vllm.v1.kv_cache_interface import ( logger = init_logger(__name__) # The DSA indexer K cache is always quantized; "auto" means fp8 (V3.2 layout) -# and mxfp4 is the opt-in Blackwell path. +# and mxfp4 is the opt-in Blackwell and gfx950 path. DSA_INDEXER_KV_DTYPES = ("fp8", "mxfp4") @@ -64,6 +64,28 @@ def dsa_indexer_uses_fp4(vllm_config: VllmConfig) -> bool: f"sparse indexer (expected one of {DSA_INDEXER_KV_DTYPES})." ) use_fp4 = kv_dtype == "mxfp4" + if use_fp4 and current_platform.is_rocm(): + from vllm._aiter_ops import rocm_aiter_ops + from vllm.platforms.rocm import get_cdna_version + + # Only DeepSeek-V4.1 is wired to the ROCm MXFP4 cache; other DSA models + # would silently keep their FP8 one. + model_config = vllm_config.model_config + if model_config is None or model_config.hf_config.model_type != "deepseek_v41": + raise ValueError( + "indexer_kv_dtype='mxfp4' on ROCm is only supported for " + "DeepSeek-V4.1-Flash." + ) + if get_cdna_version() != 4: + raise ValueError( + "indexer_kv_dtype='mxfp4' on ROCm requires CDNA4 (MI350X/MI355X)." + ) + if not rocm_aiter_ops.is_enabled(): + raise ValueError( + "indexer_kv_dtype='mxfp4' on ROCm runs on aiter's kernels; enable " + "aiter with VLLM_ROCM_USE_AITER=1." + ) + return True if use_fp4 and not current_platform.is_device_capability_family(100): raise ValueError( "indexer_kv_dtype='mxfp4' requires Blackwell datacenter GPUs " diff --git a/vllm/v1/attention/backends/mla/rocm_paged_mxfp4_indexer.py b/vllm/v1/attention/backends/mla/rocm_paged_mxfp4_indexer.py new file mode 100644 index 0000000000..cfc30204da --- /dev/null +++ b/vllm/v1/attention/backends/mla/rocm_paged_mxfp4_indexer.py @@ -0,0 +1,513 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Indexer metadata for the ROCm paged MXFP4 path. + +Extends the dense indexer metadata with what aiter's paged MXFP4 MQA-logits +kernel needs to read the cache in place: each prefill chunk as its requests, +packed through query_start_loc into one launch, and the decode rows as +next_n-row sequences on uniform steps. `RocmMxfp4IndexerMetadataBuilder` plans +this for any model's indexer, and a two-level indexer's candidate blocks once +a subclass sets their size. DeepSeek-V4.1's builder does, and with +``AttentionConfig.indexer_sparse_logits`` adds the per-step state the +candidate consumers share. +""" + +import bisect +from dataclasses import dataclass, field, fields + +import torch + +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.v1.attention.backend import ( + AttentionCGSupport, + CommonAttentionMetadata, + MultipleOf, +) +from vllm.v1.attention.backends.mla.indexer import ( + DeepseekV32IndexerMetadata, + DeepseekV32IndexerMetadataBuilder, + DeepseekV32IndexerPrefillChunkMetadata, + DeepseekV41IndexerBackend, +) +from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import ( + build_rocm_mxfp4_decode_schedule, + rocm_mxfp4_consumer_rows, + rocm_mxfp4_decode_schedule_words, + rocm_paged_mxfp4_cache_layout, +) +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec + +logger = init_logger(__name__) + +# The consumers walk the pool instead of the context once the context is this +# many pools long. Set from kernel timings, not end-to-end runs: the gather +# loads more per key than the dense walk, so it only wins on a large cut. +_GATHER_MIN_CUT_DECODE = 3.0 +_GATHER_MIN_CUT_PREFILL = 1.5 + + +@dataclass +class RocmMxfp4PrefillPlan: + """One prefill chunk as the requests the kernel launches on, all in one + launch: the kernel finds each row's request through query_start_loc, and + a request's rows share its KV tiles.""" + + requests: list[tuple[int, int, int]] + """(first row, end row, request) per request with rows in the chunk; rows + count from ``chunk.token_start``, requests index ``chunk.block_table``.""" + width: int + """Logits columns: an upper bound on the chunk's compressed contexts.""" + row_ends: torch.Tensor + """[rows] int32 exclusive compressed key bound of each row.""" + context_lens: torch.Tensor + """[chunk.num_reqs] int32 compressed context of each request.""" + query_start_loc: torch.Tensor + """[chunk.num_reqs + 1] int32 row offsets of the chunk's requests, local to + the chunk.""" + first_request: int + """The step's index of the chunk's first request.""" + block_ends: torch.Tensor | None = None + """[rows] int32 candidate blocks each row sees, for the source's pool.""" + use_gather: bool = False + + +@dataclass +class RocmMxfp4GatherLaunch: + """Query rows [token_start, token_end) of one request, which the candidate + consumers score against the pool in one launch.""" + + token_start: int + token_end: int + block_table: torch.Tensor + """[1, max_blocks] int32, the request's block table row.""" + context_len: torch.Tensor + """[1] int32 compressed context of the request.""" + row_ends: torch.Tensor + """[rows] int32 exclusive compressed key bound of each row.""" + pool: tuple[dict, torch.Tensor] | None = None + """The resolved candidate lists, built by the first consumer layer and + reused by the rest.""" + + +@dataclass +class RocmMxfp4NativeDecode: + """A decode step whose requests all have next_n query rows, as the + kernel's sequences: a workgroup then walks a KV tile once for up to + next_n rows instead of once per row.""" + + next_n: int + context_lens: torch.Tensor + """[requests] int32 compressed context of each request.""" + block_table: torch.Tensor + """[requests, max_blocks] int32, each request's first flattened row.""" + + +def native_decode( + row_lens: torch.Tensor, + row_block_table: torch.Tensor, + query_lens: list[int], + next_n: int, + context_lens: torch.Tensor, +) -> RocmMxfp4NativeDecode | None: + """The step as next_n-row sequences, or None when a request has fewer rows. + + ``row_lens`` and ``row_block_table`` are the flattened rows' bounds and + block table, ``query_lens`` the decode requests' query lengths. Trailing + empty requests are cudagraph padding: they keep their next_n rows, with no + context. Only the step's shape decides, so a FULL graph captured on a + uniform batch replays the same launch on a padded one. + """ + num_reqs = len(query_lens) + num_full = next((i for i, n in enumerate(query_lens) if n != next_n), num_reqs) + if ( + next_n <= 1 + or row_lens.shape[0] != num_reqs * next_n + or any(query_lens[num_full:]) + ): + return None + return RocmMxfp4NativeDecode( + next_n, context_lens[:num_reqs], row_block_table[::next_n] + ) + + +@dataclass +class RocmMxfp4IndexerMetadata(DeepseekV32IndexerMetadata): + prefill_plans: list[RocmMxfp4PrefillPlan] = field(default_factory=list) + """Parallel to ``prefill.chunks``.""" + decode_row_lens: torch.Tensor | None = None + """[rows] int32 compressed context of each decode query row.""" + decode_block_ends: torch.Tensor | None = None + """[rows] int32 candidate blocks each decode row sees, for the source's + pool.""" + decode_native: RocmMxfp4NativeDecode | None = None + """The same rows as next_n-row sequences, for the dense launches.""" + decode_schedule: torch.Tensor | None = None + """The dense decode launches' work schedule, built once for the step.""" + decode_use_gather: bool = False + """Whether the candidate consumers gather for the decode rows.""" + decode_gather: tuple[dict, torch.Tensor] | None = None + """The pool the first consumer resolved for the decode rows.""" + gather_launches: list[RocmMxfp4GatherLaunch] = field(default_factory=list) + """The prefill rows of the chunks that gather, as the consumers launch + them.""" + + +def plan_prefill_chunks( + chunks: list[DeepseekV32IndexerPrefillChunkMetadata], + query_start_loc: list[int], + seq_lens_cpu: torch.Tensor, + context_lens: torch.Tensor, + compress_ratio: int, + candidate_block_size: int, + min_gather_width: float | None, + query_start_loc_device: torch.Tensor, +) -> list[RocmMxfp4PrefillPlan]: + """Split each chunk into its requests. + + ``query_start_loc`` and ``seq_lens_cpu`` are the step's host copies (the + latter an upper bound), ``context_lens`` the device compressed lengths. + A chunk covers whole requests, or a query slice of one request; either + way it launches once, packed through the chunk's part of the device + ``query_start_loc``. + """ + block = candidate_block_size + plans = [] + for chunk in chunks: + t0, t1 = chunk.token_start, chunk.token_end + first = bisect.bisect_right(query_start_loc, t0) - 1 + requests = [] + for req in range(chunk.num_reqs): + lo = max(query_start_loc[first + req], t0) - t0 + hi = min(query_start_loc[first + req + 1], t1) - t0 + if hi > lo: + requests.append((lo, hi, req)) + reqs = slice(first, first + chunk.num_reqs) + packed = torch.clamp(query_start_loc_device[first : reqs.stop + 1], t0, t1) + packed -= t0 + width = int(seq_lens_cpu[reqs].max()) // compress_ratio + row_ends = chunk.cu_seqlen_ke - chunk.cu_seqlen_ks + plans.append( + RocmMxfp4PrefillPlan( + requests=requests, + width=width, + row_ends=row_ends, + context_lens=context_lens[reqs], + query_start_loc=packed, + first_request=first, + block_ends=(row_ends + block - 1) // block if block else None, + use_gather=min_gather_width is not None and width >= min_gather_width, + ) + ) + return plans + + +def split_prefill_chunks( + context_lens: torch.Tensor, + query_lens: torch.Tensor, + max_logits_bytes: int, + request_offset: int = 0, +) -> list[tuple[slice, slice]]: + """(request slice, query slice) chunks, as the base chunker returns them. + + Nothing gathers K here, so the requests' contexts do not add up: a chunk's + logits are [rows, widest compressed context] fp32, at most + ``max_logits_bytes``. A request too wide to launch whole is cut on its + query rows. + """ + + def max_rows(width: int) -> int: + return max(1, max_logits_bytes // (4 * max(width, 1))) + + chunks: list[tuple[slice, slice]] = [] + end = 0 + while end < len(context_lens): + start, rows, width = end, 0, 0 + while end < len(context_lens): + q = int(query_lens[end]) + w = max(width, int(context_lens[end])) + if rows and rows + q > max_rows(w): + break + rows, width = rows + q, w + end += 1 + step = max_rows(width) + reqs = slice(start + request_offset, end + request_offset) + chunks.extend( + (reqs, slice(lo, min(lo + step, rows))) for lo in range(0, rows, step) + ) + return chunks + + +def plan_gather_launches( + chunks: list[DeepseekV32IndexerPrefillChunkMetadata], + plans: list[RocmMxfp4PrefillPlan], + max_rows: int, +) -> list[RocmMxfp4GatherLaunch]: + """The candidate consumers' launches over the chunks that gather. + + A consumer's logits are [rows, pool] however long the context, so the + dense split, sized for the context, would launch it far more often than + its memory needs. Each request's rows go back together across the chunks + that cut them, then out again at most ``max_rows`` a launch. + """ + # Per request: (first token, [(plan, first row, end row)]), and the chunk + # and request index its block table and context come from. + runs: list[tuple[int, list, DeepseekV32IndexerPrefillChunkMetadata, int]] = [] + last: tuple[int, int] | None = None + for chunk, plan in zip(chunks, plans): + if not plan.use_gather: + last = None + continue + for lo, hi, req in plan.requests: + request, t_lo = plan.first_request + req, chunk.token_start + lo + if last != (request, t_lo): + runs.append((t_lo, [], chunk, req)) + runs[-1][1].append((plan, lo, hi)) + last = (request, chunk.token_start + hi) + launches = [] + for token_start, pieces, chunk, req in runs: + ends = [plan.row_ends[lo:hi] for plan, lo, hi in pieces] + row_ends = ends[0] if len(ends) == 1 else torch.cat(ends) + block_table = chunk.block_table[req : req + 1] + context_len = pieces[0][0].context_lens[req : req + 1] + for lo in range(0, row_ends.shape[0], max_rows): + hi = min(lo + max_rows, row_ends.shape[0]) + launches.append( + RocmMxfp4GatherLaunch( + token_start + lo, + token_start + hi, + block_table, + context_len, + row_ends[lo:hi], + ) + ) + return launches + + +class DeepseekV41RocmMxfp4IndexerBackend(DeepseekV41IndexerBackend): + @staticmethod + def get_name() -> str: + return "DEEPSEEK_V41_ROCM_MXFP4_INDEXER" + + @staticmethod + def get_supported_kernel_block_sizes( + kv_cache_spec: KVCacheSpec | None = None, + ) -> list[int | MultipleOf]: + # --block-size 128 preferred. + return [64, 128] + + @staticmethod + def get_builder_cls() -> type["DeepseekV41RocmMxfp4IndexerMetadataBuilder"]: + return DeepseekV41RocmMxfp4IndexerMetadataBuilder + + +class RocmMxfp4IndexerMetadataBuilder(DeepseekV32IndexerMetadataBuilder): + """Dense indexer metadata plus the in-place paged launches' layout. + + Requirements are checked here, at engine start. + """ + + @classmethod + def get_cudagraph_support( + cls, + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> AttentionCGSupport: + # Decode rows are flattened into persistent buffers, and whether a + # step launches them as next_n-row sequences depends on its shape + # alone (see native_decode). + return AttentionCGSupport.ALWAYS + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + **kwargs, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device, **kwargs) + name = "the ROCm MXFP4 indexer" + if self.dcp_world_size > 1 or self.use_pcp: + raise NotImplementedError( + f"{name} does not support decode/prefill context parallel." + ) + hf_config = vllm_config.model_config.hf_text_config + num_heads, head_dim = hf_config.index_n_heads, hf_config.index_head_dim + self.num_heads, self.head_dim = num_heads, head_dim + self.page_entries = kv_cache_spec.block_size // self.compress_ratio + self.cache_layout = rocm_paged_mxfp4_cache_layout( + num_heads, head_dim, self.page_entries + ) + # Ragged steps take a row per query token, as do the candidate + # consumers' gathers. Uniform steps also launch the dense layers on + # next_n-row sequences, see native_decode. + self.use_flattening = True + # A two-level (candidate block) indexer's builder sets these. + self.candidate_block_size = 0 + self.min_gather_width: float | None = None + + self.decode_block_ends_buffer = torch.zeros( + self.arange_buffer.shape[0], dtype=torch.int32, device=device + ) + self.context_lens_buffer = torch.zeros( + self.arange_buffer.shape[0], dtype=torch.int32, device=device + ) + spec_config = vllm_config.speculative_config + # Adaptive verification replays a FULL graph on drafts reallocated + # across requests, so a step's shape does not fix its query lengths. + adaptive = spec_config is not None and spec_config.enable_adaptive_verification + self.native_next_n = 1 if adaptive else self.num_speculative_tokens + 1 + # The indexer's logits width, which the decode schedule sizes slices by. + max_model_len = vllm_config.model_config.max_model_len + self.logits_width = max_model_len // self.compress_ratio + # Every dense layer of this group reads the same rows through the same + # page geometry, so the step builds their schedule once, not per layer. + self.decode_schedule_buffer = torch.empty( + rocm_mxfp4_decode_schedule_words( + num_heads, head_dim, self.page_entries, self.native_next_n + ), + dtype=torch.int32, + device=device, + ) + + def _split_indexer_prefill_chunks( # type: ignore[override] + self, + compressed_seq_lens_cpu: torch.Tensor, + prefill_query_lens_cpu: torch.Tensor, + workspace_size: int, + max_logits_bytes: int, + request_offset: int = 0, + ) -> list[tuple[slice, slice]]: + return split_prefill_chunks( + compressed_seq_lens_cpu, + prefill_query_lens_cpu, + max_logits_bytes, + request_offset, + ) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> RocmMxfp4IndexerMetadata: + base = super().build(common_prefix_len, common_attn_metadata, fast_build) + metadata = RocmMxfp4IndexerMetadata( + **{f.name: getattr(base, f.name) for f in fields(base)} + ) + cm = common_attn_metadata + context_lens = self.context_lens_buffer[: cm.num_reqs] + torch.floor_divide( + cm.seq_lens[: cm.num_reqs], self.compress_ratio, out=context_lens + ) + if base.decode is not None: + lengths = base.decode.seq_lens.view(-1) + metadata.decode_row_lens = lengths + block = self.candidate_block_size + if block: + ends = self.decode_block_ends_buffer[: lengths.shape[0]] + torch.add(lengths, block - 1, out=ends) + ends.floor_divide_(block) + metadata.decode_block_ends = ends + query_start_loc = cm.query_start_loc_cpu[: base.num_decodes + 1] + metadata.decode_native = native_decode( + lengths, + base.decode.block_table, + torch.diff(query_start_loc).tolist(), + self.native_next_n, + context_lens, + ) + metadata.decode_schedule = build_rocm_mxfp4_decode_schedule( + lengths, + self.num_heads, + self.head_dim, + self.page_entries, + self.decode_schedule_buffer, + self.logits_width, + metadata.decode_native, + ) + if base.prefill is not None: + assert cm.seq_lens_cpu_upper_bound is not None + metadata.prefill_plans = plan_prefill_chunks( + base.prefill.chunks, + cm.query_start_loc_cpu.tolist(), + cm.seq_lens_cpu_upper_bound, + context_lens, + self.compress_ratio, + self.candidate_block_size, + self.min_gather_width, + cm.query_start_loc, + ) + return metadata + + +class DeepseekV41RocmMxfp4IndexerMetadataBuilder(RocmMxfp4IndexerMetadataBuilder): + """DeepSeek-V4.1's two-level indexer: sets the candidate block size the + base plans with and, with ``indexer_sparse_logits``, adds the consumers' + gathers.""" + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + **kwargs, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device, **kwargs) + hf_config = vllm_config.model_config.hf_text_config + self.candidate_block_size = getattr(hf_config, "candidate_block_size", 0) + self.num_candidate_cols = 0 + if vllm_config.attention_config.indexer_sparse_logits: + prefix = "attention_config.indexer_sparse_logits" + block = self.candidate_block_size + topk_blocks = getattr(hf_config, "candidate_topk_blocks", 0) + layout = self.cache_layout + if block <= 0 or topk_blocks <= 0: + raise ValueError( + f"{prefix} requires a two-level (candidate block) indexer " + "model such as DeepSeek-V4.1." + ) + if topk_blocks & (topk_blocks - 1): + raise ValueError( + f"{prefix} on ROCm resolves the candidate pool with a sort " + f"that needs a power-of-two block count, not {topk_blocks}." + ) + if self.page_entries % block or block > layout.n_per_tile: + raise ValueError( + f"{prefix} on ROCm needs {block}-entry candidate blocks to " + f"tile a {self.page_entries}-entry indexer page and fit in " + f"one {layout.n_per_tile}-entry shuffle group." + ) + self.num_candidate_cols = topk_blocks * block + self.min_gather_width = self.num_candidate_cols * _GATHER_MIN_CUT_PREFILL + self.gather_rows = rocm_mxfp4_consumer_rows(self.num_candidate_cols) + logger.info_once( + "DeepSeek V4.1 indexer: candidate consumers score the %d " + "candidate positions with aiter's paged MXFP4 gather once the " + "context is %.1fx (decode) / %.1fx (prefill) as long.", + self.num_candidate_cols, + _GATHER_MIN_CUT_DECODE, + _GATHER_MIN_CUT_PREFILL, + ) + + def _gather_pays(self, context: int, cut: float) -> bool: + return self.num_candidate_cols > 0 and context >= cut * self.num_candidate_cols + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> RocmMxfp4IndexerMetadata: + metadata = super().build(common_prefix_len, common_attn_metadata, fast_build) + if metadata.decode is not None: + metadata.decode_use_gather = self._gather_pays( + metadata.max_seq_len // self.compress_ratio, _GATHER_MIN_CUT_DECODE + ) + if metadata.prefill is not None and self.num_candidate_cols: + metadata.gather_launches = plan_gather_launches( + metadata.prefill.chunks, metadata.prefill_plans, self.gather_rows + ) + return metadata diff --git a/vllm/v1/attention/ops/rocm_paged_mxfp4_indexer.py b/vllm/v1/attention/ops/rocm_paged_mxfp4_indexer.py new file mode 100644 index 0000000000..008b4729cf --- /dev/null +++ b/vllm/v1/attention/ops/rocm_paged_mxfp4_indexer.py @@ -0,0 +1,676 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sparse attention indexer on aiter's paged MXFP4 MQA-logits kernel. + +gfx950 only. The kernel reads the preshuffled paged indexer K cache in place, +so no layer gathers K into a contiguous buffer, prefill included. Any model +whose indexer supplies the inputs `rocm_mxfp4_sparse_attn_indexer` documents +can run the dense path. DeepSeek-V4.1's two-level indexer adds to it: the +candidate source takes its block maxima from the same walk that writes its +logits, and the candidate consumers can walk the candidate pool instead of +the whole context: the pool is resolved once per step and shared by all of +them. +""" + +import functools +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, NamedTuple + +import torch + +import vllm.envs as envs +from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.config import CUDAGraphMode +from vllm.forward_context import get_forward_context +from vllm.model_executor.kernels.attention.dsa.candidate_blocks import ( + apply_candidate_mask, +) +from vllm.triton_utils import tl, triton +from vllm.v1.attention.ops.rocm_aiter_mla_sparse import ( + _apply_candidate_mask_strided, + _max_decode_logits_rows, +) +from vllm.v1.worker.workspace import current_workspace_manager + +if TYPE_CHECKING: + from vllm.v1.attention.backends.mla.indexer import ( + DeepseekV32IndexerPrefillChunkMetadata, + ) + from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( + RocmMxfp4GatherLaunch, + RocmMxfp4IndexerMetadata, + RocmMxfp4NativeDecode, + RocmMxfp4PrefillPlan, + ) + +MXFP4_BLOCK_SIZE = 32 + + +@functools.cache +def _aiter(): + """The aiter module with the paged MXFP4 MQA-logits launcher, its schedule + and cache_format.""" + from aiter.ops.triton.attention import pa_mqa_logits_mxfp4 + + return pa_mqa_logits_mxfp4 + + +@functools.cache +def _aiter_topk() -> Callable[..., None]: + """The aiter per-row top-k.""" + from aiter.ops.topk import top_k_per_row_decode + + return top_k_per_row_decode + + +class RocmPagedMxfp4CacheLayout(NamedTuple): + """The byte order of an indexer K page, as the kernel reads it, and the + shuffle pattern aiter's K cache op writes it with. + + A page is cut into runs of ``n_per_tile`` tokens. Inside a run, each token's + packed values are split into ``d_per_tile``-byte chunks along the head + dimension, and the run stores chunk 0 of every token, then chunk 1, and so + on: ``[chunk, token, byte]``. Its e8m0 scales are stored as + ``[scale % scale_lanes, token, scale // scale_lanes]``. + """ + + n_per_tile: int + d_per_tile: int + scale_lanes: int + + +# The scale order the kernel reads: aiter's mode 1, a lane's scales innermost. +# cache_format does not report the order or the lane split yet. +_SCALE_ORDER = 1 +_WAVE_SIZE = 64 + + +@functools.cache +def rocm_paged_mxfp4_cache_layout( + num_heads: int, head_dim: int, page_entries: int +) -> RocmPagedMxfp4CacheLayout: + """The K page layout from aiter's ``cache_format``, the only place vLLM + reads it, so a change there cannot reach the writer unchecked. + + Raises: + ValueError: aiter rejects the page size, or describes an order this + layout cannot express. + + """ + fmt = _aiter().cache_format(num_heads, head_dim, page_entries) + try: + n_per_tile = int(fmt["n_per_tile"]) + d_per_tile = int(fmt["d_per_tile"]) + scale_order = fmt.get("scale_mode", _SCALE_ORDER) + scale_lanes = int(fmt.get("scale_lanes", _WAVE_SIZE // n_per_tile)) + except (KeyError, TypeError, ValueError, ZeroDivisionError) as e: + raise ValueError(f"aiter's cache_format() returned {fmt!r}") from e + if ( + scale_order != _SCALE_ORDER + or min(n_per_tile, d_per_tile, scale_lanes) <= 0 + or page_entries % n_per_tile + or (head_dim // 2) % d_per_tile + or (head_dim // MXFP4_BLOCK_SIZE) % scale_lanes + ): + raise ValueError( + f"aiter's cache_format() returned {fmt!r} for {num_heads} heads of " + f"{head_dim} in {page_entries}-entry pages, a K page order " + "RocmPagedMxfp4CacheLayout cannot express" + ) + return RocmPagedMxfp4CacheLayout(n_per_tile, d_per_tile, scale_lanes) + + +def rocm_mxfp4_decode_schedule_words( + num_heads: int, head_dim: int, page_entries: int, next_n: int = 1 +) -> int: + """int32 words of the largest schedule a decode step can build, flattened or + not. The slice cap can take it past target_wgs, up to SCHED_SLOT_CAP.""" + words = 4 * _aiter().SCHED_SLOT_CAP + for rows in {1, next_n}: + config = _aiter().select_config(num_heads, head_dim, rows, page_entries) + words = max(words, 4 * config["target_wgs"]) + return words + + +def build_rocm_mxfp4_decode_schedule( + row_lens: torch.Tensor, + num_heads: int, + head_dim: int, + page_entries: int, + out: torch.Tensor, + logits_width: int, + native: "RocmMxfp4NativeDecode | None" = None, +) -> torch.Tensor | None: + """Work descriptors that even out a decode step, or None where the static + grid already fills the machine. Depends only on the rows' lengths, the + cache geometry and the logits width, so one serves every layer of a group. + The width sizes the slices, capped the way the static grid caps them.""" + if native is None: + return _aiter().build_schedule( + row_lens, + 1, + num_heads, + head_dim, + page_entries, + out=out, + max_model_len=logits_width, + ) + return _aiter().build_schedule( + native.context_lens, + native.next_n, + num_heads, + head_dim, + page_entries, + out=out, + row_ends=row_lens, + max_model_len=logits_width, + ) + + +def _kv_view(kv_cache: torch.Tensor, head_dim: int) -> torch.Tensor: + """The indexer cache as [pages, entries, 1, bytes]. The page stride is the + block-major pool's, not the page's own size.""" + num_pages, entries, width = kv_cache.shape + assert kv_cache.dtype == torch.uint8 + assert width == head_dim // 2 + head_dim // MXFP4_BLOCK_SIZE, width + return torch.as_strided( + kv_cache, (num_pages, entries, 1, width), (kv_cache.stride(0), width, width, 1) + ) + + +def rocm_mxfp4_consumer_rows(num_candidate_cols: int) -> int: + """Query rows per candidate-consumer launch. Its logits are [rows, pool] + fp32 whatever the context, so the logits budget alone sizes it.""" + budget = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 + return max(1, budget // (4 * num_candidate_cols)) + + +def reserve_rocm_mxfp4_indexer_workspace( + hidden_states: torch.Tensor, + logits_width: int, + candidate_block_size: int = 0, + gather_block_size: int = 0, + num_candidate_cols: int = 0, +) -> None: + """Profiling run: claim the decode logits workspace and the peak prefill + logits, block scores included when the layer writes them, candidate lists + when it gathers.""" + rows = _max_decode_logits_rows(hidden_states.shape[0]) + specs = [((rows, logits_width), torch.float32)] + budget = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 + if candidate_block_size: + nblocks = triton.cdiv(logits_width, candidate_block_size) + specs.append(((rows, nblocks), torch.float32)) + budget += budget // candidate_block_size + if gather_block_size: + # aiter allocates the candidate lists: each gathered row, 8 B per + # candidate block (int32 slot and position). The first consumer builds + # them and they live until the step ends, across the later layers, so + # one reservation is held for the rest of the forward. + held = get_forward_context().additional_kwargs + if "rocm_mxfp4_candidate_lists" not in held: + held["rocm_mxfp4_candidate_lists"] = torch.empty( + hidden_states.shape[0] * (num_candidate_cols // gather_block_size) * 8, + dtype=torch.uint8, + device=hidden_states.device, + ) + current_workspace_manager().get_simultaneous(*specs) + torch.empty(budget, dtype=torch.uint8, device=hidden_states.device) + + +def _topk( + logits: torch.Tensor, lengths: torch.Tensor, out: torch.Tensor, k: int +) -> None: + """Top-k over each row's [0, lengths[row]); -1 pads rows shorter than k. + + aiter's kernel is faster on the candidate top-k and from 128 rows on; below + that, a wide decode step is faster on vLLM's. Shape alone decides, so a FULL + graph replays the same choice at every context length. + """ + rows, width = logits.shape + if rows == 0: + return + if width >= k and (k >= 2048 or rows >= 128): + _aiter_topk()( + logits, + 1, + lengths.reshape(-1), + out, + rows, + logits.stride(0), + logits.stride(1), + k=k, + ) + else: + torch.ops._C.top_k_per_row_decode( + logits, 1, lengths, out, rows, logits.stride(0), logits.stride(1), k + ) + + +@triton.jit +def _remap_compact_topk_kernel( + idx_ptr, + idx_stride, + pos_ptr, + pos_stride, + K: tl.constexpr, + BLOCK: tl.constexpr, + PADDED_K: tl.constexpr, +): + row = tl.program_id(0).to(tl.int64) + cols = tl.arange(0, PADDED_K) + live = cols < K + slot = tl.load(idx_ptr + row * idx_stride + cols, mask=live, other=-1) + hit = slot >= 0 + slot = tl.where(hit, slot, 0) + start = tl.load( + pos_ptr + row * pos_stride + slot // BLOCK, mask=live & hit, other=0 + ) + tl.store( + idx_ptr + row * idx_stride + cols, + tl.where(hit, start + slot % BLOCK, -1).to(tl.int32), + mask=live, + ) + + +def _remap_compact_topk( + indices: torch.Tensor, positions: torch.Tensor, block: int +) -> None: + """Candidate slots to context positions, in place: slot j sits in pool + block j // block, which starts at positions[j // block].""" + rows, k = indices.shape + if rows == 0: + return + _remap_compact_topk_kernel[(rows,)]( + indices, + indices.stride(0), + positions, + positions.stride(0), + K=k, + BLOCK=block, + PADDED_K=triton.next_power_of_2(k), + num_warps=4, + ) + + +@dataclass +class _Layer: + """One indexer layer's inputs and outputs for this step.""" + + metadata: "RocmMxfp4IndexerMetadata" + kv: torch.Tensor + q: torch.Tensor + q_scale: torch.Tensor + weights: torch.Tensor + topk_buffer: torch.Tensor + topk_tokens: int + compress_ratio: int + candidates: torch.Tensor | None + block: int + full_graph: bool + + @property + def num_heads(self) -> int: + return self.q.shape[1] + + @property + def head_dim(self) -> int: + return self.q.shape[2] * 2 + + def rows(self, lo: int, hi: int, seqs: int = 1) -> tuple[torch.Tensor, ...]: + """Q, its scales and the weights of token rows [lo, hi), as ``seqs`` + sequences of next_n rows each.""" + return ( + self.q[lo:hi].view(seqs, -1, *self.q.shape[1:]), + self.q_scale[lo:hi].view(seqs, -1, *self.q_scale.shape[1:]), + self.weights[lo:hi], + ) + + def decode_rows(self, n: int) -> tuple[torch.Tensor, ...]: + """The first n token rows, one sequence each.""" + return ( + self.q[:n].unsqueeze(1), + self.q_scale[:n].unsqueeze(1), + self.weights[:n], + ) + + +def _layer( + k_cache_prefix: str, + kv_cache: torch.Tensor, + q_values: torch.Tensor, + q_scale: torch.Tensor, + weights: torch.Tensor, + topk_indices_buffer: torch.Tensor, + topk_tokens: int, + head_dim: int, + compress_ratio: int, + candidate_blocks: torch.Tensor | None, + candidate_block_size: int, +) -> _Layer: + from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( + RocmMxfp4IndexerMetadata, + ) + + forward_context = get_forward_context() + attn_metadata = forward_context.attn_metadata + assert isinstance(attn_metadata, dict) + metadata = attn_metadata[k_cache_prefix] + assert isinstance(metadata, RocmMxfp4IndexerMetadata), ( + "the ROCm MXFP4 indexer needs RocmMxfp4IndexerMetadataBuilder metadata" + ) + kv = _kv_view(kv_cache, head_dim) + num_heads = q_values.shape[1] + return _Layer( + metadata=metadata, + kv=kv, + q=q_values, + # The fused Q kernel returns one int32 per head; the kernel reads the + # four ue8m0 bytes behind it. + q_scale=q_scale.view(torch.uint8).view(q_scale.shape[0], num_heads, -1), + weights=weights, + topk_buffer=topk_indices_buffer, + topk_tokens=topk_tokens, + compress_ratio=compress_ratio, + candidates=candidate_blocks, + block=candidate_block_size, + full_graph=forward_context.cudagraph_runtime_mode == CUDAGraphMode.FULL, + ) + + +def _block_scores(layer: _Layer, scores: torch.Tensor | None) -> dict: + if scores is None: + return {} + return dict( + calc_block_scores=True, block_scores=scores, candidate_block_size=layer.block + ) + + +def _dense_prefill( + layer: _Layer, + chunk: "DeepseekV32IndexerPrefillChunkMetadata", + plan: "RocmMxfp4PrefillPlan", + candidate_write: bool, +) -> None: + pa = _aiter() + t0, t1 = chunk.token_start, chunk.token_end + logits = layer.q.new_empty((t1 - t0, plan.width), dtype=torch.float32) + scores = None + if candidate_write: + nblocks = triton.cdiv(plan.width, layer.block) + scores = layer.q.new_empty((t1 - t0, nblocks), dtype=torch.float32) + # One launch for the chunk: the kernel finds each row's request from + # query_start_loc on its own grid. + pa.paged_mxfp4_mqa_logits( + layer.q[t0:t1], + layer.q_scale[t0:t1], + layer.kv, + layer.weights[t0:t1], + plan.context_lens, + chunk.block_table, + plan.width, + out_logits=logits, + clean_logits=False, + row_ends=plan.row_ends, + query_start_loc=plan.query_start_loc, + **_block_scores(layer, scores), + ) + candidates = None if layer.candidates is None else layer.candidates[t0:t1] + if scores is not None: + assert candidates is not None and plan.block_ends is not None + _topk(scores, plan.block_ends, candidates, candidates.shape[1]) + elif candidates is not None: + apply_candidate_mask(logits, None, plan.row_ends, candidates, layer.block) + _topk( + logits, + plan.row_ends, + layer.topk_buffer[t0:t1, : layer.topk_tokens], + layer.topk_tokens, + ) + + +def _dense_decode(layer: _Layer, logits_width: int, candidate_write: bool) -> None: + metadata = layer.metadata + lengths = metadata.decode_row_lens + assert metadata.decode is not None and lengths is not None + rows = lengths.shape[0] + specs = [((rows, logits_width), torch.float32)] + if candidate_write: + specs.append(((rows, triton.cdiv(logits_width, layer.block)), torch.float32)) + logits, *scores = current_workspace_manager().get_simultaneous(*specs) + native = metadata.decode_native + if native is not None: + # A request's next_n rows go in as one sequence, so a workgroup walks + # each KV tile once for all of them. + q, q_scale, weights = layer.rows(0, rows, native.context_lens.shape[0]) + context_lens, block_table, row_ends = ( + native.context_lens, + native.block_table, + lengths, + ) + else: + q, q_scale, weights = layer.decode_rows(rows) + context_lens, block_table, row_ends = lengths, metadata.decode.block_table, None + assert block_table.shape[0] == rows, "one block-table row per query row" + _aiter().paged_mxfp4_mqa_logits( + q, + q_scale, + layer.kv, + weights, + context_lens, + block_table, + logits_width, + out_logits=logits, + clean_logits=False, + row_ends=row_ends, + schedule=metadata.decode_schedule, + **_block_scores(layer, scores[0] if scores else None), + ) + candidates = None if layer.candidates is None else layer.candidates[:rows] + if scores: + assert candidates is not None and metadata.decode_block_ends is not None + _topk(scores[0], metadata.decode_block_ends, candidates, candidates.shape[1]) + elif candidates is not None: + _apply_candidate_mask_strided(logits, None, lengths, candidates, layer.block) + _topk( + logits, + lengths, + layer.topk_buffer[:rows, : layer.topk_tokens], + layer.topk_tokens, + ) + + +def _gather_prefill( + layer: _Layer, launch: "RocmMxfp4GatherLaunch", num_cols: int +) -> None: + pa = _aiter() + assert layer.candidates is not None + t0, t1 = launch.token_start, launch.token_end + if launch.pool is None: + launch.pool = pa.build_candidate_gather( + layer.candidates[t0:t1], + launch.row_ends, + launch.block_table.expand(t1 - t0, -1), + layer.kv, + layer.num_heads, + layer.head_dim, + layer.block, + ) + gather, slot_ends = launch.pool + compact = layer.q.new_empty((t1 - t0, num_cols), dtype=torch.float32) + q, q_scale, weights = layer.rows(t0, t1) + pa.paged_mxfp4_mqa_logits( + q, + q_scale, + layer.kv, + weights, + launch.context_len, + launch.block_table, + num_cols, + out_logits=compact, + clean_logits=False, + row_ends=slot_ends, + use_gather=True, + candidates=gather, + ) + out = layer.topk_buffer[t0:t1, : layer.topk_tokens] + _topk(compact, slot_ends, out, layer.topk_tokens) + _remap_compact_topk(out, gather["positions"], layer.block) + + +def _gather_decode(layer: _Layer, num_cols: int) -> None: + pa = _aiter() + assert layer.candidates is not None + metadata = layer.metadata + lengths = metadata.decode_row_lens + assert metadata.decode is not None and lengths is not None + block_table = metadata.decode.block_table + rows = lengths.shape[0] + if metadata.decode_gather is None: + metadata.decode_gather = pa.build_candidate_gather( + layer.candidates[:rows], + lengths, + block_table, + layer.kv, + layer.num_heads, + layer.head_dim, + layer.block, + ) + gather, slot_ends = metadata.decode_gather + (compact,) = current_workspace_manager().get_simultaneous( + ((rows, num_cols), torch.float32) + ) + q, q_scale, weights = layer.decode_rows(rows) + pa.paged_mxfp4_mqa_logits( + q, + q_scale, + layer.kv, + weights, + lengths, + block_table, + num_cols, + out_logits=compact, + clean_logits=False, + row_ends=slot_ends, + use_gather=True, + candidates=gather, + ) + out = layer.topk_buffer[:rows, : layer.topk_tokens] + _topk(compact, slot_ends, out, layer.topk_tokens) + _remap_compact_topk(out, gather["positions"], layer.block) + + +@eager_break_during_capture +def rocm_mxfp4_sparse_attn_indexer( + hidden_states: torch.Tensor, + k_cache_prefix: str, + kv_cache: torch.Tensor, + q_values: torch.Tensor, + q_scale: torch.Tensor, + weights: torch.Tensor, + topk_tokens: int, + head_dim: int, + max_model_len: int, + topk_indices_buffer: torch.Tensor, + compress_ratio: int = 1, + candidate_blocks: torch.Tensor | None = None, + candidate_block_size: int = 0, + candidate_write: bool = False, +) -> torch.Tensor: + """Dense indexer: every layer that scores the whole context, the + candidate source included. With ``candidate_blocks`` and not + ``candidate_write`` the scores are masked to the pool first, as the + shared path does. + + The model supplies ``q_values`` [T, H, D // 2] packed e2m1, ``q_scale`` + holding each head's D // 32 ue8m0 bytes (one int32 per head at D = 128), + fp32 ``weights`` with the softmax and head scales folded in, and + ``kv_cache`` pages in `rocm_paged_mxfp4_cache_layout` order for H heads. + """ + if not isinstance(get_forward_context().attn_metadata, dict): + reserve_rocm_mxfp4_indexer_workspace( + hidden_states, + max_model_len, + candidate_block_size if candidate_write else 0, + ) + return topk_indices_buffer + layer = _layer( + k_cache_prefix, + kv_cache, + q_values, + q_scale, + weights, + topk_indices_buffer, + topk_tokens, + head_dim, + compress_ratio, + candidate_blocks, + candidate_block_size, + ) + metadata = layer.metadata + topk_indices_buffer[: hidden_states.shape[0]] = -1 + if metadata.prefill is not None: + for chunk, plan in zip(metadata.prefill.chunks, metadata.prefill_plans): + _dense_prefill(layer, chunk, plan, candidate_write) + if metadata.decode is not None: + _dense_decode(layer, max_model_len, candidate_write) + return topk_indices_buffer + + +@eager_break_during_capture +def rocm_mxfp4_sparse_mqa_indexer( + hidden_states: torch.Tensor, + k_cache_prefix: str, + kv_cache: torch.Tensor, + q_values: torch.Tensor, + q_scale: torch.Tensor, + weights: torch.Tensor, + topk_tokens: int, + head_dim: int, + max_model_len: int, + topk_indices_buffer: torch.Tensor, + compress_ratio: int, + candidate_blocks: torch.Tensor, + candidate_block_size: int, + num_candidate_cols: int, +) -> torch.Tensor: + """Candidate consumer: score only the source's pool where the builder's + length gate says it pays, else the dense walk masked to the pool.""" + if not isinstance(get_forward_context().attn_metadata, dict): + reserve_rocm_mxfp4_indexer_workspace( + hidden_states, + max(max_model_len, num_candidate_cols), + gather_block_size=candidate_block_size, + num_candidate_cols=num_candidate_cols, + ) + return topk_indices_buffer + layer = _layer( + k_cache_prefix, + kv_cache, + q_values, + q_scale, + weights, + topk_indices_buffer, + topk_tokens, + head_dim, + compress_ratio, + candidate_blocks, + candidate_block_size, + ) + metadata = layer.metadata + topk_indices_buffer[: hidden_states.shape[0]] = -1 + if metadata.prefill is not None: + for chunk, plan in zip(metadata.prefill.chunks, metadata.prefill_plans): + if not plan.use_gather: + _dense_prefill(layer, chunk, plan, candidate_write=False) + for launch in metadata.gather_launches: + _gather_prefill(layer, launch, num_candidate_cols) + if metadata.decode is not None: + # A FULL graph cannot follow the per-step gate; the gather is the side + # that stays flat in the context length. + if metadata.decode_use_gather or layer.full_graph: + _gather_decode(layer, num_candidate_cols) + else: + _dense_decode(layer, max_model_len, candidate_write=False) + return topk_indices_buffer From 65ed0a2642d2fb1d897408ec39f64b1cb74f0369 Mon Sep 17 00:00:00 2001 From: burak Date: Tue, 29 Sep 2026 09:45:29 +0300 Subject: [PATCH 004/248] [Bugfix][GLM-5.3-Flash] Take video placeholder timestamps from the pixel path's frame sampler (#55647) Signed-off-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Signed-off-by: Isotr0py Co-authored-by: NNNtrance <232268405+NNNtrance@users.noreply.github.com> Co-authored-by: Isotr0py --- .../multimodal/processing/test_glm5next.py | 170 ++++++++++++++++++ vllm/models/glm5next/common/multimodal.py | 59 ++++++ 2 files changed, 229 insertions(+) create mode 100644 tests/models/multimodal/processing/test_glm5next.py diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py new file mode 100644 index 0000000000..6fb0e397c2 --- /dev/null +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Video placeholder accounting for GLM-5.3-Flash. + +``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of +placeholders per timestamp, so the timestamps returned by +``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets +while ``video_grid_thw`` decides how many rows the vision tower produces. If +the two disagree they collide in ``_merge_multimodal_embeddings``, which raises +inside a worker and takes the engine down with it. + +The checks below are arithmetic: no weights, no GPU. +""" + +import pytest + +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.transformers_utils.processors.glm5next import ( + Glm5NextVideoProcessor, + _pixel_budget, + glm_sample_frame_indices, + smart_resize, +) + +from ...utils import build_model_context + + +@pytest.fixture(scope="module") +def processor(): + ctx = build_model_context( + "zai-org/GLM-5.3-Flash", + limit_mm_per_prompt={"video": 1}, + ) + return MULTIMODAL_REGISTRY.create_processor( + ctx.model_config, + tokenizer=ctx.tokenizer, + ) + + +def _pixel_path_grid( + video_processor: Glm5NextVideoProcessor, + num_frames: int, + height: int, + width: int, +) -> tuple[int, int, int]: + """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds.""" + min_pixels, max_pixels = _pixel_budget( + video_processor.min_image_tokens, + video_processor.max_image_tokens, + video_processor.patch_size, + video_processor.merge_size, + video_processor.temporal_patch_size, + ) + factor = ( + video_processor.patch_size + * video_processor.merge_size + * video_processor.patch_expand_factor + ) + resized_height, resized_width = smart_resize( + t=num_frames, + h=height, + w=width, + t_factor=video_processor.temporal_patch_size, + h_factor=factor, + w_factor=factor, + min_pixels=min_pixels, + max_pixels=max_pixels, + ) + padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size) + return ( + padded_frames // video_processor.temporal_patch_size, + resized_height // video_processor.patch_size, + resized_width // video_processor.patch_size, + ) + + +@pytest.mark.parametrize( + ("total_num_frames", "fps", "duration", "height", "width", "expected_grid"), + [ + # 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps. + (32, 8.0, 4.0, 480, 640, (4, 36, 46)), + # 1080p, 20 s: same factor of 3 at a full-size canvas. + (600, 30.0, 20.0, 1080, 1920, (20, 58, 102)), + # 1080p, 60 s: the one duration window where the two samplers agree + # anyway -- a regression guard, the count must not move. + (1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)), + # Past 300 s the GLM-4.6V sampler asks for half as many instead. + (9030, 30.0, 301.0, 720, 1280, (301, 14, 26)), + ], +) +def test_video_placeholders_match_encoder_rows( + processor, + total_num_frames: int, + fps: float, + duration: float, + height: int, + width: int, + expected_grid: tuple[int, int, int], +): + info = processor.info + video_processor = info.get_video_processor() + + frame_indices = glm_sample_frame_indices( + total_num_frames, + fps, + duration, + target_fps=video_processor.fps_interval, + max_frame_count=video_processor.max_frame_count_dynamic, + temporal_patch_size=video_processor.temporal_patch_size, + ) + grid_t, grid_h, grid_w = _pixel_path_grid( + video_processor, len(frame_indices), height, width + ) + assert (grid_t, grid_h, grid_w) == expected_grid + + timestamps = info._get_video_second_idx_glm46v( + { + "total_num_frames": total_num_frames, + "fps": fps, + "duration": duration, + "do_sample_frames": True, + }, + total_num_frames, + ) + + merge_length = video_processor.merge_size**2 + tokens_per_frame = grid_h * grid_w // merge_length + encoder_rows = grid_t * grid_h * grid_w // merge_length + + assert len(timestamps) == grid_t + assert len(timestamps) * tokens_per_frame == encoder_rows + assert timestamps == sorted(timestamps) + assert timestamps[0] == 0 + assert timestamps[-1] <= duration + + +def test_video_placeholders_match_encoder_rows_when_presampled(processor): + """The loader may pre-sample and hand the frames over as they are.""" + info = processor.info + video_processor = info.get_video_processor() + + num_frames = 32 + grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640) + + timestamps = info._get_video_second_idx_glm46v( + { + "total_num_frames": 256, + "fps": 8.0, + "duration": 32.0, + "do_sample_frames": False, + "frames_indices": list(range(0, 256, 256 // num_frames)), + }, + num_frames, + ) + + assert len(timestamps) == grid_t + + +def test_video_shorter_than_one_sampling_interval_is_rejected(processor): + """A clip the sampler cannot pick a single frame from is a bad request.""" + with pytest.raises(ValueError, match="selected no frames"): + processor.info._get_video_second_idx_glm46v( + { + "total_num_frames": 2, + "fps": 8.0, + "duration": 0.25, + "do_sample_frames": True, + }, + 2, + ) diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index d2b21345d0..03ac3fa664 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -4,6 +4,7 @@ from collections.abc import Mapping from functools import cached_property, partial +from typing import Any import numpy as np import torch @@ -42,6 +43,7 @@ from vllm.model_executor.models.vision import ( ) from vllm.models.common.ops import fused_q_kv_rmsnorm from vllm.multimodal.parse import ImageSize, MultiModalDataItems +from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -656,6 +658,63 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): return int(override) return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1] + def _get_video_second_idx_glm46v( + self, metadata: dict[str, Any], total_frames: int + ) -> list[int]: + """Timestamps for the video placeholder, from the pixel path's sampler. + + The inherited GLM-4.6V timestamp sampling uses different constants + than this checkpoint's ``Glm5NextVideoProcessor`` pixel sampling, so + placeholder count and ``video_grid_thw`` disagree outside 30-300s + durations. Sampling with the pixel path's own policy makes one + timestamp per temporal patch == ``grid_t``. + """ + video_processor = self.get_video_processor() + temporal_patch_size = int( + getattr(video_processor, "temporal_patch_size", 1) or 1 + ) + video_fps = float(metadata["fps"]) + + if not metadata.get("do_sample_frames", True): + # The loader already picked the frames and the processor keeps them + # as they are, so the frame count is what was handed in and + # ``frames_indices`` only supplies the timestamp values. + frame_indices = [int(idx) for idx in metadata["frames_indices"]] + num_frames = int(total_frames) + else: + frame_indices = [ + int(idx) + for idx in glm_sample_frame_indices( + int(metadata.get("total_num_frames", total_frames)), + video_fps, + float(metadata.get("duration") or 0), + target_fps=video_processor.fps_interval, + max_frame_count=video_processor.max_frame_count_dynamic, + temporal_patch_size=temporal_patch_size, + ) + ] + num_frames = len(frame_indices) + + if not frame_indices or num_frames <= 0: + # ``duration * fps_interval < 1`` selects no frames at all; the + # pixel path raises an opaque IndexError on the same input a moment + # later, so say what happened while there is still a request. + raise ValueError( + "The GLM-5-Next frame sampler selected no frames for this " + f"video (fps={video_fps}, duration={metadata.get('duration')!r}," + f" total_num_frames={metadata.get('total_num_frames')!r}). " + "Clips shorter than one frame sampling interval cannot be " + "processed." + ) + + num_padded_frames = num_frames + (-num_frames % temporal_patch_size) + grid_t = max(num_padded_frames // temporal_patch_size, 1) + last = len(frame_indices) - 1 + return [ + int(frame_indices[min(i * temporal_patch_size, last)] / video_fps) + for i in range(grid_t) + ] + def _get_vision_info( self, *, From d86cf91240988310d04f75217a2ea7e76f1db3d6 Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Tue, 29 Sep 2026 08:53:38 +0200 Subject: [PATCH 005/248] [MM][Mistral3] Image preprocessing optimization (#57531) Signed-off-by: Jakub Zakrzewski Co-authored-by: OpenAI Codex Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docs/design/mm_processing.md | 23 ++-- .../multimodal/processing/test_mistral3.py | 130 +++++++++++++++++- .../layers/fusion/mm_input_norm.py | 5 + vllm/model_executor/models/lightonocr.py | 2 + vllm/model_executor/models/mistral3.py | 22 ++- vllm/model_executor/models/pixtral.py | 71 ++++++++-- 6 files changed, 228 insertions(+), 25 deletions(-) diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index d350836ca8..f53f7d2dd1 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -78,17 +78,18 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: - The flag is **enabled by default** for all models that support it. - Currently, it’s on by default for these architectures: -| name | Architecture | Example HF Models | -|--------------------|------------------------------------------|----------------------------------------| -| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | -| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | -| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | -| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | -| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. | -| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | -| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. | -| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. | -| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` | +| name | Architecture | Example HF Models | +|--------------------|------------------------------------------|-------------------------------------------------------| +| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | +| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | +| `mistral3` | `Mistral3ForConditionalGeneration` | `mistralai/Mistral-Small-3.2-24B-Instruct-2506`, etc. | +| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | +| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | +| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. | +| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | +| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. | +| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. | +| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` | #### Key Properties and Gains diff --git a/tests/models/multimodal/processing/test_mistral3.py b/tests/models/multimodal/processing/test_mistral3.py index 3445dd86fe..87ca8c842b 100644 --- a/tests/models/multimodal/processing/test_mistral3.py +++ b/tests/models/multimodal/processing/test_mistral3.py @@ -4,14 +4,29 @@ import pytest import torch +import torch.nn.functional as F from PIL import Image from transformers import AutoProcessor, BatchFeature +from transformers.models.pixtral import PixtralProcessor -from vllm.model_executor.models.lightonocr import LightOnOCRProcessingInfo +from vllm.model_executor.layers.fusion.mm_input_norm import ( + FusedMMInputNorm, + IdentityInputNorm, + build_mm_input_norm, +) +from vllm.model_executor.models.lightonocr import ( + LightOnOCRForConditionalGeneration, + LightOnOCRProcessingInfo, +) from vllm.model_executor.models.mistral3 import Mistral3HFEncoderInfo -from vllm.model_executor.models.pixtral import PixtralHFEncoderInfo +from vllm.model_executor.models.pixtral import ( + PixtralHFEncoderInfo, + pixtral_patch_embed, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalKwargsItems +from vllm.platforms import current_platform +from vllm.triton_utils import HAS_TRITON from ...utils import build_model_context @@ -21,6 +36,65 @@ _MODEL_CONFIG_KWARGS = {"config_format": "hf"} _MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" _LIGHTON_MODEL_ID = "lightonai/LightOnOCR-1B-1025" +_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073] +_RGB_STD = [0.26862954, 0.26130258, 0.27577711] +_RGB_RESCALE = 1.0 / 255.0 + + +@pytest.mark.usefixtures("default_vllm_config") +@pytest.mark.skipif( + not HAS_TRITON or not current_platform.is_cuda(), reason="CUDA Triton kernel" +) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("normalize", [False, True]) +@pytest.mark.parametrize(("channels", "patch_shape"), [(3, (4, 4)), (2, (2, 4))]) +def test_pixtral_patch_embed_preserves_image_order_and_strides( + dtype: torch.dtype, + normalize: bool, + channels: int, + patch_shape: tuple[int, int], +): + """Packed projection matches CHW normalization and patch convolution.""" + device = torch.device(current_platform.device_type) + norm = ( + FusedMMInputNorm( + _RGB_MEAN[:channels], + _RGB_STD[:channels], + _RGB_RESCALE, + channel=channels, + ).to(device) + if normalize + else IdentityInputNorm() + ) + weight = torch.randn(32, channels, *patch_shape, device=device, dtype=dtype) + images = [ + torch.randint(0, 256, (channels, 17, 18), device=device, dtype=torch.uint8), + torch.randint(0, 256, (channels, 25, 17), device=device, dtype=torch.uint8)[ + :, 1:17, 1:17 + ], + torch.randint(0, 256, (channels, 19, 20), device=device, dtype=torch.uint8)[ + :, 1:17, 2:18 + ], + ] + reference = [] + for image in images: + if normalize: + normalized = ( + image.float() * norm.weight[:, None, None] + norm.bias[:, None, None] + ) + else: + normalized = image + reference.append( + F.conv2d(normalized.to(dtype).unsqueeze(0), weight, stride=patch_shape) + ) + packed, actual = pixtral_patch_embed(images, weight, norm) + for expected, result in zip(reference, actual): + torch.testing.assert_close(result, expected, rtol=0.01, atol=0.02) + expected_packed = torch.cat( + [result.flatten(2).transpose(1, 2) for result in reference], dim=1 + ) + torch.testing.assert_close(packed, expected_packed, rtol=0.01, atol=0.02) + def _process_images_with_hf( hf_processor, @@ -128,6 +202,9 @@ def test_processor_size_override( ) processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs + vllm_hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs) + assert isinstance(vllm_hf_processor, PixtralProcessor) + hf_processor = AutoProcessor.from_pretrained( model_id, fix_mistral_regex=True, @@ -153,6 +230,53 @@ def test_processor_size_override( assert prompt_update_tokens == hf_placeholder_tokens +@pytest.mark.usefixtures("default_vllm_config") +def test_mm_device_do_normalize(): + ctx = build_model_context( + _MODEL_ID, + limit_mm_per_prompt={"image": 2}, + model_config_kwargs=_MODEL_CONFIG_KWARGS, + ) + assert ctx.model_config.multimodal_config.mm_device_do_normalize + + ctx.model_config.multimodal_config.mm_device_do_normalize = False + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + images = [ + Image.new("RGB", (31, 47), color=(17, 89, 231)), + Image.new("RGB", (48, 32), color=(201, 13, 127)), + ] + prompt = [processor.info.get_hf_config().image_token_index] * len(images) + mm_items = processor.info.parse_mm_data({"image": images}) + + normalized_inputs = processor(prompt, mm_items=mm_items) + normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + + ctx.model_config.multimodal_config.mm_device_do_normalize = True + raw_inputs = processor(prompt, mm_items=mm_items) + raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] + assert all(value.dtype == torch.uint8 for value in raw_values) + + input_norm = build_mm_input_norm(ctx.model_config) + patch_size = processor.info.get_hf_config().vision_config.patch_size + + def pack_patches(image: torch.Tensor) -> torch.Tensor: + channels, height, width = image.shape + rows = height // patch_size + cols = width // patch_size + return ( + image[:, : rows * patch_size, : cols * patch_size] + .reshape(channels, rows, patch_size, cols, patch_size) + .permute(1, 3, 0, 2, 4) + .reshape(-1, channels * patch_size**2) + ) + + for raw, normalized in zip(raw_values, normalized_values): + output = input_norm(pack_patches(raw), normalized.dtype) + torch.testing.assert_close( + output, pack_patches(normalized), rtol=1e-5, atol=1e-6 + ) + + def test_scoped_request_size_overrides_configured_flat_size(): ctx = build_model_context( _MODEL_ID, @@ -188,6 +312,8 @@ def test_scoped_request_size_overrides_configured_flat_size(): def test_lightonocr_keeps_vision_config_image_size(): + assert not LightOnOCRForConditionalGeneration.supports_mm_device_do_normalize + ctx = build_model_context( _LIGHTON_MODEL_ID, mm_processor_kwargs={"size": {"longest_edge": 1008}}, diff --git a/vllm/model_executor/layers/fusion/mm_input_norm.py b/vllm/model_executor/layers/fusion/mm_input_norm.py index f03cfa4c36..5127f1007e 100644 --- a/vllm/model_executor/layers/fusion/mm_input_norm.py +++ b/vllm/model_executor/layers/fusion/mm_input_norm.py @@ -147,6 +147,11 @@ def _load_norm_params(model_config: ModelConfig) -> NormParams: rescale_factor)`` from the processor config.""" config = get_processor_config(model_config.model, revision=model_config.revision) config = config.get("image_processor", config) + if not any( + key in config + for key in ("do_normalize", "do_rescale", "image_mean", "image_std") + ): + config = model_config.hf_image_processor_config has_norm_params = "image_mean" in config and "image_std" in config do_normalize = bool(config.get("do_normalize", has_norm_params)) diff --git a/vllm/model_executor/models/lightonocr.py b/vllm/model_executor/models/lightonocr.py index d8a1240242..bc0292ed55 100644 --- a/vllm/model_executor/models/lightonocr.py +++ b/vllm/model_executor/models/lightonocr.py @@ -115,6 +115,8 @@ class LightOnOCRMultiModalProcessor(BaseMultiModalProcessor[LightOnOCRProcessing dummy_inputs=Mistral3DummyInputsBuilder, ) class LightOnOCRForConditionalGeneration(Mistral3ForConditionalGeneration): + supports_mm_device_do_normalize = False + hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "model.vision_encoder.": "vision_tower.", diff --git a/vllm/model_executor/models/mistral3.py b/vllm/model_executor/models/mistral3.py index 593ba90ec0..40af706e3a 100644 --- a/vllm/model_executor/models/mistral3.py +++ b/vllm/model_executor/models/mistral3.py @@ -7,6 +7,8 @@ from typing import Annotated, Any, Literal, cast import torch import torch.nn as nn +import transformers +from packaging.version import Version from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig from transformers.models.pixtral import PixtralProcessor @@ -14,6 +16,7 @@ from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -56,6 +59,10 @@ from .utils import ( maybe_prefix, ) +TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version( + "5.15.0" +) + class Mistral3ImagePixelInputs(TensorSchema): """Dimensions: @@ -225,7 +232,7 @@ class Mistral3ProcessingInfo(BaseProcessingInfo): image_size = size["longest_edge"] return Mistral3HFEncoderInfo(self.get_hf_config(), image_size) - def get_hf_processor(self, **kwargs: object): + def get_hf_processor(self, **kwargs: object) -> PixtralProcessor: return self.ctx.get_hf_processor(PixtralProcessor, **kwargs) def get_supported_mm_limits(self) -> Mapping[str, int | None]: @@ -277,8 +284,12 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder): class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]): - def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: - return self.dummy_inputs.get_dummy_text(mm_counts) + def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None: + # PixtralProcessor supports image-only calls starting in transformers 5.15. + # Older releases need dummy text and take the slower tokenization path. + if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY: + return self.dummy_inputs.get_dummy_text(mm_counts) + return None def _postprocess_hf_mm_data( self, @@ -378,6 +389,7 @@ def init_vision_tower_for_mistral3( hf_config: Mistral3Config, quant_config: QuantizationConfig | None, *, + input_norm: nn.Module | None = None, require_post_norm: bool | None = None, prefix: str = "", ) -> PixtralHFVisionModel: @@ -391,6 +403,7 @@ def init_vision_tower_for_mistral3( return PixtralHFVisionModel( vision_config, quant_config=quant_config, + input_norm=input_norm, num_hidden_layers_override=num_hidden_layers, require_post_norm=require_post_norm, prefix=prefix, @@ -410,6 +423,8 @@ class Mistral3ForConditionalGeneration( SupportsEagle, SupportsEagle3, ): + supports_mm_device_do_normalize = True + packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], @@ -464,6 +479,7 @@ class Mistral3ForConditionalGeneration( self.vision_tower = init_vision_tower_for_mistral3( config, quant_config=quant_config, + input_norm=build_mm_input_norm(vllm_config.model_config), require_post_norm=False, prefix=maybe_prefix(prefix, "vision_tower"), ) diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 91ac38e110..521df62a92 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -10,6 +10,7 @@ from typing import Annotated, Literal import numpy as np import torch import torch.nn as nn +import torch.nn.functional as F from mistral_common.protocol.instruct.chunk import ImageChunk, TextChunk from mistral_common.protocol.instruct.messages import UserMessage from mistral_common.protocol.instruct.request import ChatCompletionRequest @@ -1427,12 +1428,55 @@ class PixtralHFTransformer(nn.Module): return x +def pixtral_patch_embed( + images: list[torch.Tensor], + weight: torch.Tensor, + input_norm: nn.Module, +) -> tuple[torch.Tensor, list[torch.Tensor]]: + """Normalize and project all CHW images in one matrix multiplication.""" + out_channels, in_channels, patch_height, patch_width = weight.shape + grid_sizes = [] + for image in images: + assert image.ndim == 3 and image.shape[0] == in_channels + _, height, width = image.shape + patch_rows = height // patch_height + patch_cols = width // patch_width + grid_sizes.append((patch_rows, patch_cols)) + + patch_counts = [rows * cols for rows, cols in grid_sizes] + packed = torch.empty( + (sum(patch_counts), in_channels * patch_height * patch_width), + dtype=images[0].dtype, + device=images[0].device, + ) + offset = 0 + for image, (rows, cols), count in zip(images, grid_sizes, patch_counts): + source = ( + image[:, : rows * patch_height, : cols * patch_width] + .reshape(in_channels, rows, patch_height, cols, patch_width) + .permute(1, 3, 0, 2, 4) + ) + packed[offset : offset + count].view( + rows, cols, in_channels, patch_height, patch_width + ).copy_(source) + offset += count + + normalized = input_norm(packed, weight.dtype) + projected = F.linear(normalized, weight.reshape(out_channels, -1)) + embeddings = [ + tokens.transpose(0, 1).reshape(1, out_channels, rows, cols) + for tokens, (rows, cols) in zip(projected.split(patch_counts), grid_sizes) + ] + return projected.unsqueeze(0), embeddings + + class PixtralHFVisionModel(nn.Module): def __init__( self, config: PixtralVisionConfig, quant_config: QuantizationConfig | None = None, *, + input_norm: nn.Module | None = None, num_hidden_layers_override: int | None = None, require_post_norm: bool | None = None, prefix: str = "", @@ -1440,6 +1484,7 @@ class PixtralHFVisionModel(nn.Module): super().__init__() self.config = config + self.input_norm = input_norm self.patch_conv = Conv2dLayer( in_channels=config.num_channels, @@ -1495,16 +1540,24 @@ class PixtralHFVisionModel(nn.Module): all tokens of all images of shape (N_toks, D) """ - # pass images through initial convolution independently - patch_embeds_list = [ - self.patch_conv(img.unsqueeze(0).to(self.dtype)) for img in pixel_values - ] + images = [image.to(device=self.device) for image in pixel_values] + if self.input_norm is not None: + patch_embeds, patch_embeds_list = pixtral_patch_embed( + images, + self.patch_conv.weight, + self.input_norm, + ) + else: + patch_embeds_list = [] + for image in images: + patch_embeds_list.append( + self.patch_conv(image.unsqueeze(0).to(self.dtype)) + ) + patch_embeds = torch.cat( + [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list], dim=1 + ) - patch_embeds = [p.flatten(2).permute(0, 2, 1) for p in patch_embeds_list] - embed_sizes = [p.shape[1] for p in patch_embeds] - - # flatten to a single sequence - patch_embeds = torch.cat(patch_embeds, dim=1) + embed_sizes = [p.shape[-2] * p.shape[-1] for p in patch_embeds_list] patch_embeds = self.ln_pre(patch_embeds) # positional embeddings From 0ae480ff1a1b07a6959c3b19113739ffd2b60820 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 02:06:44 -0500 Subject: [PATCH 006/248] [CI][ROCm] Mirror large-model GSM8K evaluations on MI355 (#58683) Signed-off-by: Andreas Karatzas Co-authored-by: Codex --- .buildkite/test-amd.yaml | 165 +++++++++++++++++ .buildkite/test_areas/lm_eval.yaml | 170 ++++++++++++++++++ .../gsm8k/configs/DeepSeek-R1-DP_MI355.yaml | 17 ++ .../gsm8k/configs/DeepSeek-R1-TP_MI355.yaml | 17 ++ .../gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml | 17 ++ .../gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml | 17 ++ ...Nemotron-3-Super-120B-A12B-BF16_MI355.yaml | 15 ++ .../Nemotron-3-Super-120B-A12B-FP8_MI355.yaml | 15 ++ .../configs/models-mi355-deepseek-r1-dp.txt | 1 + .../configs/models-mi355-deepseek-r1-tp.txt | 1 + .../configs/models-mi355-deepseek-v32-dp.txt | 1 + .../configs/models-mi355-deepseek-v32-tp.txt | 1 + .../configs/models-mi355-nemotron-3-super.txt | 2 + tests/evals/gsm8k/test_gsm8k_correctness.py | 12 +- 14 files changed, 446 insertions(+), 5 deletions(-) create mode 100644 tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt create mode 100644 tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 16c1e7075b..e77093bb49 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -4248,6 +4248,171 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-large.txt +- label: ":amd: (MI355) LM Eval DeepSeek R1 TP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-tp.txt + +- label: ":amd: (MI355) LM Eval DeepSeek R1 DP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt + +- label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt + +- label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-dp.txt + +- label: ":amd: (MI355) LM Eval Nemotron 3 Super" + timeout_in_minutes: 65 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + optional: true + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/mamba/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-nemotron-3-super.txt + #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------# - label: ":amd: (MI355 DPX) Examples" # TBD diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 971e308d25..c81bba10da 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -436,6 +436,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-tp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek R1 TP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-tp.txt - label: ":nvidia: (H200) LM Eval DeepSeek R1 DP" key: lm-eval-deepseek-r1-dp-8xh200 @@ -445,6 +479,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-dp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek R1 DP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt - label: ":nvidia: (H200) LM Eval DeepSeek V3.2 TP" key: lm-eval-deepseek-v32-tp-8xh200 @@ -454,6 +522,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-tp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt - label: ":nvidia: (H200) LM Eval DeepSeek V3.2 DP" key: lm-eval-deepseek-v32-dp-8xh200 @@ -463,6 +565,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-dp.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/sparse_attn_indexer.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-dp.txt - label: ":nvidia: (H200) LM Eval Nemotron 3 Super" key: lm-eval-nemotron-3-super-8xh200 @@ -472,6 +608,40 @@ steps: num_devices: 8 commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-nemotron-3-super.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval Nemotron 3 Super" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 65 + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/ + - vllm/models/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/core/kv_cache_utils.py + - vllm/model_executor/layers/layernorm.py + - csrc/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/layers/mamba/ + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-nemotron-3-super.txt - label: ":amd: (MI300) LM Eval Large Models" key: amd-lm-eval-large-models diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml new file mode 100644 index 0000000000..a482c47bff --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-R1" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --data-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml new file mode 100644 index 0000000000..74776f46b6 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-R1" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml new file mode 100644 index 0000000000..afb4bed274 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V3.2-DP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-V3.2" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --data-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA_SPARSE + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml new file mode 100644 index 0000000000..5bc5fa9af0 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V3.2-TP_MI355.yaml @@ -0,0 +1,17 @@ +model_name: "deepseek-ai/DeepSeek-V3.2" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" + VLLM_ROCM_USE_AITER_MLA: "1" + VLLM_ROCM_USE_AITER_MOE: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --attention-backend ROCM_AITER_MLA_SPARSE + --moe-backend aiter + --speculative-config '{"method":"mtp","num_speculative_tokens":3}' diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml new file mode 100644 index 0000000000..bf67ca1f7d --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16_MI355.yaml @@ -0,0 +1,15 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --mamba-backend triton + --moe-backend triton + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml new file mode 100644 index 0000000000..9d76e8f3ab --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8_MI355.yaml @@ -0,0 +1,15 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +env: + VLLM_ROCM_USE_AITER: "1" +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 4 + --enable-expert-parallel + --mamba-backend triton + --moe-backend triton + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt new file mode 100644 index 0000000000..2481c05f7b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-dp.txt @@ -0,0 +1 @@ +DeepSeek-R1-DP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt new file mode 100644 index 0000000000..5c7753bc60 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-r1-tp.txt @@ -0,0 +1 @@ +DeepSeek-R1-TP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt new file mode 100644 index 0000000000..91d687c46b --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-dp.txt @@ -0,0 +1 @@ +DeepSeek-V3.2-DP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt new file mode 100644 index 0000000000..a21dfa4589 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-deepseek-v32-tp.txt @@ -0,0 +1 @@ +DeepSeek-V3.2-TP_MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt b/tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt new file mode 100644 index 0000000000..5cb9401036 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-mi355-nemotron-3-super.txt @@ -0,0 +1,2 @@ +Nemotron-3-Super-120B-A12B-BF16_MI355.yaml +Nemotron-3-Super-120B-A12B-FP8_MI355.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index 7411ca0f40..3b66de6ad0 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -109,15 +109,17 @@ def test_gsm8k_correctness(config_filename): "Its W2A16 (uint2b2) scheme has no kernel outside CUDA." ) - # TODO(akaratza): Enable DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms if current_platform.is_rocm() and ( "deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"] or "deepseek-ai/DeepSeek-R1" in eval_config["model_name"] ): - pytest.skip( - "Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms " - "due to agent pool disk space issues and pod evictions." - ) + from vllm.platforms.rocm import get_cdna_version + + if get_cdna_version() < 4: + pytest.skip( + "Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms below " + "CDNA 4 due to agent pool disk space issues and pod evictions." + ) if current_platform.is_rocm() and ( "Qwen3.5-35B-A3B-MXFP4-AITER-TP2" in config_filename.name ): From 69d5c81fcd52f29a3b498dd4c403b088c91e4ee1 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 02:08:31 -0500 Subject: [PATCH 007/248] [ROCm][CI] Mirror split kernel groups on MI355 and fix exposed tests (#58654) Signed-off-by: Andreas Karatzas Signed-off-by: Codex Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/test-amd.yaml | 102 ++++++++++---- .buildkite/test_areas/kernels.yaml | 129 ++++++++++++++---- .buildkite/test_areas/quantization.yaml | 2 + tests/kernels/test_bf16_skinny_gemm.py | 3 + tests/kernels/test_compressor_kv_cache.py | 8 +- .../test_kpool_decode_update_batched.py | 33 +++-- 6 files changed, 209 insertions(+), 68 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index e77093bb49..830b2166ea 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -4487,6 +4487,77 @@ steps: #---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------# +- label: ":amd: (MI355) mHC Kernels" + timeout_in_minutes: 50 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/ + - vllm/ + - tests/kernels/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s kernels/mhc + +- label: ":amd: (MI355) FLA Ops Kernels" + timeout_in_minutes: 50 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/ + - vllm/ + - tests/kernels/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s kernels/fla + +- label: ":amd: (MI355) Miscellaneous Kernels" + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - csrc/ + - vllm/ + - tests/kernels/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s + kernels/test_apply_repetition_penalties.py + kernels/test_awq_int4_to_int8.py + kernels/test_bf16_skinny_gemm.py + kernels/test_cache_kernels.py + kernels/test_compressor_kv_cache.py + kernels/test_cp_gather_fp8.py + kernels/test_deepseek_v4_cpu_kernels.py + kernels/test_dsv41_mega_attn_layouts.py + kernels/test_engram.py + kernels/test_flashinfer_bf16_gemm.py + kernels/test_flex_attention.py + kernels/test_fp32_router_gemm.py + kernels/test_fused_deepseek_v32_norm_rope.py + kernels/test_fused_indexer_q_rope_quant.py + kernels/test_fused_inv_rope_fp8_quant.py + kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py + kernels/test_fused_quant_activation.py + kernels/test_gate_linear_rocm_dispatch.py + kernels/test_gemm_rs_ar.py + kernels/test_kpool_decode_update_batched.py + kernels/test_minimax_m3_amd_ops.py + kernels/test_minimax_m3_sparse_attn_fp8_scale.py + kernels/test_onednn.py + kernels/test_relu2_fp8_quant.py + kernels/test_rocm_fp32_router_gemm.py + --deselect=tests/kernels/test_bf16_skinny_gemm.py::test_every_dsv3_routed_shape_is_instantiated + - label: ":amd: (MI355 DPX) Attention DiffKV Kernels" timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -5191,36 +5262,9 @@ steps: - unset VLLM_USE_V2_MODEL_RUNNER - uv pip install --system torchao==0.17.0 - uv pip install --system conch-triton-kernels - - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT + - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py --ignore quantization/test_rocm_moe.py --ignore quantization/test_gfx950_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s rocm/test_moe_weight_replay.py; fi -# - label: Quantized MoE Test (B200-MI355) # TBD -# timeout_in_minutes: 180 -# mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] -# dind: false -# agent_pool: mi355_1 -# working_dir: "/vllm-workspace/" -# source_file_dependencies: -# - tests/quantization/test_gfx950_moe.py -# - vllm/model_executor/models/deepseek_v2.py -# - vllm/model_executor/models/gpt_oss.py -# - vllm/model_executor/models/llama4.py -# - vllm/model_executor/layers/fused_moe -# - vllm/model_executor/layers/quantization/compressed_tensors -# - vllm/model_executor/layers/quantization/modelopt.py -# - vllm/model_executor/layers/quantization/mxfp4.py -# - vllm/v1/attention/backends/triton_attn.py -# - vllm/v1/attention/backends/rocm_attn.py -# - vllm/v1/attention/backends/rocm_aiter_fa.py -# - vllm/v1/attention/backends/mla/ -# - vllm/v1/attention/selector.py -# - vllm/model_executor/layers/layernorm.py -# - vllm/_aiter_ops.py -# - vllm/platforms/rocm.py -# - vllm/model_executor/model_loader/ -# commands: -# - pytest -s -v tests/quantization/test_gfx950_moe.py - #--------------------------------------------------------- mi355 · samplers ----------------------------------------------------------# - label: ":amd: (MI355 DPX) Samplers Multimodal Beam Search" @@ -6078,6 +6122,7 @@ steps: timeout_in_minutes: 45 working_dir: /vllm-workspace/ source_file_dependencies: + - tests/quantization/test_gfx950_moe.py - tests/quantization/test_rocm_moe.py - tests/utils.py - vllm/model_executor/models/deepseek_v2.py @@ -6094,6 +6139,7 @@ steps: - vllm/envs.py commands: - pytest -v -s tests/quantization/test_rocm_moe.py + - pytest -v -s tests/quantization/test_gfx950_moe.py dind: false agent_pool: mi355_1 num_gpus: 1 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index c939553f64..515c3966f5 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -184,6 +184,20 @@ steps: - tests/kernels/ commands: - pytest -v -s kernels/mhc + mirror: + amd: + label: ":amd: (MI355) mHC Kernels" + dind: false + device: mi355_1 + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/ + - tests/kernels/ + - vllm/platforms/rocm.py - label: ":nvidia: (B200) FLA Ops Kernels" key: kernels-fla-ops-test-b200 @@ -196,6 +210,20 @@ steps: - tests/kernels/ commands: - pytest -v -s kernels/fla + mirror: + amd: + label: ":amd: (MI355) FLA Ops Kernels" + dind: false + device: mi355_1 + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/ + - tests/kernels/ + - vllm/platforms/rocm.py # Catch-all for test files at the tests/kernels root. This job collects # the whole root so new files are wired by default. @@ -231,6 +259,50 @@ steps: --ignore=kernels/test_shuffle_rows.py # BROKEN on main, pending kernel fixes (B200): # test_shuffle_rows.py (1: test_shuffle_rows_edge_cases) + mirror: + amd: + label: ":amd: (MI355) Miscellaneous Kernels" + dind: false + device: mi355_1 + timeout_in_minutes: 55 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/ + - tests/kernels/ + - vllm/platforms/rocm.py + # Keep the ROCm exclusions from #50519: incompatible FlyDSL TurboQuant + # and a CUDA source check whose files are absent from the AMD test image. + commands: + - pytest -v -s + kernels/test_apply_repetition_penalties.py + kernels/test_awq_int4_to_int8.py + kernels/test_bf16_skinny_gemm.py + kernels/test_cache_kernels.py + kernels/test_compressor_kv_cache.py + kernels/test_cp_gather_fp8.py + kernels/test_deepseek_v4_cpu_kernels.py + kernels/test_dsv41_mega_attn_layouts.py + kernels/test_engram.py + kernels/test_flashinfer_bf16_gemm.py + kernels/test_flex_attention.py + kernels/test_fp32_router_gemm.py + kernels/test_fused_deepseek_v32_norm_rope.py + kernels/test_fused_indexer_q_rope_quant.py + kernels/test_fused_inv_rope_fp8_quant.py + kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py + kernels/test_fused_quant_activation.py + kernels/test_gate_linear_rocm_dispatch.py + kernels/test_gemm_rs_ar.py + kernels/test_kpool_decode_update_batched.py + kernels/test_minimax_m3_amd_ops.py + kernels/test_minimax_m3_sparse_attn_fp8_scale.py + kernels/test_onednn.py + kernels/test_relu2_fp8_quant.py + kernels/test_rocm_fp32_router_gemm.py + --deselect=tests/kernels/test_bf16_skinny_gemm.py::test_every_dsv3_routed_shape_is_instantiated - label: ":nvidia: (L4) Attention Kernels Shard %N" key: kernels-attention-test @@ -725,6 +797,35 @@ steps: - pytest -v -s kernels/moe/test_flashinfer_moe.py - pytest -v -s kernels/moe/test_nvfp4_moe.py - pytest -v -s kernels/moe/test_ocp_mx_moe.py + mirror: + amd: + label: ":amd: (MI355 DPX) FP4 and INT4 MoE Kernels" + dind: false + device: mi355_dpx + num_devices: 1 + timeout_in_minutes: 40 + optional: false + working_dir: /vllm-workspace/tests + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/moe/ + - csrc/rocm/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/envs.py + - vllm/config/ + - tests/kernels/moe/test_rocm_aiter_moe.py + - tests/kernels/moe/test_ocp_mx_moe.py + - tests/kernels/moe/test_flydsl_moe.py + - tests/kernels/moe/utils.py + - tests/kernels/utils.py + commands: + - pytest -v -s kernels/moe/test_rocm_aiter_moe.py -k 'mi350_mxfp4 or quant_scheme_support_matches_gfx950' + - pytest -v -s kernels/moe/test_ocp_mx_moe.py::test_rocm_mxfp4_moe_oracle -k AITER_MXFP4 + - pytest -v -s kernels/moe/test_flydsl_moe.py - label: ":nvidia: (H100) FusedMoE Layer Kernels" @@ -847,34 +948,6 @@ steps: - pytest -v -s kernels/quantization/test_rocm_mxfp8_linear.py - pytest -v -s kernels/quantization/test_rocm_skinny_gemms.py -k wvsplitkrc -- label: ":amd: (MI355 DPX) FP4 and INT4 MoE Kernels" - key: amd-fp4-moe-kernels-mi355 - dind: false - device: mi355_dpx - num_devices: 1 - timeout_in_minutes: 40 - working_dir: /vllm-workspace/tests - depends_on: - - image-build-amd - source_file_dependencies: - - csrc/moe/ - - csrc/rocm/ - - vllm/model_executor/layers/fused_moe/ - - vllm/model_executor/layers/quantization/ - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - - vllm/envs.py - - vllm/config/ - - tests/kernels/moe/test_rocm_aiter_moe.py - - tests/kernels/moe/test_ocp_mx_moe.py - - tests/kernels/moe/test_flydsl_moe.py - - tests/kernels/moe/utils.py - - tests/kernels/utils.py - commands: - - pytest -v -s kernels/moe/test_rocm_aiter_moe.py -k 'mi350_mxfp4 or quant_scheme_support_matches_gfx950' - - pytest -v -s kernels/moe/test_ocp_mx_moe.py::test_rocm_mxfp4_moe_oracle -k AITER_MXFP4 - - pytest -v -s kernels/moe/test_flydsl_moe.py - - label: ":amd: (MI355 DPX) FP8 MoE Kernels" key: amd-fp8-moe-kernels-mi355 dind: false diff --git a/.buildkite/test_areas/quantization.yaml b/.buildkite/test_areas/quantization.yaml index 122679553f..78ea92d232 100644 --- a/.buildkite/test_areas/quantization.yaml +++ b/.buildkite/test_areas/quantization.yaml @@ -102,6 +102,7 @@ steps: depends_on: - image-build-amd source_file_dependencies: + - tests/quantization/test_gfx950_moe.py - tests/quantization/test_rocm_moe.py - tests/utils.py - vllm/model_executor/models/deepseek_v2.py @@ -118,6 +119,7 @@ steps: - vllm/envs.py commands: - pytest -v -s tests/quantization/test_rocm_moe.py + - pytest -v -s tests/quantization/test_gfx950_moe.py - label: ":nvidia: (H200 MIG 35GB) Quantized Models" device: h200_35gb diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index c16b4d44e8..d452354e5f 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -269,6 +269,9 @@ def test_kda_overlap_configs_match_measured_table() -> None: } +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="KDA CuTeDSL helper requires CUDA" +) @pytest.mark.parametrize( "compute_capability,supported", [((9, 0), False), ((10, 0), True), ((10, 3), True), ((12, 0), True)], diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 278c0ef5d0..3e40a3882f 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -2182,7 +2182,9 @@ def test_v41_rocm_csa2_full_pipeline(): ) q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result - assert torch.equal(q_out, q) + assert torch.equal( + q_out, q.view(num_tokens, attention.n_local_heads, attention.head_dim) + ) assert torch.equal(kv_out, kv) assert torch.equal(index_q_out, index_q) assert torch.equal(index_q_scale_out, index_q_scale) @@ -2273,7 +2275,9 @@ def test_v41_rocm_csa2_reindex_pipeline(): ) q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result - assert torch.equal(q_out, q) + assert torch.equal( + q_out, q.view(num_tokens, attention.n_local_heads, attention.head_dim) + ) assert torch.equal(kv_out, kv) assert torch.equal(index_q_out, index_q) assert torch.equal(index_q_scale_out, index_q_scale) diff --git a/tests/kernels/test_kpool_decode_update_batched.py b/tests/kernels/test_kpool_decode_update_batched.py index b0fbcf7faf..1f5693bad9 100644 --- a/tests/kernels/test_kpool_decode_update_batched.py +++ b/tests/kernels/test_kpool_decode_update_batched.py @@ -233,6 +233,26 @@ def _assert_eq(r_ref, r_kern): ) +def _cache_pool_bytes(kv_cache: torch.Tensor, pool_slot: int) -> torch.Tensor: + """Read a logical pool's K and scale bytes from the platform cache layout.""" + page_size = kv_cache.shape[1] + head_dim = kv_cache.shape[2] - 4 + page_idx, token_offset = divmod(pool_slot, page_size) + flat = kv_cache[page_idx].reshape(-1) + dims = torch.arange(head_dim, device=kv_cache.device) + if current_platform.is_rocm() and page_size > 1: + k_offsets = ( + (token_offset // 16) * 16 * head_dim + + (dims // 16) * 16 * 16 + + (token_offset % 16) * 16 + + dims % 16 + ) + else: + k_offsets = token_offset * head_dim + dims + scale_offset = page_size * head_dim + 4 * token_offset + return torch.cat((flat[k_offsets], flat[scale_offset : scale_offset + 4])) + + @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required") def test_amd_prefill_writer_uses_preshuffled_cache_layout(): from vllm.models.glm5next.amd.ops.kpool_compress import ( @@ -343,7 +363,7 @@ def test_decode_writer_matches_prefill_writer(pool_size, ring_pools): p for p in range(n_pools) if not torch.equal( - kv_prefill[p // page, p % page], kv_decode[p // page, p % page] + _cache_pool_bytes(kv_prefill, p), _cache_pool_bytes(kv_decode, p) ) ] assert not differing, ( @@ -379,13 +399,6 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools): kv = torch.zeros_like(kv_ref) tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev) - def pool_bytes(cache, p): - # Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales]. - flat = cache[0].reshape(-1) - k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM] - s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)] - return torch.cat([k_bytes, s_bytes]) - def step(positions, keys, scores): pos = torch.tensor([positions], dtype=torch.int32, device=dev) slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions] @@ -415,7 +428,7 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools): ) step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected for p in (1, 2): - assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p + assert torch.equal(_cache_pool_bytes(kv, p), _cache_pool_bytes(kv_ref, p)), p # Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts # 8 and 9 overwrite the slots of positions 4 and 5, which are read by the @@ -430,7 +443,7 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools): torch.cat([score[6:7], draft_scores]), ) step([7, 8, 9, 10], k[7:11], score[7:11]) - pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1)) + pool1_ok = torch.equal(_cache_pool_bytes(kv, 1), _cache_pool_bytes(kv_ref, 1)) if ring_pools == 1: assert not pool1_ok, "expected a one-pool ring to corrupt pool 1" else: From 78fc1e6274d326c869432e4dfde8dc5b4c6e85a8 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 02:10:26 -0500 Subject: [PATCH 008/248] [ROCm][CI] Add MI355 TP2 AR-RMS and B200 fusion mirrors (#58284) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .buildkite/test-amd.yaml | 50 ++++++++++++++++ .buildkite/test_areas/compile.yaml | 52 ++++++++++++++++ tests/compile/fusions_e2e/test_tp2_ar_rms.py | 4 +- .../compile/fusions_e2e/test_tp2_rocm_moe.py | 60 +++++++++++++++++++ 4 files changed, 165 insertions(+), 1 deletion(-) create mode 100644 tests/compile/fusions_e2e/test_tp2_rocm_moe.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 830b2166ea..55f7f382c8 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3228,6 +3228,56 @@ steps: commands: - pytest -v -s compile/h100/test_startup.py +- label: ":amd: (MI355) Fusion E2E TP2 AR-RMS FP8 Config Sweep" + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/ + - vllm/distributed/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/compile/fusions_e2e/ + - tests/utils.py + - tests/conftest.py + - csrc/quantization/ + commands: + # Existing MI300 jobs own BF16; TP2 Quick owns the excluded Llama cell. + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_CUSTOM_AR=1 pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k 'test_tp2_ar_rms_fp8_fusions and Llama-3 and ROCM_AITER_UNIFIED_ATTN and (dynamo_partition or +quant_fp8 or +rms_norm)' + +- label: ":amd: (MI355) Fusion E2E TP2 FP8 BF16 rewrites" + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/ + - vllm/distributed/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/compile/fusions_e2e/ + - tests/utils.py + - tests/conftest.py + - csrc/quantization/ + commands: + # Dummy-weight rewrite coverage; trained FP8 AR/RMS equivalence is unresolved. + # TODO: FP4 output-quant and trained MXFP4 AR/RMS fusion remain unsupported. + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_CUSTOM_AR=1 pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k 'test_tp2_ar_rms_fp8_fusions and qwen3 and ROCM_AITER_UNIFIED_ATTN and +quant_fp8 and (dynamo_partition or not +rms_norm)' + - pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_moe.py + - label: ":amd: (MI355) Fusion and Compile FP8" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 221c2e8838..3819c2b3a3 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -267,6 +267,31 @@ steps: - nvidia-smi # Run just llama3 (fp8 & bf16) for all config combinations - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "llama-3" + mirror: + amd: + label: ":amd: (MI355) Fusion E2E TP2 AR-RMS FP8 Config Sweep" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/ + - vllm/distributed/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/compile/fusions_e2e/ + - tests/utils.py + - tests/conftest.py + - csrc/quantization/ + commands: + # Existing MI300 jobs own BF16; TP2 Quick owns the excluded Llama cell. + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_CUSTOM_AR=1 pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k 'test_tp2_ar_rms_fp8_fusions and Llama-3 and ROCM_AITER_UNIFIED_ATTN and (dynamo_partition or +quant_fp8 or +rms_norm)' - label: ":nvidia: (H100) Fusion E2E TP2 AsyncTP Config Sweep" key: fusion-e2e-tp2-asynctp-config-sweep-h100 @@ -334,3 +359,30 @@ steps: # for ar-rms-quant-fp4, also sweep llama3 - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k "(FLASHINFER and inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))) or Llama-3.1-8B-Instruct-FP4" - pytest -v -s tests/compile/fusions_e2e/test_tp2_async_tp.py -k "FLASHINFER and inductor_partition and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek))" + mirror: + amd: + label: ":amd: (MI355) Fusion E2E TP2 FP8 BF16 rewrites" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/ + - vllm/distributed/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/compile/fusions_e2e/ + - tests/utils.py + - tests/conftest.py + - csrc/quantization/ + commands: + # Dummy-weight rewrite coverage; trained FP8 AR/RMS equivalence is unresolved. + # TODO: FP4 output-quant and trained MXFP4 AR/RMS fusion remain unsupported. + - amd-smi + - VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_CUSTOM_AR=1 pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py -k 'test_tp2_ar_rms_fp8_fusions and qwen3 and ROCM_AITER_UNIFIED_ATTN and +quant_fp8 and (dynamo_partition or not +rms_norm)' + - pytest -v -s tests/compile/fusions_e2e/test_tp2_rocm_moe.py diff --git a/tests/compile/fusions_e2e/test_tp2_ar_rms.py b/tests/compile/fusions_e2e/test_tp2_ar_rms.py index f35bc1bd9c..b0a3d71888 100644 --- a/tests/compile/fusions_e2e/test_tp2_ar_rms.py +++ b/tests/compile/fusions_e2e/test_tp2_ar_rms.py @@ -111,7 +111,9 @@ def test_tp2_ar_rms_fp8_fusions( model_kwargs["max_model_len"] = 1024 model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False} model_kwargs["disable_custom_all_reduce"] = False - if current_platform.is_rocm() and "llama-3" in model_name_lower: + if current_platform.is_rocm() and ( + "llama-3" in model_name_lower or "qwen" in model_name_lower + ): model_kwargs.update( gpu_memory_utilization=0.01, kv_cache_memory_bytes=64 * 1024**2, diff --git a/tests/compile/fusions_e2e/test_tp2_rocm_moe.py b/tests/compile/fusions_e2e/test_tp2_rocm_moe.py new file mode 100644 index 0000000000..9a228ca76c --- /dev/null +++ b/tests/compile/fusions_e2e/test_tp2_rocm_moe.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm MoE coverage for the TP2 GEMM/collective fusion contract.""" + +import pytest + +from tests.utils import multi_gpu_test +from vllm.config import PassConfig +from vllm.platforms import current_platform + +from .common import INDUCTOR_GRAPH_PARTITION +from .models import TRITON_ATTN, qwen3_a3b + + +@multi_gpu_test(num_gpus=2) +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm AsyncTP contract") +@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION[:1]) +@pytest.mark.parametrize("attn_backend", [TRITON_ATTN]) +def test_tp2_rocm_moe_async_tp_bf16_fusions( + inductor_graph_partition, attn_backend, run_e2e_fusion_test +): + """MoE dense projections retain the Qwen SP/AsyncTP rewrite counts.""" + n_layers = 4 + run_e2e_fusion_test( + qwen3_a3b.model_name, + qwen3_a3b.matches(n_layers), + dict( + hf_overrides=dict( + num_hidden_layers=n_layers, + hidden_size=512, + intermediate_size=1024, + moe_intermediate_size=128, + num_attention_heads=8, + num_key_value_heads=2, + head_dim=64, + num_experts=8, + num_experts_per_tok=2, + ), + load_format="dummy", + max_model_len=1024, + kv_cache_memory_bytes=64 * 1024**2, + gpu_memory_utilization=0.01, + max_num_batched_tokens=1024, + max_num_seqs=32, + ), + attn_backend, + dict( + use_inductor_graph_partition=inductor_graph_partition, + custom_ops=["-rms_norm"], + pass_config=PassConfig( + enable_qk_norm_rope_fusion=True, + enable_sp=True, + fuse_gemm_comms=True, + fuse_allreduce_rms=False, + sp_min_token_num=512, + ), + ), + ["norm_rope_fusion", "sequence_parallel", "async_tp"], + tp_size=2, + ) From 6cbbea872141851421dcad5fca009aa6cec93dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20=C4=8Cadek?= Date: Tue, 29 Sep 2026 09:11:56 +0200 Subject: [PATCH 009/248] [Bugfix] Hoist $defs/definitions in Cohere parser tool schema composition (#49602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Václav Čadek Co-authored-by: Cursor Agent --- tests/parser/cohere/test_structural_tags.py | 96 +++++++++++++++++++++ vllm/parser/cohere_command.py | 29 +++---- 2 files changed, 109 insertions(+), 16 deletions(-) diff --git a/tests/parser/cohere/test_structural_tags.py b/tests/parser/cohere/test_structural_tags.py index 27440d52af..5083215d3b 100644 --- a/tests/parser/cohere/test_structural_tags.py +++ b/tests/parser/cohere/test_structural_tags.py @@ -10,6 +10,7 @@ from types import SimpleNamespace import pytest from pydantic import ValidationError +from xgrammar.testing import _is_grammar_accept_string from vllm.entrypoints.generate.base.protocol import ( JsonSchemaResponseFormat, @@ -20,9 +21,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.cohere_command import ( CohereCommandParser, + CohereNormalizedTool, _has_effective_tools, _response_format_type, _schema_dict_from_structured_outputs, + collect_tool_schema, convert_schema_to_structural_tags, ) from vllm.sampling_params import StructuredOutputsParams @@ -41,6 +44,48 @@ GET_WEATHER_TOOL = { }, }, } +NESTED_DEFS_PARAMETERS = { + "$defs": { + "Location": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "country": {"type": "string"}, + }, + "required": ["city"], + } + }, + "type": "object", + "properties": {"location": {"$ref": "#/$defs/Location"}}, + "required": ["location"], +} +NESTED_DEFS_TOOL = { + "type": "function", + "function": {"name": "get_weather", "parameters": NESTED_DEFS_PARAMETERS}, +} +RECURSIVE_DEFS_PARAMETERS = { + "$defs": { + "Finding": { + "type": "object", + "properties": { + "note": {"type": "string"}, + "related": {"type": "array", "items": {"$ref": "#/$defs/Finding"}}, + }, + } + }, + "type": "object", + "properties": {"finding": {"$ref": "#/$defs/Finding"}}, +} +DRAFT07_DEFINITIONS_PARAMETERS = { + "definitions": { + "Location": { + "type": "object", + "properties": {"city": {"type": "string"}}, + } + }, + "type": "object", + "properties": {"location": {"$ref": "#/definitions/Location"}}, +} VALID_STRUCTURAL_TAG = { "type": "structural_tag", "format": { @@ -360,3 +405,54 @@ class TestAdjustRequestTools: types = _content_types(o.structured_outputs.structural_tag) assert "grammar" in types assert "json_schema" in types + + +class TestCollectToolSchemaDefs: + def test_nested_defs_tool_adjust_request(self, parser) -> None: + o = parser.adjust_request( + _make_chat_request(tools=[NESTED_DEFS_TOOL], tool_choice="auto"), + ) + assert "grammar" in _content_types(o.structured_outputs.structural_tag) + + @staticmethod + def _tool_calls_json(name: str, parameters: dict) -> str: + return json.dumps( + [{"tool_call_id": "0", "tool_name": name, "parameters": parameters}] + ) + + def test_recursive_defs(self) -> None: + grammar = collect_tool_schema( + [CohereNormalizedTool(name="report", parameters=RECURSIVE_DEFS_PARAMETERS)] + ) + nested = {"note": "a", "related": [{"note": "b", "related": []}]} + assert _is_grammar_accept_string( + grammar, self._tool_calls_json("report", {"finding": nested}) + ) + bad = {"note": 1, "related": []} + assert not _is_grammar_accept_string( + grammar, self._tool_calls_json("report", {"finding": bad}) + ) + + def test_draft07_definitions(self) -> None: + grammar = collect_tool_schema( + [ + CohereNormalizedTool( + name="get_weather", parameters=DRAFT07_DEFINITIONS_PARAMETERS + ) + ] + ) + assert _is_grammar_accept_string( + grammar, + self._tool_calls_json("get_weather", {"location": {"city": "Prague"}}), + ) + assert not _is_grammar_accept_string( + grammar, + self._tool_calls_json("get_weather", {"location": {"city": 1}}), + ) + + def test_input_parameters_not_mutated(self) -> None: + params = json.loads(json.dumps(NESTED_DEFS_PARAMETERS)) + collect_tool_schema( + [CohereNormalizedTool(name="get_weather", parameters=params)] + ) + assert params == NESTED_DEFS_PARAMETERS diff --git a/vllm/parser/cohere_command.py b/vllm/parser/cohere_command.py index c37cc569ac..a1df491401 100644 --- a/vllm/parser/cohere_command.py +++ b/vllm/parser/cohere_command.py @@ -127,22 +127,19 @@ def collect_tool_schema(tool_schema: list[CohereNormalizedTool]) -> str: tool_dictionary: dict[str, str] = {} for tool in tool_schema: tool_name = tool["name"] - tool_parameters = json.dumps(tool["parameters"]) - json_schema = f"""{{ - "type": "object", - "properties": {{ - "tool_call_id": {{ - "type": "string", - "pattern": "^[0-9]+$" - }}, - "tool_name": {{ - "type": "string", - "const": "{tool_name}" - }}, - "parameters": {tool_parameters} - }} - }}""" - tool_grammar = str(xgr.Grammar.from_json_schema(json_schema)) + tool_parameters = dict(tool["parameters"]) + json_schema: dict[str, Any] = { + "type": "object", + "properties": { + "tool_call_id": {"type": "string", "pattern": "^[0-9]+$"}, + "tool_name": {"type": "string", "const": tool_name}, + "parameters": tool_parameters, + }, + } + for defs_key in ("$defs", "definitions"): + if defs_key in tool_parameters: + json_schema[defs_key] = tool_parameters.pop(defs_key) + tool_grammar = str(xgr.Grammar.from_json_schema(json.dumps(json_schema))) for match in re.findall(r"\b(\w+)\s*::=", tool_grammar): tool_grammar = re.sub( rf"\b{re.escape(match)}\b", tool_name + match, tool_grammar From 30f5c01d4ebb5c493ed86195db24d0ec5b3ce3d7 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Tue, 29 Sep 2026 00:31:30 -0700 Subject: [PATCH 010/248] [DSv4.1] Avoid runtime recompiles of _ring_slot_mapping_kernel (#59119) Signed-off-by: Juntian Liu Co-authored-by: Claude Opus 5.5 (1M context) --- vllm/models/deepseek_v41/compressor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/models/deepseek_v41/compressor.py b/vllm/models/deepseek_v41/compressor.py index 42335ae84b..54dc9c4a13 100644 --- a/vllm/models/deepseek_v41/compressor.py +++ b/vllm/models/deepseek_v41/compressor.py @@ -57,7 +57,7 @@ class CompressorMetadata: token_to_req_indices: torch.Tensor # [num_tokens] -@triton.jit +@triton.jit(do_not_specialize=["block_table_stride", "num_actual_tokens", "num_tokens"]) def _ring_slot_mapping_kernel( slot_mapping_ptr, block_table_ptr, From 77d9cda688a21087539a495562134a6f2f205efe Mon Sep 17 00:00:00 2001 From: Jin Tao Date: Tue, 29 Sep 2026 10:57:23 +0300 Subject: [PATCH 011/248] [ROCm][Model][Bugfix] Fix GLM-5.2 shared-expert fusion and MTP on the ROCm DSA path (#58904) Signed-off-by: Jin Tao Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Shanshan Shen <467638484@qq.com> --- vllm/models/deepseek_v32/amd/model.py | 133 +++++++++++++++++++------- vllm/models/deepseek_v32/amd/mtp.py | 9 ++ 2 files changed, 106 insertions(+), 36 deletions(-) diff --git a/vllm/models/deepseek_v32/amd/model.py b/vllm/models/deepseek_v32/amd/model.py index f5067bd0b9..d2325372d9 100644 --- a/vllm/models/deepseek_v32/amd/model.py +++ b/vllm/models/deepseek_v32/amd/model.py @@ -12,6 +12,9 @@ from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.utils import ( + is_model_fused_shared_expert_compatible, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, @@ -158,6 +161,12 @@ class DeepseekV32Model(torch.nn.Module): prefix=f"{prefix}.layers", ) + self.is_fused_shared_expert_enabled = is_model_fused_shared_expert_compatible( + self.layers, + DeepseekV2MoE, + "mlp", + ) + if get_pp_group().is_last_rank: self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) else: @@ -226,7 +235,12 @@ class DeepseekV32Model(torch.nn.Module): ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts, + num_experts=self.config.n_routed_experts + + ( + self.config.n_shared_experts + if self.is_fused_shared_expert_enabled + else 0 + ), num_redundant_experts=self.num_redundant_experts, ) @@ -239,6 +253,9 @@ class DeepseekV32Model(torch.nn.Module): continue if get_spec_layer_idx_from_weight_name(self.config, name) is not None: continue + is_fusion_moe_shared_experts_layer = ( + self.is_fused_shared_expert_enabled and ("mlp.shared_experts" in name) + ) if _try_load_fp8_indexer_wk( name, loaded_weight, @@ -254,6 +271,8 @@ class DeepseekV32Model(torch.nn.Module): continue if ("mlp.experts." in name) and name not in params_dict: continue + if is_fusion_moe_shared_experts_layer: + continue name_mapped = name.replace(weight_name, param_name) if ( param_name == "fused_qkv_a_proj" @@ -269,43 +288,85 @@ class DeepseekV32Model(torch.nn.Module): break else: is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] - if weight_name not in name: - continue - is_expert_weight = True - name_mapped = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name_mapped, self): - continue - param = params_dict[name_mapped] - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader + + # When AITER fusion_shared_experts is enabled the checkpoint + # provides a single widened shared_experts tensor per layer with + # no expert index. Split it evenly and route the slices into the + # expert slots appended after the routed experts. + num_chunks = 1 + if is_fusion_moe_shared_experts_layer: + num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 + # gate/up are ColumnParallel (dim 0), down is RowParallel (dim 1). + split_dim = ( + 1 + if ("down_proj.weight" in name and loaded_weight.ndim > 1) + else 0 ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, + total = loaded_weight.shape[split_dim] + assert total % num_chunks == 0, ( + f"Shared expert weight dim {total} " + f"not divisible by num_chunks {num_chunks}" ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - continue - if name.endswith(".bias") and name not in params_dict: - continue - name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - loader = getattr(param, "weight_loader", default_weight_loader) - loader(param, loaded_weight) - loaded_params.add(name) + chunk_size = total // num_chunks + + for j in range(num_chunks): + chunk_name = name + weight_to_load = loaded_weight + + if is_fusion_moe_shared_experts_layer: + chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) + if loaded_weight.ndim == 1: + weight_to_load = loaded_weight[chunk_slice] + elif split_dim == 0: + weight_to_load = loaded_weight[chunk_slice, :] + else: + weight_to_load = loaded_weight[:, chunk_slice] + chunk_name = name.replace( + "mlp.shared_experts", + f"mlp.experts.{self.config.n_routed_experts + j}", + ) + + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping # type: ignore[assignment] + if weight_name not in chunk_name: + continue + is_expert_weight = True + name_mapped = chunk_name.replace(weight_name, param_name) + if is_pp_missing_parameter(name_mapped, self): + continue + param = params_dict[name_mapped] + weight_loader = typing.cast( + Callable[..., bool], param.weight_loader + ) + success = weight_loader( + param, + weight_to_load, + name_mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + if success: + if not is_fusion_moe_shared_experts_layer: + name = name_mapped + else: + loaded_params.add(name_mapped) + break + else: + if is_expert_weight: + continue + if name.endswith(".bias") and name not in params_dict: + continue + name = maybe_remap_kv_scale_name(name, params_dict) # type: ignore[assignment] + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + loader = getattr(param, "weight_loader", default_weight_loader) + loader(param, loaded_weight) + if name is not None and not is_fusion_moe_shared_experts_layer: + loaded_params.add(name) return loaded_params diff --git a/vllm/models/deepseek_v32/amd/mtp.py b/vllm/models/deepseek_v32/amd/mtp.py index 46c35dec98..70ddeae374 100644 --- a/vllm/models/deepseek_v32/amd/mtp.py +++ b/vllm/models/deepseek_v32/amd/mtp.py @@ -130,6 +130,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module): if self_attn is not None and hasattr(self_attn, "skip_topk"): self_attn.skip_topk = skip + def compact_topk_indices(self, slot_ids: torch.Tensor): + """Gather the top-k index rows at ``slot_ids`` to the front of the buffer.""" + num_slots = slot_ids.numel() + for layer in self.layers.values(): + self_attn = getattr(layer.mtp_block, "self_attn", None) + if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"): + topk_indices_buffer = self_attn.topk_indices_buffer + topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids] + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) From 9af952c55568f596c1b76c68ced8c5aeb5593a39 Mon Sep 17 00:00:00 2001 From: Yashasvi Asthana Date: Tue, 29 Sep 2026 13:31:52 +0530 Subject: [PATCH 012/248] [Bugfix][Rust Frontend] Support raise_exception in chat templates (#59020) Co-authored-by: Claude Co-authored-by: Bugen Zhao Signed-off-by: Yashasvi Asthana Signed-off-by: Bugen Zhao --- rust/src/chat/src/error.rs | 3 + rust/src/chat/src/renderer/hf/mod.rs | 24 +++++- rust/src/chat/src/renderer/hf/template.rs | 92 ++++++++++++++++++++++- rust/src/server/src/error.rs | 21 ++++++ 4 files changed, 138 insertions(+), 2 deletions(-) diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index 7d124aaaf0..eee5a683df 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -17,6 +17,8 @@ pub enum Error { MissingChatTemplate, #[error("chat template error: {0}")] ChatTemplate(String), + #[error("{message}")] + ChatTemplateThrown { message: String }, #[error("{0}")] InvalidReasoningEffort(String), #[error("{message}")] @@ -112,6 +114,7 @@ impl Error { pub fn is_request_validation_error(&self) -> bool { match self { Self::PromptTooLong { .. } + | Self::ChatTemplateThrown { .. } | Self::InvalidReasoningEffort(_) | Self::InvalidReasoningControl { .. } | Self::DuplicateToolName { .. } diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index f406e32da6..5cb0262191 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -226,7 +226,12 @@ impl HfChatRenderer { template_kwargs: Some(&effective_template_kwargs), special_tokens: self.special_tokens.as_ref(), }) - .map_err(|error| Error::ChatTemplate(error.to_report_string()))?; + .map_err(|error| match error.thrown_message() { + Some(message) => Error::ChatTemplateThrown { + message: message.to_owned(), + }, + None => Error::ChatTemplate(error.to_report_string()), + })?; let prompt = match &final_message_text { Some(final_message_text) => { @@ -973,6 +978,23 @@ mod tests { assert!(matches!(error, Error::ChatTemplate(_))); } + #[test] + fn raise_exception_in_template_is_a_request_validation_error() { + let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hi")]); + + let error = render( + Some("{{- raise_exception('No user query found in messages.') }}"), + &request, + ) + .unwrap_err(); + + assert!( + matches!(&error, Error::ChatTemplateThrown { message } if message == "No user query found in messages."), + "expected an exception thrown by the template, got: {error:?}" + ); + assert!(error.is_request_validation_error()); + } + #[test] fn chat_template_flattens_text_parts_for_string_templates() { let request = sample_request(vec![ChatMessage::user(vec![ diff --git a/rust/src/chat/src/renderer/hf/template.rs b/rust/src/chat/src/renderer/hf/template.rs index de447e2c34..efcfd420ed 100644 --- a/rust/src/chat/src/renderer/hf/template.rs +++ b/rust/src/chat/src/renderer/hf/template.rs @@ -8,12 +8,15 @@ //! - special tokens are wired to `vllm_text::backends::hf::HfSpecialTokens` use std::collections::HashMap; +use std::error::Error as _; use std::fs; +use std::iter; use std::path::Path; -use minijinja::Environment; +use minijinja::{Environment, Error as MinijinjaError, ErrorKind, Value}; use serde::{Deserialize, Serialize}; use serde_json::{self}; +use thiserror::Error as ThisError; use vllm_text::backend::hf::HfSpecialTokens; use super::error::TemplateError; @@ -38,10 +41,44 @@ fn build_environment(template: String) -> Result> { env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback); env.add_filter("tojson", hf_tojson_filter); env.add_function("__hf_generation", render_generation); + env.add_function("raise_exception", raise_exception); Ok(env) } +/// Error source attached by [`raise_exception`], so that an exception thrown +/// by the template can be told apart from a template that fails to render. +#[derive(Debug, ThisError)] +#[error("{message}")] +struct ThrownTemplateException { + message: String, +} + +/// Throw an exception with the template's own message, aborting rendering. +fn raise_exception(message: Value) -> std::result::Result { + let thrown = ThrownTemplateException { + message: message.to_string(), + }; + let error = MinijinjaError::new( + ErrorKind::InvalidOperation, + "chat template threw an exception", + ); + Err(error.with_source(thrown)) +} + +impl TemplateError { + /// Return the template's own message if rendering was aborted by + /// [`raise_exception`]. + pub(super) fn thrown_message(&self) -> Option<&str> { + let Self::Jinja(error) = self else { + return None; + }; + iter::successors(error.source(), |&error| error.source()) + .find_map(|error| error.downcast_ref::()) + .map(|thrown| thrown.message.as_str()) + } +} + #[serde_with::skip_serializing_none] #[derive(Default, Serialize)] pub(super) struct TemplateContext<'a> { @@ -189,6 +226,59 @@ mod tests { ); } + #[test] + fn test_raise_exception_returns_template_message() { + let template = CompiledChatTemplate::new( + "{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '.') }}" + .to_string(), + ChatTemplateContentFormatOption::Auto, + ) + .unwrap(); + let mut kwargs = HashMap::new(); + kwargs.insert("reasoning_effort".to_string(), serde_json::json!("high")); + + let error = template + .apply(TemplateContext { + template_kwargs: Some(&kwargs), + ..Default::default() + }) + .unwrap_err(); + + assert_eq!( + error.thrown_message(), + Some("Unexpected reasoning effort high."), + "expected an exception thrown by the template, got: {error:?}" + ); + } + + #[test] + fn test_raise_exception_in_untaken_branch_renders() { + let template = CompiledChatTemplate::new( + "{% if messages %}{{ raise_exception('unreachable') }}{% endif %}ok".to_string(), + ChatTemplateContentFormatOption::Auto, + ) + .unwrap(); + + assert_eq!(template.apply(TemplateContext::default()).unwrap(), "ok"); + } + + #[test] + fn test_unknown_function_is_not_reported_as_thrown() { + let template = CompiledChatTemplate::new( + "{{ undefined_helper('not registered') }}".to_string(), + ChatTemplateContentFormatOption::Auto, + ) + .unwrap(); + + let error = template.apply(TemplateContext::default()).unwrap_err(); + + assert_eq!( + error.thrown_message(), + None, + "expected a jinja render error, got: {error:?}" + ); + } + #[test] fn test_special_tokens_injected_into_context() { let template = "{{ bos_token }}hello{{ eos_token }}"; diff --git a/rust/src/server/src/error.rs b/rust/src/server/src/error.rs index 3280cd1dd5..5f7adc8fbb 100644 --- a/rust/src/server/src/error.rs +++ b/rust/src/server/src/error.rs @@ -193,6 +193,27 @@ mod tests { } } + #[test] + fn chat_template_throw_maps_to_invalid_request() { + let error = vllm_chat::Error::ChatTemplateThrown { + message: "Unexpected reasoning effort high.".to_string(), + }; + let api_error = chat_submit_error("failed to submit chat request", error); + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST); + let response = api_error.to_error_response(); + assert_eq!(response.error.error_type, "invalid_request_error"); + assert_eq!(response.error.message, "Unexpected reasoning effort high."); + } + + #[test] + fn chat_template_render_failure_stays_internal() { + let error = vllm_chat::Error::ChatTemplate( + "failed to render jinja template: unknown function".to_string(), + ); + let api_error = chat_submit_error("failed to submit chat request", error); + assert_eq!(api_error.status_code(), StatusCode::INTERNAL_SERVER_ERROR); + } + #[test] fn llm_wrapped_empty_prompt_maps_to_invalid_request() { let error = vllm_text::Error::Llm(vllm_llm::Error::EmptyPromptTokenIds { From 4b2e1cfa7b8094f1a9547aee41c39e05d2ce5c97 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Tue, 29 Sep 2026 01:25:46 -0700 Subject: [PATCH 013/248] [Model] Decoder-side SWA bounded replay for DeepSeek-V4.1 (#58132) Signed-off-by: Yifan Qiao Co-authored-by: Claude Fable 5.1 --- .buildkite/test_areas/models_basic.yaml | 4 +- tests/distributed/test_engram_dp_shard.py | 31 +- ...test_deepseek_v41_decoder_replay_layers.py | 104 ++++++ .../models/test_deepseek_v41_replay_batch.py | 259 ++++++++++++++ vllm/config/cache.py | 4 +- .../deepseek_v41/decoder_replay_layers.py | 67 ++++ vllm/models/deepseek_v41/nvidia/model.py | 262 +++++++++++--- .../models/deepseek_v41/nvidia/model_state.py | 323 +++++++++++++++++- vllm/models/deepseek_v41/nvidia/vl_model.py | 5 + 9 files changed, 1000 insertions(+), 59 deletions(-) create mode 100644 tests/models/test_deepseek_v41_decoder_replay_layers.py create mode 100644 tests/models/test_deepseek_v41_replay_batch.py create mode 100644 vllm/models/deepseek_v41/decoder_replay_layers.py diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 62805e90f6..54130d0d02 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -66,8 +66,10 @@ steps: - tests/models/transformers/test_backend.py - tests/models/test_registry.py - tests/models/test_deepseek_v4_vl_rocm.py + - tests/models/test_deepseek_v41_decoder_replay_layers.py + - tests/models/test_deepseek_v41_replay_batch.py commands: - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py mirror: amd: label: ":amd: (MI355 DPX) Basic Models (Other)" diff --git a/tests/distributed/test_engram_dp_shard.py b/tests/distributed/test_engram_dp_shard.py index ba03387aac..84dd62851e 100644 --- a/tests/distributed/test_engram_dp_shard.py +++ b/tests/distributed/test_engram_dp_shard.py @@ -507,20 +507,22 @@ def _check_dummy_hash_model_forward( assert torch.all(hashes == engram_ops.DEAD_ID) output = engram.embed(hashes[:, 0]) # Trailing None is previous_aux; this stub captures no aux states. - return output, None, None, None, None, None + return output, hidden, hidden, hidden, hidden, None - model = SimpleNamespace( - use_mega_moe=False, - use_sequence_parallel=False, - fuse_mhc_all_reduce=False, - engram_hash=state, - engram_swa_prefix="swa", - engram_dp_shared_memory=dp_shared_memory, - layers=[Decoder(engram=engram)], - start_layer=0, - end_layer=1, - aux_hidden_state_layers=(), - ) + model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model) + torch.nn.Module.__init__(model) + model.use_mega_moe = False + model.use_sequence_parallel = False + model.fuse_mhc_all_reduce = False + model.engram_hash = state + model.engram_swa_prefix = "swa" + model.engram_dp_shared_memory = dp_shared_memory + model.layers = [Decoder(engram=engram)] + model.start_layer = 0 + model.end_layer = 1 + model.decoder_replay_start = model.end_layer + model.decoder_replay_layers = None + model.aux_hidden_state_layers = () metadata = ( None if dp_rank == 1 @@ -550,8 +552,7 @@ def _check_dummy_hash_model_forward( dtype=torch.int32, ), ): - output = model_ops.DeepseekV4Model.forward( - model, + output = model( torch.arange(tokens, device="cuda"), torch.arange(tokens, device="cuda"), intermediate_tensors=None, diff --git a/tests/models/test_deepseek_v41_decoder_replay_layers.py b/tests/models/test_deepseek_v41_decoder_replay_layers.py new file mode 100644 index 0000000000..4af797a1e6 --- /dev/null +++ b/tests/models/test_deepseek_v41_decoder_replay_layers.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DecoderReplayLayers runs the replay layers on the forward's replay batch.""" + +import pytest +import torch + +from vllm.forward_context import ( + ForwardContext, + get_forward_context, + override_forward_context, +) +from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +DEVICE = torch.device("cuda") +NUM_TOKENS = 401 +HC, HIDDEN = 2, 3 +# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300) +WINDOW = 128 +REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)] + + +def _context(metadata): + return ForwardContext( + no_compile_layers={}, + attn_metadata=metadata, + slot_mapping={}, + is_padding=torch.zeros(NUM_TOKENS, dtype=torch.bool, device=DEVICE), + ) + + +def _set_replay_batch(layers, rows, metadata): + layers.rows = torch.tensor(rows, device=DEVICE) + layers.forward_context = ForwardContext( + no_compile_layers={}, attn_metadata=metadata, slot_mapping={} + ) + + +def _states(num_tokens: int): + """(hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual), + shaped as the model's: hidden states [T, H], mixes [T, HC], residual [T, HC, H].""" + hidden = torch.arange(num_tokens, device=DEVICE, dtype=torch.float32) + hidden = hidden[:, None].expand(num_tokens, HIDDEN).contiguous() + mix = hidden[:, :HC].contiguous() + residual = hidden[:, None, :].expand(num_tokens, HC, HIDDEN).contiguous() + return ( + hidden, + torch.arange(num_tokens, device=DEVICE), + None, + mix + 1, + mix + 2, + mix + 3, + residual, + ) + + +def test_run_gathers_states_and_realigns_shared_indexer_buffers(): + topk = torch.arange(NUM_TOKENS * 4, device=DEVICE).view(NUM_TOKENS, 4).int() + candidates = torch.arange(NUM_TOKENS * 3, device=DEVICE).view(NUM_TOKENS, 3).int() + topk_before, candidates_before = topk.clone(), candidates.clone() + seen = {} + + def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual): + seen["hidden_states"] = hidden_states.clone() + replay_context = get_forward_context() + seen["attn_metadata"] = replay_context.attn_metadata + seen["is_padding"] = replay_context.is_padding + return residual, pre_mix + + layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates]) + states = _states(NUM_TOKENS) + hidden, residual = states[0], states[-1] + full, sub = object(), object() + _set_replay_batch(layers, REPLAY_ROWS, sub) + context = _context(full) + with override_forward_context(context): + outputs = layers(*states) + assert get_forward_context() is context + + rows = torch.tensor(REPLAY_ROWS, device=DEVICE) + assert torch.equal(seen["hidden_states"], hidden[rows]) + assert seen["attn_metadata"] is sub and seen["is_padding"] is None + assert torch.equal(topk[: len(REPLAY_ROWS)], topk_before[rows]) + assert torch.equal(candidates[: len(REPLAY_ROWS)], candidates_before[rows]) + assert len(outputs) == 2 and outputs[0].shape == residual.shape + assert torch.equal(outputs[0][rows], residual[rows]) + assert outputs[0][1:173].abs().sum() == 0 + + +def test_no_replay_batch_runs_the_whole_batch(): + seen = {} + + def run_layers(hidden_states, *rest): + seen["attn_metadata"] = get_forward_context().attn_metadata + return (hidden_states,) + + layers = DecoderReplayLayers(WINDOW, run_layers, []) + states = _states(NUM_TOKENS) + full = object() + with override_forward_context(_context(full)): + (output,) = layers(*states) + assert output is states[0] and seen["attn_metadata"] is full diff --git a/tests/models/test_deepseek_v41_replay_batch.py b/tests/models/test_deepseek_v41_replay_batch.py new file mode 100644 index 0000000000..f6efe5c516 --- /dev/null +++ b/tests/models/test_deepseek_v41_replay_batch.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The decoder-side SWA bounded replay batch DeepseekV41ModelState prepares.""" + +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +import torch + +from vllm.config import CUDAGraphMode +from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers +from vllm.v1.worker.gpu.model_states.default import DefaultModelState + +pytestmark = pytest.mark.skipif( + not torch.cuda.is_available(), reason="the replay buffers live on the GPU" +) + +WINDOW = 128 +DEVICE = torch.device("cuda") +GROUPS = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace( + layer_names=["swa"], + kv_cache_spec=SimpleNamespace( + prefix_cacheable=False, prefix_replay_tokens=WINDOW + ), + ), + SimpleNamespace( + layer_names=["mla"], + kv_cache_spec=SimpleNamespace( + prefix_cacheable=True, prefix_replay_tokens=0 + ), + ), + ] +) +BLOCK_TABLES = (torch.zeros(4, 4, device=DEVICE),) * 2 + +# decode (1 token), trimmed prefill (300 of 300), untrimmed prefill (100 of 300) +QUERY_LENS = [1, 300, 100] +SEQ_LENS = [500, 300, 300] +PREFILLING = [False, True, True] +REPLAY_ROWS = [0, *range(301 - WINDOW, 301), *range(301, 401)] + + +@pytest.fixture +def state(monkeypatch): + """A DeepseekV41ModelState whose attention builds are recorded, not run.""" + cfg = MagicMock() + cfg.model_config.enable_prompt_embeds = False + cfg.model_config.uses_mrope = False + cfg.model_config.is_multimodal_model = False + cfg.scheduler_config.max_num_seqs = 16 + cfg.scheduler_config.max_num_batched_tokens = 1024 + cfg.parallel_config.data_parallel_size = 1 + cfg.compilation_config.fast_moe_cold_start = False + layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None) + model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers) + builds: list = [] + + def prepare_attn(self, input_batch, cg_mode, block_tables, slot_mappings, *a, **kw): + build = SimpleNamespace( + batch=input_batch, + cg_mode=cg_mode, + slot_mappings=slot_mappings.clone(), + replay_start=kw["model_specific_attn_metadata"].replay_start, + attn_metadata=object(), + ) + builds.append(build) + return {"swa": build.attn_metadata} + + monkeypatch.setattr(DefaultModelState, "prepare_attn", prepare_attn) + state = DeepseekV41ModelState(cfg, model, None, DEVICE) + state.builds = builds # type: ignore[attr-defined] + return state + + +def _input_batch( + query_lens: list[int], + seq_lens: list[int], + is_prefilling: list[bool], + device_query_lens: list[int] | None = None, + num_tokens_after_padding: int | None = None, + max_query_len: int | None = None, +) -> InputBatch: + num_reqs, num_tokens = len(query_lens), sum(query_lens) + num_padded = num_tokens_after_padding or num_tokens + batch = InputBatch.make_dummy(num_reqs, num_tokens, InputBuffers(16, 1024, DEVICE)) + return replace( + batch, + num_tokens=num_tokens, + num_tokens_after_padding=num_padded, + query_start_loc=torch.tensor( + [0, *np.cumsum(device_query_lens or query_lens)], + dtype=torch.int32, + device=DEVICE, + ), + query_start_loc_np=np.array([0, *np.cumsum(query_lens)], dtype=np.int32), + seq_lens=torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE), + seq_lens_cpu_upper_bound=torch.tensor(seq_lens, dtype=torch.int32), + is_prefilling_np=np.array(is_prefilling, dtype=np.bool_), + positions=torch.cat( + [torch.arange(s - q, s) for q, s in zip(query_lens, seq_lens)] + + [torch.zeros(num_padded - num_tokens, dtype=torch.int64)] + ).to(DEVICE), + is_padding=torch.arange(num_padded, device=DEVICE) >= num_tokens, + max_query_len=max_query_len, + ) + + +def _slot_mappings(num_tokens: int) -> torch.Tensor: + return torch.stack([torch.arange(num_tokens), torch.arange(num_tokens) * 10]).to( + DEVICE + ) + + +def _prepare(state, batch, cg_mode): + """Runs prepare_attn; returns the replay batch and the replay build's inputs.""" + state.prepare_attn( + batch, + cg_mode, + BLOCK_TABLES, + _slot_mappings(batch.num_tokens_after_padding), + [], + GROUPS, + ) + layers = state.decoder_replay_layers + if layers.rows is None: + assert layers.forward_context is None + return None, None + return SimpleNamespace( + rows=layers.rows, forward_context=layers.forward_context + ), state.builds[-1] + + +def test_replay_batch_keeps_each_request_window(state): + state._replay_start_np[2] = 50 # the encoder-side replay start of request 2 + batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING) + replay, build = _prepare(state, batch, CUDAGraphMode.NONE) + assert replay is not None + rows = torch.tensor(REPLAY_ROWS, device=DEVICE) + assert torch.equal(replay.rows, rows) + sub = build.batch + assert build.cg_mode == CUDAGraphMode.NONE + assert sub.query_start_loc.tolist() == [0, 1, 129, 229] + assert sub.query_start_loc_np.tolist() == [0, 1, 129, 229] + assert sub.num_tokens == sub.num_tokens_after_padding == 229 + assert sub.max_query_len == WINDOW + # The trimmed request's window starts at its replay window; a higher + # encoder-side replay start stands. + assert build.replay_start.tolist() == [0, 300 - WINDOW, 50] + assert torch.equal(sub.positions, batch.positions[rows]) + assert torch.equal(build.slot_mappings, _slot_mappings(401)[:, rows]) + context = replay.forward_context + assert context.attn_metadata == {"swa": build.attn_metadata} + assert torch.equal(context.slot_mapping["mla"], _slot_mappings(401)[1, rows]) + assert context.dp_metadata is None and context.is_padding is None + + +def test_replay_batch_keeps_device_decode_boundaries(state): + """Adaptive verification resizes decodes on the GPU (CPU [2, 2], device + [1, 3]): their rows stay and the boundaries shift only by trimmed rows.""" + batch = _input_batch( + [2, 2, 300], [10, 10, 300], PREFILLING, device_query_lens=[1, 3, 300] + ) + replay, build = _prepare(state, batch, CUDAGraphMode.NONE) + assert replay is not None + assert replay.rows.tolist() == [0, 1, 2, 3, *range(304 - WINDOW, 304)] + assert build.batch.query_start_loc.tolist() == [0, 1, 4, 4 + WINDOW] + + +def test_prompt_logprobs_requests_keep_their_rows(state): + """Every prompt row of a prompt-logprobs request is read, so it never + trims; the other prefills still do.""" + state._keeps_rows_np[1] = True + batch = _input_batch([1, 300, 300], [500, 300, 300], PREFILLING) + replay, build = _prepare(state, batch, CUDAGraphMode.NONE) + assert replay is not None + assert replay.rows.tolist() == [0, *range(1, 301), *range(601 - WINDOW, 601)] + assert build.batch.max_query_len == 300 + + +def test_replay_batch_keeps_adaptive_verification_query_bound(state): + """Adaptive verification bounds the decodes' device-side query lengths + above their CPU lengths; the replay batch keeps that bound.""" + batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, max_query_len=200) + replay, build = _prepare(state, batch, CUDAGraphMode.NONE) + assert replay is not None and build.batch.max_query_len == 200 + + +def test_only_eager_steps_trim(state): + """A CUDA graph keeps the layers on its whole batch; an eager step trims a + padded batch to its real rows.""" + batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512) + for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL): + assert _prepare(state, batch, cg_mode) == (None, None) + replay, build = _prepare(state, batch, CUDAGraphMode.NONE) + assert replay is not None + assert replay.rows.tolist() == REPLAY_ROWS + assert build.cg_mode == CUDAGraphMode.NONE + assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS) + + +def test_whole_batch_forwards_get_no_replay_batch(state): + short = _input_batch([100, 100], [100, 100], [True, True]) + for cg_mode in (CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL): + assert _prepare(state, short, cg_mode) == (None, None) + assert len(state.builds) == 3 # the batch's own metadata only + # Dummy batches, captures included (prepared as eager), are not prefills and + # keep their rows. + dummy = _input_batch( + QUERY_LENS, SEQ_LENS, [False] * 3, num_tokens_after_padding=512 + ) + assert _prepare(state, dummy, CUDAGraphMode.NONE) == (None, None) + + +@pytest.fixture +def dp_state(state, monkeypatch): + """`state` on DP rank 0 of 2; `state.other` sets what rank 1 reports + (trims, replay tokens).""" + import vllm.models.deepseek_v41.nvidia.model_state as module + + state.vllm_config.parallel_config.data_parallel_size = 2 + state.vllm_config.parallel_config.data_parallel_rank = 0 + state.vllm_config.parallel_config.is_moe_model = True + state.other = (False, 0) # type: ignore[attr-defined] + + def all_reduce(tensor, group): + tensor[:, 1] = torch.tensor(state.other, dtype=torch.int32) + + monkeypatch.setattr(module.dist, "all_reduce", all_reduce) + monkeypatch.setattr(module, "get_dp_group", lambda: SimpleNamespace(cpu_group=None)) + return state + + +def test_dp_ranks_replay_together(dp_state): + """A trimming rank makes the others replay too, on every rank's count.""" + short = _input_batch([100, 100], [100, 100], [True, True]) + dp_state.other = (False, 200) + assert _prepare(dp_state, short, CUDAGraphMode.NONE) == (None, None) + + dp_state.other = (True, 300) # rank 1 trims to 300 rows + replay, build = _prepare(dp_state, short, CUDAGraphMode.NONE) + assert replay is not None + assert replay.rows.tolist() == list(range(200)) # this rank keeps every row + dp_metadata = replay.forward_context.dp_metadata + assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [200, 300] + + +def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state): + dummy = _input_batch([512], [512], [False]) + dp_state.other = (True, 129) + replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE) + assert replay is not None and replay.rows.shape[0] == WINDOW + dp_metadata = replay.forward_context.dp_metadata + assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129] diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 3121f49532..eeff06933e 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -242,7 +242,9 @@ class CacheConfig: swa_bounded_replay: bool = True """Keep the sliding-window KV of models that support it (DeepSeek-V4.1) out of prefix caching and rebuild it after a prefix hit by recomputing the - hit's last window. Requires model runner V2.""" + hit's last window. The layers past the last KV-source layer then also run + eager prefill steps on each request's trailing window only. Requires model + runner V2.""" kv_cache_memory_bytes: int | None = None """Size of KV Cache per GPU in bytes. By default, this is set to None diff --git a/vllm/models/deepseek_v41/decoder_replay_layers.py b/vllm/models/deepseek_v41/decoder_replay_layers.py new file mode 100644 index 0000000000..869029aa74 --- /dev/null +++ b/vllm/models/deepseek_v41/decoder_replay_layers.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Decoder-side SWA bounded replay: running the replay layers on their batch. + +The layers past the last KV-source layer own nothing but sliding-window KV, so +in eager prefill steps they run on each request's last ``window`` rows only. +``DeepseekV41ModelState`` prepares those rows as a sub-batch with attention +metadata and a forward context of its own, like a microbatch; +``DecoderReplayLayers`` gathers the layer inputs by its rows, runs the layers +under that context and scatters the outputs back to batch rows. Steps that run +in a CUDA graph keep the layers on the whole batch, inside the graph. +""" + +from collections.abc import Callable + +import torch + +from vllm.forward_context import ForwardContext, override_forward_context +from vllm.model_executor.layers.fused_moe.moe_output import MoEOutput + + +class DecoderReplayLayers: + """Runs the replay layers on the step's replay batch. + + ``run_layers`` takes a batch's layer inputs and returns its per-row + outputs. ``row_buffers`` hold per-row results the source layer's indexer + publishes for the layers after it; they are compacted to the replay rows + in place. + """ + + def __init__( + self, + window: int, + run_layers: Callable[..., tuple[torch.Tensor, ...]], + row_buffers: list[torch.Tensor], + ) -> None: + self.window = window + self.run_layers = run_layers + self.row_buffers = row_buffers + # The replay batch, set by the model state every step: its rows of the + # batch and its forward context. None runs the layers on the batch. + self.rows: torch.Tensor | None = None + self.forward_context: ForwardContext | None = None + + def __call__( + self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None + ) -> tuple[torch.Tensor, ...]: + rows = self.rows + if rows is None: + return self.run_layers(hidden_states, *states) + # A trimming step holds a prefill longer than the window, more tokens + # than any step whose MoE leaves its finalize to the next layer. + assert isinstance(hidden_states, torch.Tensor) + num_rows = rows.shape[0] + for buf in self.row_buffers: + buf[:num_rows].copy_(buf.index_select(0, rows)) + with override_forward_context(self.forward_context): + row_outputs = self.run_layers( + hidden_states.index_select(0, rows), + *(None if t is None else t.index_select(0, rows) for t in states), + ) + # The trimmed rows' outputs stay zero; nothing reads them. + num_tokens = hidden_states.shape[0] + return tuple( + out.new_zeros((num_tokens, *out.shape[1:])).index_copy_(0, rows, out) + for out in row_outputs + ) diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index 9e7a35f0fc..f7647ec81d 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -83,6 +83,7 @@ from vllm.models.deepseek_v4.nvidia.model import ( prepare_mega_gate_routing_metadata, ) from vllm.models.deepseek_v41.attention import DeepseekV4Attention +from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers from vllm.models.deepseek_v41.nvidia.flash_mla_mega_attn import ( DeepseekV4MegaAttnAttention, ) @@ -741,6 +742,35 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): if successor.engram is None: layer.ffn.defer_finalize() + # Decoder-side SWA bounded replay: in eager prefill steps the layers past + # the last KV source run on each request's trailing window only + # (decoder_replay_layers.py). + self.decoder_replay_layers: DecoderReplayLayers | None = None + self.decoder_replay_start = self.end_layer + cut = max(config.kv_source_layer_ids) + if ( + cut < self.end_layer - 1 + and self._decoder_replay_supported(vllm_config, cut) + and self.layers[cut].attn.swa_cache_layer.bounded_replay + ): + self.decoder_replay_start = cut + 1 + self.decoder_replay_layers = DecoderReplayLayers( + config.sliding_window, + self._run_replay_layers, + [ + buf + for buf in (self.topk_indices_buffer, self.candidate_block_buffer) + if buf is not None + ], + ) + logger.info_once( + "Decoder SWA bounded replay: in eager prefill steps, layers " + "%d-%d run on each request's last %d tokens only.", + cut + 1, + self.end_layer - 1, + config.sliding_window, + ) + # The n-gram hash needs a slot-keyed rolling store of compressed ids # (chunked prefill / decode lookback); key it off the first local # layer's sliding-window KV cache. Only PP ranks owning an engram @@ -904,29 +934,28 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): hidden_states = sp_shard(hidden_states) input_ids = sp_shard(input_ids) - mega_gate_metadata = None - if self.use_mega_moe: - mega_gate_metadata = prepare_mega_gate_routing_metadata( - input_ids, - has_hash_routing=False, - image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID - if getattr(self.config, "vision_n_layers", 0) > 0 - else None, - ) - residual, post_mix, res_mix = None, None, None pre_mix: torch.Tensor | None = None if not get_pp_group().is_first_rank: assert intermediate_tensors is not None pre_mix = intermediate_tensors["pre_mix"] - # Every layer's post runs inside the next layer's fused pre, so aux - # hidden states are read back from there instead of recomputed. aux_hidden_by_layer: dict[int, torch.Tensor] = {} - for idx, layer in enumerate( - islice(self.layers, self.start_layer, self.end_layer), - start=self.start_layer, - ): - hidden_states, residual, post_mix, res_mix, pre_mix, previous_aux = layer( + hidden_states, residual, post_mix, res_mix, pre_mix = self._run_layers( + range(self.start_layer, self.decoder_replay_start), + hidden_states, + positions, + input_ids, + pre_mix, + post_mix, + res_mix, + residual, + aux_hidden_by_layer, + engram_hashes, + engram_mask, + ) + late_aux: list[torch.Tensor] = [] + if self.decoder_replay_layers is not None: + hidden_states, pre_mix, *late_aux = self.decoder_replay_layers( hidden_states, positions, input_ids, @@ -934,36 +963,22 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): post_mix, res_mix, residual, - engram_hashes, - engram_mask, - capture_previous_aux=idx in self.aux_hidden_state_layers, - mega_gate_metadata=mega_gate_metadata, ) - if previous_aux is not None: - # idx is the one-based id of the layer whose post this is. - if self.use_sequence_parallel: - previous_aux = sp_all_gather(previous_aux)[:full_num_tokens] - aux_hidden_by_layer[idx] = previous_aux - # Without a successor boundary, the last layer finalized its own MoE. - assert isinstance(hidden_states, torch.Tensor) - if layer is not None: - # The last layer has no successor to fold its post into. - if self.fuse_mhc_all_reduce: - hidden_states = tensor_model_parallel_all_reduce(hidden_states) - hidden_states = mhc_post_tilelang( - hidden_states, residual, post_mix, res_mix + else: + hidden_states = self._collapse( + hidden_states, + residual, + post_mix, + res_mix, + aux_hidden_by_layer, + full_num_tokens, ) - if self.end_layer in self.aux_hidden_state_layers: - final_aux = hidden_states.mean(dim=1) - if self.use_sequence_parallel: - final_aux = sp_all_gather(final_aux)[:full_num_tokens] - aux_hidden_by_layer[self.end_layer] = final_aux aux_hidden_states = [ aux_hidden_by_layer[layer_id] for layer_id in self.aux_hidden_state_layers if layer_id in aux_hidden_by_layer - ] + ] + late_aux if not get_pp_group().is_last_rank: return IntermediateTensors( @@ -992,6 +1007,169 @@ class DeepseekV4Model(nn.Module, EagleModelMixin): return hidden_states, aux_hidden_states return hidden_states + def _mega_gate_metadata( + self, input_ids: torch.Tensor | None + ) -> MegaGateRoutingMetadata | None: + if not self.use_mega_moe: + return None + assert input_ids is not None + return prepare_mega_gate_routing_metadata( + input_ids, + has_hash_routing=False, + image_sentinel_base_id=IMAGE_SENTINEL_BASE_ID + if getattr(self.config, "vision_n_layers", 0) > 0 + else None, + ) + + def _run_layers( + self, + layer_ids: range, + hidden_states: torch.Tensor | MoEOutput, + positions: torch.Tensor, + input_ids: torch.Tensor | None, + pre_mix: torch.Tensor | None, + post_mix: torch.Tensor | None, + res_mix: torch.Tensor | None, + residual: torch.Tensor | None, + aux_hidden_by_layer: dict[int, torch.Tensor], + engram_hashes: torch.Tensor | None = None, + engram_mask: torch.Tensor | None = None, + ) -> tuple[ + torch.Tensor | MoEOutput, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor + ]: + # Every layer's post runs inside the next layer's fused pre, so aux + # hidden states are read back from there instead of recomputed. + full_num_tokens = positions.shape[0] + mega_gate_metadata = self._mega_gate_metadata(input_ids) + for idx in layer_ids: + hidden_states, residual, post_mix, res_mix, pre_mix, previous_aux = ( + self.layers[idx]( + hidden_states, + positions, + input_ids, + pre_mix, + post_mix, + res_mix, + residual, + engram_hashes, + engram_mask, + capture_previous_aux=idx in self.aux_hidden_state_layers, + mega_gate_metadata=mega_gate_metadata, + ) + ) + if previous_aux is not None: + # idx is the one-based id of the layer whose post this is. + if self.use_sequence_parallel: + previous_aux = sp_all_gather(previous_aux)[:full_num_tokens] + aux_hidden_by_layer[idx] = previous_aux + assert residual is not None and post_mix is not None + assert res_mix is not None and pre_mix is not None + return hidden_states, residual, post_mix, res_mix, pre_mix + + def _collapse( + self, + hidden_states: torch.Tensor | MoEOutput, + residual: torch.Tensor, + post_mix: torch.Tensor, + res_mix: torch.Tensor, + aux_hidden_by_layer: dict[int, torch.Tensor], + full_num_tokens: int, + ) -> torch.Tensor: + # Without a successor boundary, the last layer finalized its own MoE. + assert isinstance(hidden_states, torch.Tensor) + # The last layer has no successor to fold its post into. + if self.fuse_mhc_all_reduce: + hidden_states = tensor_model_parallel_all_reduce(hidden_states) + hidden_states = mhc_post_tilelang(hidden_states, residual, post_mix, res_mix) + if self.end_layer in self.aux_hidden_state_layers: + final_aux = hidden_states.mean(dim=1) + if self.use_sequence_parallel: + final_aux = sp_all_gather(final_aux)[:full_num_tokens] + aux_hidden_by_layer[self.end_layer] = final_aux + return hidden_states + + def _run_replay_layers( + self, + hidden_states: torch.Tensor | MoEOutput, + positions: torch.Tensor, + input_ids: torch.Tensor | None, + pre_mix: torch.Tensor, + post_mix: torch.Tensor, + res_mix: torch.Tensor, + residual: torch.Tensor, + ) -> tuple[torch.Tensor, ...]: + """The layers past the last KV source, on whatever rows they are given; + returns their output, the last FFN's pre-mix and the aux hidden states + they capture.""" + aux_hidden_by_layer: dict[int, torch.Tensor] = {} + hidden_states, residual, post_mix, res_mix, pre_mix = self._run_layers( + range(self.decoder_replay_start, self.end_layer), + hidden_states, + positions, + input_ids, + pre_mix, + post_mix, + res_mix, + residual, + aux_hidden_by_layer, + ) + hidden_states = self._collapse( + hidden_states, + residual, + post_mix, + res_mix, + aux_hidden_by_layer, + positions.shape[0], + ) + return ( + hidden_states, + pre_mix, + *( + aux_hidden_by_layer[layer_id] + for layer_id in self.aux_hidden_state_layers + if layer_id in aux_hidden_by_layer + ), + ) + + def _decoder_replay_supported(self, vllm_config: VllmConfig, cut: int) -> bool: + """Whether this rank may trim the layers after ``cut``; warns when not.""" + parallel_config = vllm_config.parallel_config + spec_config = vllm_config.speculative_config + draft_config = spec_config.draft_model_config if spec_config else None + draft_hf_config = getattr(draft_config, "hf_config", None) + draft_window = getattr(draft_hf_config, "sliding_window", None) + draft_layer_types = getattr(draft_hf_config, "layer_types", None) or () + window = self.config.sliding_window + if self.start_layer > cut or self.end_layer < self.config.num_hidden_layers: + reason = ( + "the pipeline stage holding the last KV source layer must also " + "hold every layer after it" + ) + elif ( + self.use_sequence_parallel + or parallel_config.prefill_context_parallel_size > 1 + or parallel_config.use_ubatching + ): + reason = ( + "the replay-layer batch shrinks per rank, which sequence and " + "prefill-context parallelism and microbatching cannot follow" + ) + elif any(i > cut for i in getattr(self.config, "engram_layer_ids", ())): + reason = "an Engram layer sits after the last KV source layer" + elif draft_config is not None and ( + draft_window is None + or draft_window > window + or any(t != "sliding_attention" for t in draft_layer_types) + ): + reason = ( + f"the drafter (sliding window {draft_window}) reads hidden states " + f"outside the target's {window}-token window" + ) + else: + return True + logger.warning_once("Decoder SWA bounded replay is off: %s.", reason) + return False + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: stacked_params_mapping = [ # (param_name, shard_name, shard_id) @@ -1397,6 +1575,10 @@ class DeepseekV41LLMForCausalLM( engram_hash = self.model.engram_hash return engram_hash.lookback_depth if engram_hash is not None else 0 + @property + def decoder_replay_layers(self) -> DecoderReplayLayers | None: + return self.model.decoder_replay_layers + def forward( self, input_ids: torch.Tensor, diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index af20976ded..30b287db22 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -1,17 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import replace from typing import Any import numpy as np import torch +import torch.distributed as dist import torch.nn as nn from vllm.config import CUDAGraphMode, VllmConfig +from vllm.distributed.parallel_state import get_dp_group +from vllm.forward_context import DPMetadata, create_forward_context +from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.worker.dp_utils import should_skip_dp_coordination +from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache @@ -81,6 +88,61 @@ def _pad_replayed_slots_kernel( ) +@triton.jit +def _gather_replay_batch_kernel( + query_start_loc_ptr, # [num_reqs + 1] + dropped_before_ptr, # [num_reqs + 1] rows trimmed before each boundary + seq_lens_ptr, # [num_reqs] + replay_start_ptr, # [num_reqs] the encoder-side replay start + positions_ptr, + slot_mappings_ptr, # [num_groups, num_tokens] + slot_mappings_stride, + rows_ptr, # out: [num_kept] batch row of every kept row + kept_query_start_loc_ptr, # out: [num_reqs + 1] + kept_positions_ptr, # out: [num_kept] + kept_slot_mappings_ptr, # out: [num_groups, num_kept] + kept_slot_mappings_stride, + kept_kv_start_ptr, # out: [num_reqs] + window, + NUM_GROUPS: tl.constexpr, + BLOCK: tl.constexpr, +): + """One program per request: its kept rows are the last ones before its + boundary, and land at that boundary minus the rows trimmed before it.""" + req = tl.program_id(0) + begin = tl.load(query_start_loc_ptr + req) + end = tl.load(query_start_loc_ptr + req + 1) + kept_begin = begin - tl.load(dropped_before_ptr + req) + kept_end = end - tl.load(dropped_before_ptr + req + 1) + if req == 0: + tl.store(kept_query_start_loc_ptr, kept_begin) + tl.store(kept_query_start_loc_ptr + req + 1, kept_end) + + # A trimmed request holds no replay-layer window KV below its kept rows. + kv_start = tl.load(replay_start_ptr + req) + if kept_end - kept_begin < end - begin: + kv_start = tl.maximum(kv_start, tl.load(seq_lens_ptr + req) - window) + tl.store(kept_kv_start_ptr + req, kv_start) + + first_row = end - (kept_end - kept_begin) + for tok in range(kept_begin, kept_end, BLOCK): + offs = tok + tl.arange(0, BLOCK) + mask = offs < kept_end + row = first_row + (offs - kept_begin) + tl.store(rows_ptr + offs, row.to(tl.int64), mask=mask) + tl.store( + kept_positions_ptr + offs, + tl.load(positions_ptr + row, mask=mask), + mask=mask, + ) + for g in tl.static_range(NUM_GROUPS): + tl.store( + kept_slot_mappings_ptr + g * kept_slot_mappings_stride + offs, + tl.load(slot_mappings_ptr + g * slot_mappings_stride + row, mask=mask), + mask=mask, + ) + + class ReplayAttnMetadata(ModelSpecificAttnMetadata): """Hands the batch's replay starts to the sliding-window builders.""" @@ -111,7 +173,10 @@ class DeepseekV41ModelState(DefaultModelState): ``prepare_attn`` gathers those per batch, pads the replayed tokens' slots in the prefix-cacheable groups so the cached KV stays as is, and hands the starts to the sliding-window metadata builders, whose kernels read no - window KV below them. + window KV below them. With the decoder side on + (``model.decoder_replay_layers``) it also prepares the replay layers' + batch in eager steps: each prefill's last ``window`` rows as a sub-batch + with metadata and a forward context of its own, like a microbatch. """ def __init__( @@ -140,9 +205,40 @@ class DeepseekV41ModelState(DefaultModelState): # the KV cache config on first use; None until then. self._replay: tuple[int, torch.Tensor] | None = None + self.decoder_replay_layers: DecoderReplayLayers | None = getattr( + model, "decoder_replay_layers", None + ) + if self.decoder_replay_layers is not None: + # Requests whose every prompt row is read (prompt logprobs) never trim. + self._keeps_rows_np = np.zeros(self.max_num_reqs, dtype=np.bool_) + self._dropped_before = UvaBufferPool(self.max_num_reqs + 1, torch.int32) + # The replay batch's inputs: its rows of the batch, query offsets, + # positions, slot mappings (sized with the KV cache groups on first + # use) and the position from which each request holds replay-layer + # window KV. + self._kept_rows = torch.zeros( + self.max_num_tokens, dtype=torch.int64, device=device + ) + self._kept_query_start_loc = torch.zeros( + self.max_num_reqs + 1, dtype=torch.int32, device=device + ) + self._kept_positions = torch.zeros( + self.max_num_tokens, dtype=torch.int64, device=device + ) + self._kept_slot_mappings: torch.Tensor | None = None + self._kept_kv_start = torch.zeros( + self.max_num_reqs, dtype=torch.int32, device=device + ) + self._replay_attn_groups: list[list[AttentionGroup]] | None = None + def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: super().add_request(req_index, new_req_data) self._replay_start_np[req_index] = new_req_data.replay_start + if self.decoder_replay_layers is not None: + params = new_req_data.sampling_params + self._keeps_rows_np[req_index] = ( + params is not None and params.prompt_logprobs is not None + ) def prepare_inputs( self, input_batch: InputBatch, req_states: RequestState @@ -196,7 +292,16 @@ class DeepseekV41ModelState(DefaultModelState): device=self.device, ), ) + if self.decoder_replay_layers is not None: + self._kept_slot_mappings = torch.zeros( + len(specs), + self.max_num_tokens, + dtype=torch.int64, + device=self.device, + ) + self._warm_up_replay_kernels(input_batch, slot_mappings) window, cacheable_groups = self._replay + replay_start: torch.Tensor | None = None if window: num_reqs = input_batch.num_reqs # Decode rows sit above the hit, so only prefills carry a replay @@ -226,7 +331,7 @@ class DeepseekV41ModelState(DefaultModelState): ) assert model_specific_attn_metadata is None model_specific_attn_metadata = ReplayAttnMetadata(replay_start) - return super().prepare_attn( + attn_metadata = super().prepare_attn( input_batch, cudagraph_mode, block_tables, @@ -237,3 +342,217 @@ class DeepseekV41ModelState(DefaultModelState): ubatch_idx=ubatch_idx, model_specific_attn_metadata=model_specific_attn_metadata, ) + if self.decoder_replay_layers is not None: + assert replay_start is not None + self._prepare_replay_batch( + input_batch, + cudagraph_mode, + block_tables, + slot_mappings, + attn_groups, + kv_cache_config, + replay_start, + ) + return attn_metadata + + def _warm_up_replay_kernels( + self, input_batch: InputBatch, slot_mappings: torch.Tensor + ) -> None: + """Compile the replay kernels on the buffers they run on: the startup + dummy batches never replay or trim, so they would not launch them.""" + assert self._replay is not None + window, cacheable_groups = self._replay + if not window: + return + _pad_replayed_slots_kernel.warmup( + slot_mappings, + slot_mappings.stride(0), + cacheable_groups, + input_batch.query_start_loc, + input_batch.positions, + self._replay_start, + window, + PAD_SLOT_ID, + NUM_GROUPS=cacheable_groups.numel(), + BLOCK=1024, + grid=(1,), + ) + if self.decoder_replay_layers is None: + return + assert self._kept_slot_mappings is not None + _gather_replay_batch_kernel.warmup( + input_batch.query_start_loc, + self._dropped_before.copy_to_uva(np.zeros(1, dtype=np.int32)), + input_batch.seq_lens, + self._replay_start, + input_batch.positions, + slot_mappings, + slot_mappings.stride(0), + self._kept_rows, + self._kept_query_start_loc, + self._kept_positions, + self._kept_slot_mappings, + self._kept_slot_mappings.stride(0), + self._kept_kv_start, + self.decoder_replay_layers.window, + NUM_GROUPS=slot_mappings.shape[0], + BLOCK=1024, + grid=(1,), + ) + + def _prepare_replay_batch( + self, + input_batch: InputBatch, + cudagraph_mode: CUDAGraphMode, + block_tables: tuple[torch.Tensor, ...], + slot_mappings: torch.Tensor, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + replay_start: torch.Tensor, + ) -> None: + """Set the replay layers' batch for this forward, or none when nothing + trims. Only eager steps trim: a CUDA graph keeps the layers on the whole + batch, and ranks share the graph mode. Under data parallelism every rank + replays if any does.""" + layers = self.decoder_replay_layers + assert layers is not None + layers.rows = layers.forward_context = None + if cudagraph_mode != CUDAGraphMode.NONE: + return + + num_reqs = input_batch.num_reqs + query_lens = np.diff(input_batch.query_start_loc_np[: num_reqs + 1]) + kept_lens = np.where( + self._keeps_rows_np[input_batch.idx_mapping_np[:num_reqs]], + query_lens, + np.minimum(query_lens, layers.window), + ) + # Dummy batches (captures, an idle DP rank's) are not prefills: they + # keep their rows unless a rank trims. + trims = bool( + (kept_lens < query_lens)[input_batch.is_prefilling_np[:num_reqs]].any() + ) + num_tokens = int(kept_lens.sum()) + dp_metadata = None + if self.vllm_config.parallel_config.data_parallel_size > 1: + trims, dp_metadata = self._agree_across_dp(trims, num_tokens) + if not trims: + return + + kept_batch, kept_slot_mappings = self._kept_input_batch( + input_batch, slot_mappings, replay_start, kept_lens + ) + attn_metadata = super().prepare_attn( + kept_batch, + CUDAGraphMode.NONE, + block_tables, + kept_slot_mappings, + self._replay_groups(attn_groups), + kv_cache_config, + model_specific_attn_metadata=ReplayAttnMetadata( + self._kept_kv_start[:num_reqs] + ), + ) + layers.rows = self._kept_rows[:num_tokens] + layers.forward_context = create_forward_context( + attn_metadata, + self.vllm_config, + dp_metadata=dp_metadata, + slot_mapping=build_slot_mappings_by_layer( + kept_slot_mappings, kv_cache_config + ), + ) + + def _kept_input_batch( + self, + input_batch: InputBatch, + slot_mappings: torch.Tensor, + replay_start: torch.Tensor, + kept_lens: np.ndarray, + ) -> tuple[InputBatch, torch.Tensor]: + """Build the sub-``InputBatch`` of each request's last ``kept_lens`` + rows, and its slot mappings. + + The kept rows follow the device boundaries minus the rows trimmed before + them: adaptive verification resizes the decodes on the GPU alone, and + decodes never trim. Like a microbatch's, the sub-batch describes the + forward only; its sampling fields are carried over and must not be read. + """ + num_reqs = input_batch.num_reqs + num_tokens = int(kept_lens.sum()) + kept_query_start_loc_np = np.zeros(num_reqs + 1, dtype=np.int32) + np.cumsum(kept_lens, out=kept_query_start_loc_np[1:]) + dropped_before = self._dropped_before.copy_to_uva( + input_batch.query_start_loc_np[: num_reqs + 1] - kept_query_start_loc_np + ) + assert self._kept_slot_mappings is not None + kept_slot_mappings = self._kept_slot_mappings[:, :num_tokens] + _gather_replay_batch_kernel[(num_reqs,)]( + input_batch.query_start_loc, + dropped_before, + input_batch.seq_lens, + replay_start, + input_batch.positions, + slot_mappings, + slot_mappings.stride(0), + self._kept_rows, + self._kept_query_start_loc, + self._kept_positions, + kept_slot_mappings, + kept_slot_mappings.stride(0), + self._kept_kv_start, + self.decoder_replay_layers.window, # type: ignore[union-attr] + NUM_GROUPS=slot_mappings.shape[0], + BLOCK=1024, + ) + # Adaptive verification may lengthen the decodes on the device, up to + # the batch's bound. + max_query_len = max(int(kept_lens.max()), input_batch.max_query_len or 0) + kept_batch = replace( + input_batch, + num_tokens=num_tokens, + num_tokens_after_padding=num_tokens, + query_start_loc=self._kept_query_start_loc[: num_reqs + 1], + query_start_loc_np=kept_query_start_loc_np, + max_query_len=max_query_len, + positions=self._kept_positions[:num_tokens], + fast_prefill=None, + ) + return kept_batch, kept_slot_mappings + + def _agree_across_dp( + self, trims: bool, num_tokens: int + ) -> tuple[bool, DPMetadata | None]: + """Whether any rank trims and, then, the replay layers' DP metadata from + every rank's replay token count.""" + parallel_config = self.vllm_config.parallel_config + agreed = torch.zeros(2, parallel_config.data_parallel_size, dtype=torch.int32) + agreed[:, parallel_config.data_parallel_rank] = torch.tensor( + [trims, num_tokens] + ) + if not should_skip_dp_coordination(): + dist.all_reduce(agreed, group=get_dp_group().cpu_group) + if not agreed[0].any(): + return False, None + return True, DPMetadata.make(parallel_config, num_tokens, agreed[1]) + + def _replay_groups( + self, attn_groups: list[list[AttentionGroup]] + ) -> list[list[AttentionGroup]]: + """The attention groups with metadata builders of their own, like each + microbatch's: a builder keeps the metadata it built, and the runner's + hold the batch's.""" + if self._replay_attn_groups is None: + self._replay_attn_groups = [] + for groups in attn_groups: + replay_groups = [] + for group in groups: + replay_group = replace(group, metadata_builders=[]) + replay_group.create_metadata_builders( + self.vllm_config, + self.device, + group.metadata_builders[0].kernel_block_size, + ) + replay_groups.append(replay_group) + self._replay_attn_groups.append(replay_groups) + return self._replay_attn_groups diff --git a/vllm/models/deepseek_v41/nvidia/vl_model.py b/vllm/models/deepseek_v41/nvidia/vl_model.py index 901985bdd7..d67753405a 100644 --- a/vllm/models/deepseek_v41/nvidia/vl_model.py +++ b/vllm/models/deepseek_v41/nvidia/vl_model.py @@ -51,6 +51,7 @@ from ..common.mm_preprocess import ( DeepseekV4VLProcessingInfo, ) from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin +from ..decoder_replay_layers import DecoderReplayLayers from .model import ( DeepseekV41LLMForCausalLM, _linear_scale_param_name, @@ -281,6 +282,10 @@ class DeepseekV41ForCausalLM( def token_lookback_depth(self) -> int: return self.language_model.token_lookback_depth + @property + def decoder_replay_layers(self) -> DecoderReplayLayers | None: + return self.language_model.decoder_replay_layers + def forward( self, input_ids: torch.Tensor, From 61349e342d5a79489e9642042ae03be17610427f Mon Sep 17 00:00:00 2001 From: Chauncey Date: Tue, 29 Sep 2026 16:31:22 +0800 Subject: [PATCH 014/248] [Misc] Avoid repeated warnings for merged Anthropic system messages (#59172) Signed-off-by: chaunceyjiang --- vllm/entrypoints/anthropic/serving.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 22484c744d..cd31908508 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -218,7 +218,7 @@ class AnthropicServingMessages(OpenAIServingChat): for tools in tool_variants ) if merge: - logger.warning( + logger.warning_once( "The chat template requires system-first ordering, so inline " "system messages in /v1/messages requests (e.g. Claude Code's " "per-turn reminders) are merged into the leading system prompt. " From 70dc122f81346032ebb658b3d00346f31ff1714d Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Tue, 29 Sep 2026 10:33:52 +0200 Subject: [PATCH 015/248] [MM] Enable device normalization for Llama Nemotron VL Embed/Rerank (#57928) Signed-off-by: Jakub Zakrzewski Co-authored-by: OpenAI Codex --- docs/design/mm_processing.md | 26 ++++----- .../pooling/test_llama_nemotron_vl.py | 54 +++++++++++++++++++ vllm/model_executor/models/nemotron_vl.py | 20 +++++++ .../processors/nemotron_vl.py | 52 +++++++++++++++--- 4 files changed, 132 insertions(+), 20 deletions(-) diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index f53f7d2dd1..3a37a2d8a4 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -78,18 +78,20 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: - The flag is **enabled by default** for all models that support it. - Currently, it’s on by default for these architectures: -| name | Architecture | Example HF Models | -|--------------------|------------------------------------------|-------------------------------------------------------| -| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | -| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | -| `mistral3` | `Mistral3ForConditionalGeneration` | `mistralai/Mistral-Small-3.2-24B-Instruct-2506`, etc. | -| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | -| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | -| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. | -| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | -| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. | -| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. | -| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` | +| name | Architecture | Example HF Models | +| -------------------------- | ------------------------------------------ | ----------------------------------------------------- | +| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | +| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | +| `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` | +| `llama-nemotron-vl-rerank` | `LlamaNemotronVLForSequenceClassification` | `nvidia/llama-nemotron-rerank-vl-1b-v2` | +| `mistral3` | `Mistral3ForConditionalGeneration` | `mistralai/Mistral-Small-3.2-24B-Instruct-2506`, etc. | +| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | +| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | +| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. | +| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | +| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. | +| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. | +| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` | #### Key Properties and Gains diff --git a/tests/models/multimodal/pooling/test_llama_nemotron_vl.py b/tests/models/multimodal/pooling/test_llama_nemotron_vl.py index 52802939ce..2a0ba96722 100644 --- a/tests/models/multimodal/pooling/test_llama_nemotron_vl.py +++ b/tests/models/multimodal/pooling/test_llama_nemotron_vl.py @@ -23,7 +23,13 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionContentPartTextParam, ) from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam +from vllm.model_executor.layers.fusion.mm_input_norm import FusedMMInputNorm from vllm.platforms import current_platform +from vllm.transformers_utils.processors.nemotron_vl import ( + SIGLIP_MEAN, + SIGLIP_STD, + LlamaNemotronVLEmbedImageProcessor, +) from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner from ....utils import ROCM_ENGINE_KWARGS @@ -52,6 +58,54 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts( MODELS = ["nvidia/llama-nemotron-embed-vl-1b-v2"] +@pytest.mark.usefixtures("default_vllm_config") +def test_device_normalization_matches_cpu_transform() -> None: + device = current_platform.device_type + image = Image.new("RGB", (48, 32), color=(12, 128, 240)) + processor_kwargs = dict( + image_size=32, + min_dynamic_patch=1, + max_dynamic_patch=1, + dynamic_image_size=True, + use_thumbnail=False, + ) + + cpu_processor = LlamaNemotronVLEmbedImageProcessor(**processor_kwargs) + device_processor = LlamaNemotronVLEmbedImageProcessor( + **processor_kwargs, do_rescale=False, do_normalize=False + ) + + expected = cpu_processor._images_to_pixel_values_lst([image])[0] + raw_pixels = device_processor._images_to_pixel_values_lst([image])[0] + input_norm = FusedMMInputNorm( + image_mean=list(SIGLIP_MEAN), + image_std=list(SIGLIP_STD), + rescale_factor=1 / 255, + ).to(device) + actual = input_norm( + raw_pixels.flatten(start_dim=1).to(device), expected.dtype + ).view_as(expected) + + assert raw_pixels.dtype == torch.uint8 + torch.testing.assert_close(actual, expected.to(device)) + + +@pytest.mark.parametrize(("do_rescale", "do_normalize"), [(True, False), (False, True)]) +def test_device_normalization_rejects_partial_cpu_processing( + do_rescale: bool, do_normalize: bool +) -> None: + with pytest.raises(ValueError, match="both enabled or both disabled"): + LlamaNemotronVLEmbedImageProcessor( + image_size=32, + min_dynamic_patch=1, + max_dynamic_patch=1, + dynamic_image_size=True, + use_thumbnail=False, + do_rescale=do_rescale, + do_normalize=do_normalize, + ) + + def _run_test( hf_runner: type[HfRunner], vllm_runner: type[VllmRunner], diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index 0ec4589e51..b5948048da 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -8,6 +8,10 @@ import torch.nn as nn from transformers import AutoModel, PreTrainedConfig from vllm.config import VllmConfig +from vllm.model_executor.layers.fusion.mm_input_norm import ( + FusedMMInputNorm, + IdentityInputNorm, +) from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.layers.quantization import QuantizationConfig @@ -26,6 +30,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_image_processor_from_config from vllm.transformers_utils.processors.nemotron_vl import ( + SIGLIP_MEAN, + SIGLIP_STD, LlamaNemotronNanoVLImageProcessor, LlamaNemotronNanoVLProcessor, LlamaNemotronVLEmbedImageProcessor, @@ -474,6 +480,7 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) """ is_pooling_model = True + supports_mm_device_do_normalize = True # Weight mapping from checkpoint format to vLLM format # Different from parent class due to different vision model structure @@ -498,6 +505,15 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) # Override: get img_context_token_id from config (parent sets None) self.img_context_token_id = getattr(config, "img_context_token_id", None) + if self.model_config.get_multimodal_config().mm_device_do_normalize: + self.input_norm = FusedMMInputNorm( + image_mean=list(SIGLIP_MEAN), + image_std=list(SIGLIP_STD), + rescale_factor=1 / 255, + ) + else: + self.input_norm = IdentityInputNorm() + # Initialize pooler for embedding output pooler_config = vllm_config.model_config.pooler_config assert pooler_config is not None @@ -537,6 +553,10 @@ class LlamaNemotronVLForEmbedding(LlamaNemotronVLChatModel, VllmModelForPooling) def _call_vision_model(self, pixel_values: torch.Tensor) -> torch.Tensor: """Override to handle SigLIP interface.""" + original_shape = pixel_values.shape + pixel_values = self.input_norm( + pixel_values.flatten(start_dim=1), self.vision_model.dtype + ).view(original_shape) return self.vision_model(pixel_values) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: diff --git a/vllm/transformers_utils/processors/nemotron_vl.py b/vllm/transformers_utils/processors/nemotron_vl.py index 9504f5ee2b..fefb05cff4 100644 --- a/vllm/transformers_utils/processors/nemotron_vl.py +++ b/vllm/transformers_utils/processors/nemotron_vl.py @@ -244,20 +244,53 @@ SIGLIP_MEAN = (0.5, 0.5, 0.5) SIGLIP_STD = (0.5, 0.5, 0.5) -def build_siglip_transform(input_size: int): +def build_siglip_transform( + input_size: int, + *, + normalize_on_cpu: bool = True, +): """Build transform for SigLIP vision encoder with normalization. Extends the base transform from nemotron_vl with SigLIP-specific normalization. """ - return T.Compose( - [ - build_transform(input_size=input_size), - T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD), - ] - ) + transforms = [ + T.Lambda(lambda img: convert_image_mode(img, "RGB")), + T.Resize((input_size, input_size), interpolation=T.InterpolationMode.BICUBIC), + T.ToTensor() if normalize_on_cpu else T.PILToTensor(), + ] + if normalize_on_cpu: + transforms.append(T.Normalize(mean=SIGLIP_MEAN, std=SIGLIP_STD)) + + return T.Compose(transforms) class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor): + def __init__( + self, + image_size: int, + min_dynamic_patch: int, + max_dynamic_patch: int, + dynamic_image_size: bool, + use_thumbnail: bool, + *, + do_rescale: bool = True, + do_normalize: bool = True, + ) -> None: + if do_rescale != do_normalize: + raise ValueError( + "LlamaNemotronVLEmbedImageProcessor requires do_rescale and " + "do_normalize to be both enabled or both disabled." + ) + + super().__init__( + image_size=image_size, + min_dynamic_patch=min_dynamic_patch, + max_dynamic_patch=max_dynamic_patch, + dynamic_image_size=dynamic_image_size, + use_thumbnail=use_thumbnail, + ) + self.normalize_on_cpu = do_normalize + def _images_to_pixel_values_lst( self, images: list[Image.Image], @@ -279,7 +312,10 @@ class LlamaNemotronVLEmbedImageProcessor(InternVLImageProcessor): min_num=min_num, max_num=max_num, use_thumbnail=self.use_thumbnail, - transform=build_siglip_transform(self.image_size), + transform=build_siglip_transform( + self.image_size, + normalize_on_cpu=self.normalize_on_cpu, + ), ) for image in images ] From 6ebb5bd64f5ef75dbe1471ee88c009003b3d03ec Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 03:40:07 -0500 Subject: [PATCH 016/248] [ROCm][CI] Expand MI355 mirrors and route MIG-sized jobs to DPX (#59137) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 262 ++++++++++++++---- .buildkite/test_areas/compile.yaml | 77 +++++ .buildkite/test_areas/distributed.yaml | 21 ++ .buildkite/test_areas/jit_monitor.yaml | 4 +- .buildkite/test_areas/lm_eval.yaml | 72 ++++- .buildkite/test_areas/spec_decode.yaml | 26 ++ .../compile/correctness_e2e/test_async_tp.py | 61 ++-- .../correctness_e2e/test_sequence_parallel.py | 15 +- tests/distributed/test_mnnvl_alltoall.py | 24 +- ...emotron-3-Super-120B-A12B-NVFP4-MI355.yaml | 18 ++ ...Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml | 17 ++ ...wen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml | 19 ++ .../configs/models-blackwell-ep-mi355.txt | 5 + .../gsm8k/configs/models-gfx950-small.txt | 3 + .../gsm8k/configs/models-qwen35-mi355.txt | 1 + 15 files changed, 534 insertions(+), 91 deletions(-) create mode 100644 tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml create mode 100644 tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml create mode 100644 tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml create mode 100644 tests/evals/gsm8k/configs/models-blackwell-ep-mi355.txt diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 55f7f382c8..69dd0d8cff 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3301,6 +3301,85 @@ steps: - pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py - pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py +- label: ":amd: (MI355) Sequence Parallel Correctness" + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - .buildkite/test_areas/compile.yaml + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/ + - vllm/model_executor/models/llama.py + - vllm/v1/attention/ + - vllm/v1/worker/ + - vllm/v1/cudagraph_dispatcher.py + - vllm/platforms/rocm.py + - tests/compile/correctness_e2e/test_sequence_parallel.py + - tests/models/registry.py + - tests/utils.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py + +- label: ":amd: (MI355) AsyncTP Correctness" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - .buildkite/test_areas/compile.yaml + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/layers/ + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/llama.py + - vllm/model_executor/models/qwen3.py + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/transformers_utils/repo_utils.py + - tests/compile/correctness_e2e/test_async_tp.py + - tests/compile/fusions_e2e/common.py + - tests/models/registry.py + - tests/utils.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_rocm_async_tp_bf16_output_correctness + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_async_tp_pass_correctness -k RedHatAI + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_async_tp_pass_nvfp4_correctness + +- label: ":amd: (MI355 DPX) Fusion E2E FP8 Config Sweep" + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - .buildkite/test_areas/compile.yaml + - csrc/quantization/ + - vllm/compilation/ + - vllm/model_executor/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/compile/fusions_e2e/ + commands: + - rocm-smi + - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "test_tp1_fp8_fusions and inductor_partition and not +rms_norm and not True and ((Llama-3 and TRITON_ATTN) or (Qwen3-30B and +quant_fp8 and TRITON_ATTN) or (DeepSeek-Coder and +quant_fp8 and TRITON_MLA))" + #-------------------------------------------------------- mi355 · distributed --------------------------------------------------------# - label: ":amd: (MI355 DPX) EPLB Algorithm" # TBD @@ -3572,6 +3651,26 @@ steps: - pytest -v -s tests/distributed/test_rocm_quick_reduce.py - pytest -v -s tests/distributed/test_quick_all_reduce.py +- label: ":amd: (MI355) Distributed AgRs All2All" + timeout_in_minutes: 30 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - .buildkite/test_areas/distributed.yaml + - vllm/distributed/ + - vllm/config/parallel.py + - vllm/config/vllm.py + - vllm/forward_context.py + - vllm/platforms/rocm.py + - tests/distributed/test_mnnvl_alltoall.py + - tests/utils.py + commands: + - pytest -v -s tests/distributed/test_mnnvl_alltoall.py::test_args_dispatch_combine + - label: ":amd: (MI355) Fusion E2E TP2 Quick" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3953,6 +4052,30 @@ steps: commands: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt +- label: ":amd: (MI355 DPX) LM Eval Small Models" + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_dpx + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - csrc/ + - tests/evals/gsm8k/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-gfx950-small.txt + - label: ":amd: (MI355 DPX) LM Eval Watermarking" timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -3999,7 +4122,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-watermark-dspark-tp2.txt - label: ":amd: (MI355) LM Eval Qwen3-5 Models" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false agent_pool: mi355_2 @@ -4018,6 +4141,10 @@ steps: - vllm/distributed/ - vllm/model_executor/layers/fused_moe/ - vllm/third_party/flash_linear_attention/ops/ + - vllm/model_executor/layers/quantization/modelopt.py + - vllm/model_executor/kernels/linear/nvfp4/ + - vllm/v1/attention/ + - vllm/v1/worker/ - tests/evals/gsm8k/ - tests/utils.py - vllm/_aiter_ops.py @@ -4025,6 +4152,48 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-mi355.txt +- label: ":amd: (MI355) LM Eval Large Models EP" + timeout_in_minutes: 120 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - csrc/ + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/config.py + - vllm/model_executor/models/registry.py + - vllm/model_executor/models/utils.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/model_executor/models/nemotron_h.py + - vllm/model_executor/models/nemotron_h_mtp.py + - vllm/transformers_utils/configs/qwen3_next.py + - vllm/transformers_utils/configs/nemotron_h.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/mamba/ + - vllm/model_executor/layers/layernorm.py + - vllm/model_executor/layers/linear.py + - vllm/model_executor/kernels/linear/ + - vllm/third_party/flash_linear_attention/ops/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep-mi355.txt + - label: ":amd: (MI355) Qwen3-8-Flash-Next-FP8 Accuracy Eval" timeout_in_minutes: 105 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -4046,11 +4215,11 @@ steps: - uv pip install --system 'evalscope==1.10.0' - pytest -s -v evals/qwen4_exp/test_accuracy.py --config-list-file=configs/models-rocm.txt -- label: ":amd: (MI355) LM Eval TurboQuant k8v4" +- label: ":amd: (MI355 DPX) LM Eval TurboQuant k8v4" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4072,11 +4241,11 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4-rocm.txt -- label: ":amd: (MI355) LM Eval TurboQuant t4nc" +- label: ":amd: (MI355 DPX) LM Eval TurboQuant t4nc" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4098,11 +4267,11 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc-rocm.txt -- label: ":amd: (MI355) LM Eval TurboQuant k3v4nc" +- label: ":amd: (MI355 DPX) LM Eval TurboQuant k3v4nc" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4124,11 +4293,11 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc-rocm.txt -- label: ":amd: (MI355) LM Eval TurboQuant t3nc" +- label: ":amd: (MI355 DPX) LM Eval TurboQuant t3nc" timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/tests" @@ -4500,41 +4669,6 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 -- label: ":amd: (MI355) Examples" # TBD - timeout_in_minutes: 75 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_1 - working_dir: "/vllm-workspace/examples" - source_file_dependencies: - - vllm/entrypoints - - vllm/multimodal - - examples/ - - vllm/platforms/rocm.py - optional: true - commands: - # Basic - # Multi-modal models - # Pooling models - # Features demo - - pip install --no-deps tensorizer - - python3 basic/offline_inference/chat.py - - python3 basic/offline_inference/generate.py --model facebook/opt-125m - - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - - python3 basic/offline_inference/classify.py - - python3 basic/offline_inference/embed.py - - python3 basic/offline_inference/score.py - - python3 generate/multimodal/audio_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_offline.py --seed 0 - - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 - - python3 pooling/embed/vision_embedding_offline.py --seed 0 - - python3 features/automatic_prefix_caching/prefix_caching_offline.py - - python3 deployment/llm_engine_example.py - - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 - - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - #---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------# - label: ":amd: (MI355) mHC Kernels" @@ -5291,12 +5425,12 @@ steps: #------------------------------------------------------- mi355 · quantization --------------------------------------------------------# -- label: ":amd: (MI355) Quantization Shard %N" +- label: ":amd: (MI355 DPX) Quantization Shard %N" timeout_in_minutes: 115 parallelism: 4 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/ @@ -5339,6 +5473,34 @@ steps: #------------------------------------------------------------ mi355 · v1 -------------------------------------------------------------# +- label: ":amd: (MI355) Spec Decode AL DFlash2 Nightly" + timeout_in_minutes: 60 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_1 + num_gpus: 1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - .buildkite/test_areas/spec_decode.yaml + - tests/v1/e2e/spec_decode/ + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/worker/gpu/sample/ + - vllm/v1/worker/gpu/model_runner.py + - vllm/v1/attention/ + - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/qwen3_dflash2.py + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "dflash2" + - label: ":amd: (MI355 DPX) Spec Decode Draft Model" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -6056,11 +6218,11 @@ steps: - pytest -v -s v1/test_outputs.py -- label: ":amd: (MI355) V1 Spec Decode" # TBD +- label: ":amd: (MI355 DPX) V1 Spec Decode" # TBD timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -6230,11 +6392,11 @@ steps: - amdmi355 optional: true -- label: ":amd: (MI355) MRV2 Sampler JIT Warmup" +- label: ":amd: (MI355 DPX) MRV2 Sampler JIT Warmup" timeout_in_minutes: 10 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_1 + agent_pool: mi355_dpx num_gpus: 1 optional: true working_dir: "/vllm-workspace/" diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index 3819c2b3a3..5fd0af69c1 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -12,6 +12,30 @@ steps: commands: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py + mirror: + amd: + label: ":amd: (MI355) Sequence Parallel Correctness" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 180 + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/compile.yaml + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/ + - vllm/model_executor/models/llama.py + - vllm/v1/attention/ + - vllm/v1/worker/ + - vllm/v1/cudagraph_dispatcher.py + - vllm/platforms/rocm.py + - tests/compile/correctness_e2e/test_sequence_parallel.py + - tests/models/registry.py + - tests/utils.py - label: ":nvidia: (B200) AsyncTP Correctness" key: asynctp-correctness-tests-b200 @@ -23,6 +47,38 @@ steps: commands: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py + mirror: + amd: + label: ":amd: (MI355) AsyncTP Correctness" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/compile.yaml + - vllm/compilation/ + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/layers/ + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/llama.py + - vllm/model_executor/models/qwen3.py + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/transformers_utils/repo_utils.py + - tests/compile/correctness_e2e/test_async_tp.py + - tests/compile/fusions_e2e/common.py + - tests/models/registry.py + - tests/utils.py + commands: + - export VLLM_TEST_CLEAN_GPU_MEMORY=1 + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_rocm_async_tp_bf16_output_correctness + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_async_tp_pass_correctness -k RedHatAI + - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py::test_async_tp_pass_nvfp4_correctness - label: ":nvidia: (H100) Distributed Compile" key: distributed-compile-unit-tests-2xh100 @@ -188,6 +244,27 @@ steps: # Qwen/Deepseek requires +quant_fp8 as -quant_fp8 rms+quant fusion is not supported # Run just llama3 (fp8 & fp4) for all config combinations (only inductor partition) - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "inductor_partition and (FLASHINFER and not +rms_norm and (not +quant_fp8 or +quant_fp8 and (qwen3 or deepseek)) or llama-3)" + mirror: + amd: + label: ":amd: (MI355 DPX) Fusion E2E FP8 Config Sweep" + dind: false + device: mi355_dpx + timeout_in_minutes: 45 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/compile.yaml + - csrc/quantization/ + - vllm/compilation/ + - vllm/model_executor/ + - vllm/v1/attention/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/compile/fusions_e2e/ + commands: + - rocm-smi + - pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k "test_tp1_fp8_fusions and inductor_partition and not +rms_norm and not True and ((Llama-3 and TRITON_ATTN) or (Qwen3-30B and +quant_fp8 and TRITON_ATTN) or (DeepSeek-Coder and +quant_fp8 and TRITON_MLA))" - label: ":nvidia: (H100) Fusion E2E TP2 Quick" key: fusion-e2e-tp2-quick-h100 diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 0ed630daab..a7f65417a6 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -439,6 +439,27 @@ steps: - pytest -v -s tests/distributed/test_nccl_symm_mem.py - pytest -v -s tests/v1/distributed/test_dbo.py - pytest -v -s tests/distributed/test_mnnvl_alltoall.py + mirror: + amd: + label: ":amd: (MI355) Distributed AgRs All2All" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/" + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/distributed.yaml + - vllm/distributed/ + - vllm/config/parallel.py + - vllm/config/vllm.py + - vllm/forward_context.py + - vllm/platforms/rocm.py + - tests/distributed/test_mnnvl_alltoall.py + - tests/utils.py + commands: + - pytest -v -s tests/distributed/test_mnnvl_alltoall.py::test_args_dispatch_combine - label: ":nvidia: (B200) GEMM-RS/AR" key: gemm-rs-ar-2xb200 diff --git a/.buildkite/test_areas/jit_monitor.yaml b/.buildkite/test_areas/jit_monitor.yaml index 27de713fa2..9e40c9d1af 100644 --- a/.buildkite/test_areas/jit_monitor.yaml +++ b/.buildkite/test_areas/jit_monitor.yaml @@ -23,9 +23,9 @@ steps: mirror: amd: # Focused MRV2 regression; the broader model/backend matrix is pending. - label: ':amd: (MI355) MRV2 Sampler JIT Warmup' + label: ':amd: (MI355 DPX) MRV2 Sampler JIT Warmup' dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 10 working_dir: /vllm-workspace/ diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index c81bba10da..ce74f3a447 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -151,15 +151,18 @@ steps: label: ":amd: (MI355 DPX) LM Eval Small Models" dind: false device: mi355_dpx + num_devices: 1 timeout_in_minutes: 45 working_dir: /vllm-workspace/tests depends_on: - image-build-amd source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml - csrc/ - tests/evals/gsm8k/ - vllm/model_executor/layers/quantization - vllm/model_executor/kernels/linear/ + - vllm/model_executor/layers/fused_moe/ - vllm/model_executor/models/ - vllm/model_executor/model_loader/ - vllm/v1/attention/backends/ @@ -383,6 +386,49 @@ steps: - vllm/model_executor/layers/quantization commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt + mirror: + amd: + label: ":amd: (MI355) LM Eval Large Models EP" + dind: false + device: mi355_2 + num_devices: 2 + timeout_in_minutes: 120 + working_dir: "/vllm-workspace/tests" + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/lm_eval.yaml + - tests/evals/gsm8k/ + - tests/utils.py + - tests/conftest.py + - csrc/ + - vllm/config/ + - vllm/distributed/ + - vllm/model_executor/models/config.py + - vllm/model_executor/models/registry.py + - vllm/model_executor/models/utils.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/qwen3_next_mtp.py + - vllm/model_executor/models/nemotron_h.py + - vllm/model_executor/models/nemotron_h_mtp.py + - vllm/transformers_utils/configs/qwen3_next.py + - vllm/transformers_utils/configs/nemotron_h.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/mamba/ + - vllm/model_executor/layers/layernorm.py + - vllm/model_executor/layers/linear.py + - vllm/model_executor/kernels/linear/ + - vllm/third_party/flash_linear_attention/ops/ + - vllm/v1/attention/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/ + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep-mi355.txt - label: ":nvidia: (B200) LM Eval Qwen3.5 Models" key: lm-eval-qwen3-5-models-2xb200 @@ -405,7 +451,7 @@ steps: label: ":amd: (MI355) LM Eval Qwen3.5 Models" dind: false device: mi355_2 - timeout_in_minutes: 40 + timeout_in_minutes: 75 depends_on: - image-build-amd source_file_dependencies: @@ -418,10 +464,18 @@ steps: - vllm/model_executor/models/qwen3_next.py - vllm/model_executor/models/qwen3_next_mtp.py - vllm/third_party/flash_linear_attention/ops/ + - vllm/distributed/ + - vllm/model_executor/layers/fused_moe/ + - vllm/model_executor/layers/quantization/modelopt.py + - vllm/model_executor/kernels/linear/nvfp4/ + - vllm/v1/attention/ + - vllm/v1/worker/ - tests/evals/gsm8k/configs/models-qwen35-mi355.txt - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-DEP2.yaml + - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-FP8-DEP2.yaml - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml - tests/evals/gsm8k/configs/Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml + - tests/evals/gsm8k/configs/Qwen3.5-397B-A17B-NVFP4-DEP2.yaml - tests/evals/gsm8k/test_gsm8k_correctness.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py @@ -855,9 +909,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k8v4.txt mirror: amd: - label: ":amd: (MI355) LM Eval TurboQuant k8v4" + label: ":amd: (MI355 DPX) LM Eval TurboQuant k8v4" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 30 working_dir: /vllm-workspace/tests @@ -895,9 +949,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t4nc.txt mirror: amd: - label: ":amd: (MI355) LM Eval TurboQuant t4nc" + label: ":amd: (MI355 DPX) LM Eval TurboQuant t4nc" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 30 working_dir: /vllm-workspace/tests @@ -935,9 +989,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-k3v4nc.txt mirror: amd: - label: ":amd: (MI355) LM Eval TurboQuant k3v4nc" + label: ":amd: (MI355 DPX) LM Eval TurboQuant k3v4nc" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 30 working_dir: /vllm-workspace/tests @@ -975,9 +1029,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant-t3nc.txt mirror: amd: - label: ":amd: (MI355) LM Eval TurboQuant t3nc" + label: ":amd: (MI355 DPX) LM Eval TurboQuant t3nc" dind: false - device: mi355_1 + device: mi355_dpx num_devices: 1 timeout_in_minutes: 30 working_dir: /vllm-workspace/tests diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 73d944a7a4..af7e8f071a 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -423,6 +423,32 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode/acceptance_rates/dflash/ -k "dflash2" + mirror: + amd: + label: ":amd: (MI355) Spec Decode AL DFlash2 Nightly" + dind: false + device: mi355_1 + num_devices: 1 + timeout_in_minutes: 60 + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/spec_decode.yaml + - tests/v1/e2e/spec_decode/ + - tests/evals/gsm8k/ + - tests/utils.py + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/worker/gpu/sample/ + - vllm/v1/worker/gpu/model_runner.py + - vllm/v1/attention/ + - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/qwen3_dflash2.py + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/quantization/ + - vllm/model_executor/kernels/linear/ + - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 35GB) Spec Decode AL DSpark Nightly" key: spec-decode-dspark-nightly diff --git a/tests/compile/correctness_e2e/test_async_tp.py b/tests/compile/correctness_e2e/test_async_tp.py index 6b8c26cadc..307f39039b 100644 --- a/tests/compile/correctness_e2e/test_async_tp.py +++ b/tests/compile/correctness_e2e/test_async_tp.py @@ -3,6 +3,7 @@ import json import logging +from contextlib import nullcontext import pytest @@ -174,13 +175,18 @@ def test_async_tp_pass_correctness( @create_new_process_for_each_test() -def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int): - if ( - not current_platform.is_cuda() - or not current_platform.is_device_capability_family(100) - ): - pytest.skip("NVFP4 requires Blackwell") - if not has_flashinfer(): +def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, caplog_mp_spawn): + is_blackwell = current_platform.is_cuda() and ( + current_platform.is_device_capability_family(100) + ) + is_rocm_cdna = False + if current_platform.is_rocm(): + from vllm.platforms.rocm import on_cdna + + is_rocm_cdna = on_cdna() + if not (is_blackwell or is_rocm_cdna): + pytest.skip("NVFP4 AsyncTP requires Blackwell or ROCm CDNA") + if is_blackwell and not has_flashinfer(): pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path") tp_size = 2 @@ -196,8 +202,7 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int): "8", "--load-format", "dummy", - "--linear-backend", - "flashinfer_cutlass", + *(["--linear-backend", "flashinfer_cutlass"] if is_blackwell else []), "--hf-overrides", json.dumps(NVFP4_HF_OVERRIDES), ] @@ -213,6 +218,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int): "sp_min_token_num": 1, }, } + if is_rocm_cdna: + compilation_config["inductor_compile_config"] = {"force_disable_caches": True} async_tp_args = [ *common_args, @@ -232,10 +239,32 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int): "mp", ] - compare_two_settings( - NVFP4_MODEL_ID, - async_tp_args, - tp_args, - method="generate", - force_v1_runner=True, - ) + env = {"VLLM_DISABLE_COMPILE_CACHE": "1"} if is_rocm_cdna else None + log_context = caplog_mp_spawn(logging.DEBUG) if is_rocm_cdna else nullcontext() + with log_context as logs: + compare_two_settings( + NVFP4_MODEL_ID, + async_tp_args, + tp_args, + env1=env, + env2=env, + method="generate", + force_v1_runner=True, + ) + if is_rocm_cdna: + assert logs is not None + matches_by_rank = { + rank: [ + int(count) + for line in logs.text.splitlines() + if line.startswith(f"(Worker_TP{rank} pid=") + for count in FUSION_LOG_PATTERNS["async_tp"].findall(line) + ] + for rank in range(tp_size) + } + assert all( + any(count > 0 for count in matches) for matches in matches_by_rank.values() + ), ( + "Expected positive NVFP4 AsyncTP rewrites on both TP workers, " + f"found {matches_by_rank}" + ) diff --git a/tests/compile/correctness_e2e/test_sequence_parallel.py b/tests/compile/correctness_e2e/test_sequence_parallel.py index e5f8d1870e..10d25320c0 100644 --- a/tests/compile/correctness_e2e/test_sequence_parallel.py +++ b/tests/compile/correctness_e2e/test_sequence_parallel.py @@ -374,11 +374,16 @@ def test_tp_sp_generation_prompt_embeds( @create_new_process_for_each_test() def test_tp_sp_nvfp4_generation(num_gpus_available: int): - if ( - not current_platform.is_cuda() - or not current_platform.is_device_capability_family(100) - ): - pytest.skip("NVFP4 requires Blackwell") + is_blackwell = current_platform.is_cuda() and ( + current_platform.is_device_capability_family(100) + ) + is_rocm_cdna = False + if current_platform.is_rocm(): + from vllm.platforms.rocm import on_cdna + + is_rocm_cdna = on_cdna() + if not (is_blackwell or is_rocm_cdna): + pytest.skip("NVFP4 sequence parallelism requires Blackwell or ROCm CDNA") comparison = _build_sp_args( NVFP4_MODEL_ID, diff --git a/tests/distributed/test_mnnvl_alltoall.py b/tests/distributed/test_mnnvl_alltoall.py index 362ffe4b15..89fe6eb6ad 100644 --- a/tests/distributed/test_mnnvl_alltoall.py +++ b/tests/distributed/test_mnnvl_alltoall.py @@ -8,6 +8,7 @@ Run: pytest tests/distributed/test_mnnvl_alltoall.py -v import os import traceback +from contextlib import contextmanager import pytest import torch @@ -148,16 +149,17 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port): ensure_model_parallel_initialized(1, 1) +@contextmanager def _make_forward_context(rank, world_size, num_tokens_per_rank): - """Create a forward context with mock DP metadata for AgRs tests. + """Keep the vLLM config and mock-DP forward context active together. Returns a context manager suitable for ``with`` statements. The real DPMetadata (with sp_local_sizes etc.) is created internally by set_forward_context from num_tokens_across_dp; the attn_metadata placeholder just satisfies the "attn_metadata is not None" guard. """ + from vllm.config import VllmConfig, set_current_vllm_config from vllm.config.parallel import ParallelConfig - from vllm.config.vllm import VllmConfig from vllm.forward_context import set_forward_context class _AttnMeta: @@ -173,14 +175,18 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank): is_moe_model=True, data_parallel_rank=rank, ) - return set_forward_context( - _AttnMeta(), - vllm_config, - num_tokens=num_tokens_per_rank, - num_tokens_across_dp=torch.tensor( - [num_tokens_per_rank] * world_size, dtype=torch.int + with ( + set_current_vllm_config(vllm_config), + set_forward_context( + _AttnMeta(), + vllm_config, + num_tokens=num_tokens_per_rank, + num_tokens_across_dp=torch.tensor( + [num_tokens_per_rank] * world_size, dtype=torch.int + ), ), - ) + ): + yield # --------------------------------------------------------------------------- diff --git a/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml new file mode 100644 index 0000000000..473342569f --- /dev/null +++ b/tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml @@ -0,0 +1,18 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" +accuracy_threshold: 0.93 +num_questions: 1319 +num_fewshot: 5 +max_concurrency: 16 +rocm_request_timeout_seconds: 1800 +min_acceptance_length: 2.0 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --max-num-seqs 16 + --max-num-batched-tokens 32768 + --gpu-memory-utilization 0.5 + --tensor-parallel-size 2 + --enable-expert-parallel + --mamba-backend triton + --speculative-config '{"method":"mtp","num_speculative_tokens":5}' diff --git a/tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml b/tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml new file mode 100644 index 0000000000..ab62af4ac1 --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml @@ -0,0 +1,17 @@ +model_name: "nm-testing/Qwen3-Next-80B-A3B-Instruct-NVFP4" +accuracy_threshold: 0.75 +num_questions: 1319 +num_fewshot: 5 +max_concurrency: 16 +rocm_request_timeout_seconds: 1800 +server_args: >- + --enforce-eager + --max-model-len 4096 + --max-num-seqs 16 + --max-num-batched-tokens 32768 + --gpu-memory-utilization 0.5 + --tensor-parallel-size 2 + --enable-expert-parallel + --all2all-backend allgather_reducescatter + --attention-backend TRITON_ATTN + --no-async-scheduling diff --git a/tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml b/tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml new file mode 100644 index 0000000000..e20facf21a --- /dev/null +++ b/tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml @@ -0,0 +1,19 @@ +model_name: "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4" +accuracy_threshold: 0.85 +num_questions: 1319 +num_fewshot: 5 +max_concurrency: 16 +rocm_request_timeout_seconds: 1800 +min_acceptance_length: 1.5 +server_args: >- + --enforce-eager + --max-model-len 4096 + --max-num-seqs 16 + --max-num-batched-tokens 32768 + --gpu-memory-utilization 0.5 + --tensor-parallel-size 2 + --enable-expert-parallel + --all2all-backend allgather_reducescatter + --attention-backend TRITON_ATTN + --no-async-scheduling + --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}' diff --git a/tests/evals/gsm8k/configs/models-blackwell-ep-mi355.txt b/tests/evals/gsm8k/configs/models-blackwell-ep-mi355.txt new file mode 100644 index 0000000000..1bb71eaa05 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-blackwell-ep-mi355.txt @@ -0,0 +1,5 @@ +# Exact B200 compressed-tensors checkpoint; it does not contain MTP weights. +Qwen3-Next-80B-A3B-NVFP4-EP2-CT-MI355.yaml +# MTP-capable ModelOpt checkpoint exercises NVFP4 EP2 with MTP1. +Qwen3-Next-80B-A3B-NVFP4-EP2-MTP-MI355.yaml +Nemotron-3-Super-120B-A12B-NVFP4-MI355.yaml diff --git a/tests/evals/gsm8k/configs/models-gfx950-small.txt b/tests/evals/gsm8k/configs/models-gfx950-small.txt index d818a067af..05b78e934c 100644 --- a/tests/evals/gsm8k/configs/models-gfx950-small.txt +++ b/tests/evals/gsm8k/configs/models-gfx950-small.txt @@ -1,2 +1,5 @@ Qwen3-0.6B-FP8.yaml Qwen3-1.7B-MXFP4.yaml +Qwen3-30B-A3B-NVFP4.yaml +Laguna-XS.2-NVFP4.yaml +Laguna-XS-2.1-INT4.yaml diff --git a/tests/evals/gsm8k/configs/models-qwen35-mi355.txt b/tests/evals/gsm8k/configs/models-qwen35-mi355.txt index 13ff05f870..9fad858d24 100644 --- a/tests/evals/gsm8k/configs/models-qwen35-mi355.txt +++ b/tests/evals/gsm8k/configs/models-qwen35-mi355.txt @@ -2,3 +2,4 @@ Qwen3.5-35B-A3B-DEP2.yaml Qwen3.5-35B-A3B-FP8-DEP2.yaml Qwen3.5-35B-A3B-MXFP4-AITER-TP2.yaml Qwen3.5-35B-A3B-MXFP4-EMU-TP2.yaml +Qwen3.5-397B-A17B-NVFP4-DEP2.yaml From 3acddf01e709b13693d3a90d25fdb74d8124049c Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 04:14:26 -0500 Subject: [PATCH 017/248] [Bugfix][ROCm] Keep Mooncake bootstrap ports bound during startup (#58967) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- docs/features/mooncake_connector_usage.md | 1 + .../v1/kv_connector/rocm_pd_accuracy_utils.py | 13 +- .../unit/test_mooncake_connector.py | 147 ++++++++++++++++-- .../v1/mooncake/mooncake_connector.py | 15 ++ .../v1/mooncake/mooncake_utils.py | 86 +++++++--- 5 files changed, 226 insertions(+), 36 deletions(-) diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 7c5852a68b..701a1d0049 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -76,6 +76,7 @@ Now you can send requests to the proxy server through port 8000. - **num_workers**: Size of thread pool for one prefiller worker to transfer KV caches by mooncake. (default 10) - **mooncake_protocol**: Mooncake connector protocol. (default "rdma") - **device_name**: Comma-separated whitelist of RDMA devices (e.g. `"mlx5_0,mlx5_1"`) to restrict topology discovery to. Empty discovers every device. Useful on hosts exposing a mix of InfiniBand and RoCE ports, where both peers must settle on the same link layer. +- **bootstrap_server_address**: Optional `"host:port"` of an already running Mooncake bootstrap server. All prefiller ranks register with this endpoint instead of launching an embedded server. The caller owns the server lifetime and must configure the proxy to use the same endpoint. The example proxy assumes the bootstrap server is on the prefiller API host; a different host requires a proxy that supplies that address in `remote_bootstrap_addr`. Omit this option to use the embedded server and `VLLM_MOONCAKE_BOOTSTRAP_PORT`. ## Example Scripts/Code diff --git a/tests/v1/kv_connector/rocm_pd_accuracy_utils.py b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py index e515c0a0bd..37d56ef6e4 100644 --- a/tests/v1/kv_connector/rocm_pd_accuracy_utils.py +++ b/tests/v1/kv_connector/rocm_pd_accuracy_utils.py @@ -158,8 +158,18 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch): "VLLM_ROCM_USE_AITER": "1", "VLLM_KV_CACHE_LAYOUT": "HND", } - bootstrap_port = get_open_port() with contextlib.ExitStack() as stack: + bootstrap_port = 0 + if connector == "mooncake": + from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import ( # noqa: E501 + MooncakeBootstrapServer, + ) + + bootstrap = MooncakeBootstrapServer("127.0.0.1", 0) + stack.callback(bootstrap.shutdown) + bootstrap.start() + bootstrap_port = bootstrap.port + extra["bootstrap_server_address"] = f"127.0.0.1:{bootstrap_port}" servers: list[RemoteOpenAIServer] = [] stack.callback(RemoteOpenAIServer.shutdown_many, servers) for role, tp, assigned in ( @@ -176,7 +186,6 @@ def run_rocm_pd_accuracy(connector, model, prefill_tp, decode_tp, monkeypatch): selector: ",".join(assigned), "VLLM_PORT": str(get_open_port()), "VLLM_NIXL_SIDE_CHANNEL_PORT": str(get_open_port()), - "VLLM_MOONCAKE_BOOTSTRAP_PORT": str(bootstrap_port), } servers.append( RemoteOpenAIServer( diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index 5a5fffac3d..be70a46bf7 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -3,6 +3,8 @@ import asyncio import contextlib +import errno +import socket import threading import time from types import SimpleNamespace @@ -15,8 +17,11 @@ import zmq.asyncio from tests.v1.attention.utils import dense_kv_cache_views from vllm import envs -from vllm.config import set_current_vllm_config -from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import mooncake_connector +from vllm.config import KVTransferConfig, set_current_vllm_config +from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import ( + mooncake_connector, + mooncake_utils, +) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( KVConnectorRole, MooncakeConnector, @@ -38,7 +43,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_utils import MooncakeBootstrapServer, RegisterWorkerPayload, ) -from vllm.utils.network_utils import get_open_port from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheConfig, @@ -535,15 +539,99 @@ def test_prompt_less_than_block_size(): @pytest.fixture def bootstrap_server(): """Fixture to launch and cleanup a Mooncake Bootstrap HTTP Server.""" - port = get_open_port() - server = MooncakeBootstrapServer("127.0.0.1", port) - server.start() - yield server - server.shutdown() + server = MooncakeBootstrapServer("127.0.0.1", 0) + try: + server.start() + yield server + finally: + server.shutdown() + + +def test_bootstrap_server_occupied_port_raises_to_caller(): + with socket.create_server(("127.0.0.1", 0)) as listener: + server = MooncakeBootstrapServer("127.0.0.1", listener.getsockname()[1]) + try: + with pytest.raises(OSError) as exc_info: + server.start(timeout=0.05) + assert exc_info.value.errno == errno.EADDRINUSE + finally: + server.shutdown() + + +def test_bootstrap_server_retains_and_releases_assigned_port(bootstrap_server): + port = bootstrap_server.port + assert port > 0 + with ( + pytest.raises(OSError) as exc_info, + socket.create_server(("127.0.0.1", port)), + ): + pass + assert exc_info.value.errno == errno.EADDRINUSE + + bootstrap_server.shutdown() + with socket.create_server(("127.0.0.1", port)): + pass + + +@pytest.mark.parametrize("failure", ["exit", "error", "timeout"]) +def test_bootstrap_server_startup_failure_releases_port(monkeypatch, failure): + stop = threading.Event() + + def run_without_starting(server, *, sockets): + if failure == "error": + raise SystemExit(1) + if failure == "timeout": + while not server.should_exit and not stop.wait(0.01): + pass + + monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_without_starting) + server = MooncakeBootstrapServer("127.0.0.1", 0) + try: + error = TimeoutError if failure == "timeout" else RuntimeError + with pytest.raises(error): + server.start(timeout=0.05 if failure == "timeout" else 1.0) + assert server.port > 0 + with socket.create_server(("127.0.0.1", server.port)): + pass + finally: + stop.set() + server.shutdown() + + +@pytest.mark.parametrize("state", ["stopping", "exited"]) +def test_bootstrap_server_rejects_stale_started_state(monkeypatch, state): + release = threading.Event() + + def run_until_released(server, *, sockets): + server.started = True + release.wait() + + monkeypatch.setattr(mooncake_utils.uvicorn.Server, "run", run_until_released) + server = MooncakeBootstrapServer("127.0.0.1", 0) + try: + server.start() + thread = server.server_thread + assert thread is not None + if state == "stopping": + with patch.object(thread, "join", return_value=None): + server.shutdown() + else: + release.set() + thread.join(timeout=5) + assert not thread.is_alive() + + with pytest.raises(RuntimeError, match="still starting or stopping"): + server.start() + finally: + release.set() + server.shutdown() @pytest.mark.asyncio -async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch): +@pytest.mark.parametrize("external", [False, True]) +async def test_register_worker_recovers_from_slow_bootstrap_server( + monkeypatch, external +): """End-to-end: a real HTTP server that responds slower than the configured timeout must be retried on the wire, not treated as fatal.""" original = MooncakeBootstrapServer.register_worker @@ -562,20 +650,24 @@ async def test_register_worker_recovers_from_slow_bootstrap_server(monkeypatch): monkeypatch.setattr(MooncakeBootstrapServer, "register_worker", slow_register) - port = get_open_port() - monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(port)) monkeypatch.setenv("VLLM_MOONCAKE_CONNECTOR_TIMEOUT", "0.5") monkeypatch.setattr(mooncake_connector, "_BOOTSTRAP_MAX_ATTEMPTS", 5) - server = MooncakeBootstrapServer("127.0.0.1", port) + server = MooncakeBootstrapServer("127.0.0.1", 0) server.start() try: + monkeypatch.setenv("VLLM_MOONCAKE_BOOTSTRAP_PORT", str(server.port)) worker = _make_local_register_worker_stub() + if external: + worker.vllm_config = _make_bootstrap_vllm_config( + bootstrap_server_address=f"127.0.0.1:{server.port}" + ) + assert not should_launch_bootstrap_server(worker.vllm_config) await MooncakeConnectorWorker.register_worker_with_bootstrap(worker) assert state["calls"] == 2 async with httpx.AsyncClient() as client: - response = await client.get(f"http://127.0.0.1:{port}/query") + response = await client.get(f"http://127.0.0.1:{server.port}/query") assert response.status_code == 200 assert response.json()["0"]["engine_id"] == "eng-1" finally: @@ -666,8 +758,16 @@ def _make_bootstrap_vllm_config( data_parallel_rank_local: int = 0, data_parallel_index: int = 0, nnodes_within_dp: int = 1, + bootstrap_server_address: str | None = None, ) -> SimpleNamespace: return SimpleNamespace( + kv_transfer_config=KVTransferConfig( + kv_connector="MooncakeConnector", + kv_role="kv_producer", + kv_connector_extra_config={ + "bootstrap_server_address": bootstrap_server_address + }, + ), parallel_config=SimpleNamespace( local_engines_only=local_engines_only, data_parallel_rank_local=data_parallel_rank_local, @@ -675,13 +775,14 @@ def _make_bootstrap_vllm_config( nnodes_within_dp=nnodes_within_dp, master_addr="model-parallel-master", data_parallel_master_ip="data-parallel-master", - ) + ), ) def _make_local_register_worker_stub(): return SimpleNamespace( vllm_config=SimpleNamespace( + kv_transfer_config=None, parallel_config=SimpleNamespace( local_engines_only=False, data_parallel_rank_local=0, @@ -689,7 +790,7 @@ def _make_local_register_worker_stub(): nnodes_within_dp=1, master_addr="127.0.0.1", data_parallel_master_ip="127.0.0.1", - ) + ), ), hostname="127.0.0.1", side_channel_port=1234, @@ -889,6 +990,22 @@ def test_get_mooncake_bootstrap_addr_selects_expected_host( ) +@pytest.mark.parametrize( + ("address", "expected"), + [("external:12345", ("external", 12345)), ("[::1]:12345", ("::1", 12345))], +) +def test_get_mooncake_bootstrap_addr_uses_external_endpoint(address, expected): + config = _make_bootstrap_vllm_config(bootstrap_server_address=address) + assert get_mooncake_bootstrap_addr(config) == expected + + +@pytest.mark.parametrize("address", [":12345", "host:0", "host:65536"]) +def test_get_mooncake_bootstrap_addr_rejects_invalid_external_address(address): + config = _make_bootstrap_vllm_config(bootstrap_server_address=address) + with pytest.raises(ValueError, match="bootstrap_server_address"): + get_mooncake_bootstrap_addr(config) + + def test_scheduler_request_finished(): """Tests the scheduler-side logic when a request finishes. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index 5ef4f8f670..e3fb5b95c9 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -2329,6 +2329,10 @@ def _async_loop(loop: asyncio.AbstractEventLoop): def should_launch_bootstrap_server(vllm_config: VllmConfig) -> bool: + if (kv_config := vllm_config.kv_transfer_config) is not None and ( + kv_config.get_from_extra_config("bootstrap_server_address", None) is not None + ): + return False assert (parallel_config := vllm_config.parallel_config) # Only the TP=0, PP=0 worker of the designated engine should launch it. if get_tensor_model_parallel_rank() != 0: @@ -2351,6 +2355,17 @@ def get_mooncake_bootstrap_addr(vllm_config: VllmConfig) -> tuple[str, int]: This is only used by prefillers to register workers. Decoders should get addr from kv_transfer_params. """ + if (kv_config := vllm_config.kv_transfer_config) is not None: + address = kv_config.get_from_extra_config("bootstrap_server_address", None) + if address is not None: + from vllm.utils.network_utils import split_host_port + + host, port = split_host_port(address) + if not host or not 1 <= port <= 65535: + raise ValueError( + "bootstrap_server_address must specify a host and port" + ) + return host, port assert (parallel_config := vllm_config.parallel_config) if parallel_config.local_engines_only: # In hybrid or external LB mode, connect to local server. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py index dfe0641b36..5f5fba157f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import socket import threading import time from dataclasses import dataclass @@ -32,9 +33,7 @@ class EngineEntry: class MooncakeBootstrapServer: - """A centralized server running on the global rank 0 prefiller worker. - Prefiller workers register their connection info (IP, port, ranks) here. - """ + """A centralized registry for prefiller connection info (IP, port, ranks).""" def __init__(self, host: str, port: int): self.workers: dict[int, EngineEntry] = {} @@ -45,6 +44,8 @@ class MooncakeBootstrapServer: self._register_routes() self.server_thread: threading.Thread | None = None self.server: uvicorn.Server | None = None + self._socket: socket.socket | None = None + self._startup_error: BaseException | None = None def __del__(self): self.shutdown() @@ -54,27 +55,74 @@ class MooncakeBootstrapServer: self.app.post("/register")(self.register_worker) self.app.get("/query", response_model=dict[int, EngineEntry])(self.query) - def start(self): + def start(self, *, timeout: float = 30.0): if self.server_thread: - return + if ( + self.server_thread.is_alive() + and self.server is not None + and self.server.started + and not self.server.should_exit + and self._socket is not None + and self._socket.fileno() >= 0 + ): + return + raise RuntimeError( + "Mooncake bootstrap server is still starting or stopping" + ) - config = uvicorn.Config(app=self.app, host=self.host, port=self.port) - self.server = uvicorn.Server(config=config) - self.server_thread = threading.Thread( - target=self.server.run, name="mooncake_bootstrap_server", daemon=True - ) - self.server_thread.start() - while not self.server.started: - time.sleep(0.1) # Wait for the server to start + try: + # Bind in the caller so errors propagate, and retain the listener + # when handing it to Uvicorn, including for automatically chosen ports. + family = socket.AF_INET6 if ":" in self.host else socket.AF_INET + self._socket = socket.create_server((self.host, self.port), family=family) + self.port = self._socket.getsockname()[1] + config = uvicorn.Config(app=self.app, host=self.host, port=self.port) + server = self.server = uvicorn.Server(config=config) + listener = self._socket + self._startup_error = None + + def run(): + try: + server.run(sockets=[listener]) + except BaseException as exc: + self._startup_error = exc + + self.server_thread = threading.Thread( + target=run, name="mooncake_bootstrap_server", daemon=True + ) + self.server_thread.start() + deadline = time.monotonic() + timeout + while not server.started: + if not self.server_thread.is_alive(): + raise RuntimeError( + "Mooncake bootstrap server exited during startup" + ) from self._startup_error + if time.monotonic() >= deadline: + raise TimeoutError( + "Mooncake bootstrap server did not start in time" + ) + time.sleep(0.01) + except BaseException: + self.shutdown() + raise logger.info("Mooncake Bootstrap Server started at %s:%d", self.host, self.port) def shutdown(self): - if self.server_thread is None or self.server is None or not self.server.started: - return - - self.server.should_exit = True - self.server_thread.join() - logger.info("Mooncake Bootstrap Server stopped.") + was_started = self.server is not None and self.server.started + if self.server is not None: + self.server.should_exit = True + if self.server_thread is not None and self.server_thread.ident is not None: + self.server_thread.join(timeout=5) + if self._socket is not None: + self._socket.close() + self._socket = None + if self.server_thread is not None and self.server_thread.is_alive(): + logger.warning("Mooncake bootstrap server did not stop in time") + else: + self.server_thread = None + self.server = None + if was_started: + logger.info("Mooncake Bootstrap Server stopped.") async def register_worker(self, payload: RegisterWorkerPayload): """Handles registration of a prefiller worker.""" From 3e2a7e74a556623ddee3a327299d7bf04055501e Mon Sep 17 00:00:00 2001 From: Chauncey Date: Tue, 29 Sep 2026 17:14:30 +0800 Subject: [PATCH 018/248] [Feat][Model] Enable KDA prefill checkpoints for GLM-5.3-Flash (#56960) Signed-off-by: chaunceyjiang --- tests/models/glm5next/test_kda_recurrent.py | 167 ++++++++++++++++++ .../v1/attention/test_gdn_metadata_builder.py | 63 +++++++ .../model_executor/layers/mamba/checkpoint.py | 2 + vllm/models/glm5next/common/kda.py | 62 ++++++- vllm/models/kimi_k3/nvidia/kda_metadata.py | 8 - vllm/v1/attention/backends/gdn_attn.py | 16 ++ 6 files changed, 307 insertions(+), 11 deletions(-) diff --git a/tests/models/glm5next/test_kda_recurrent.py b/tests/models/glm5next/test_kda_recurrent.py index 234a63fb73..ddda0e0c34 100644 --- a/tests/models/glm5next/test_kda_recurrent.py +++ b/tests/models/glm5next/test_kda_recurrent.py @@ -193,3 +193,170 @@ def test_fused_recurrent_kda_rejects_unaddressable_layouts(): broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"] with pytest.raises(AssertionError, match=r"torch.Size"): run_kernel(broken, state) + + +@pytest.mark.parametrize("offset", [16, 144, 256]) +@pytest.mark.parametrize("dim_first", [False, True]) +@pytest.mark.parametrize("num_spec", [0, 3]) +@torch.inference_mode() +def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset): + """Restoring both cached states must reproduce an uninterrupted prefill.""" + from types import SimpleNamespace + + from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, + ) + from vllm.model_executor.layers.mamba.kda_checkpoint import ( + FlashKDAPrefillCheckpointExporter, + ) + from vllm.models.glm5next.common import kda + from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata + from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata + from vllm.v1.worker.workspace import WorkspaceManager + + if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda": + pytest.skip("requires FlashKDA") + import vllm._flashkda_C # noqa: F401 + + torch.manual_seed(42) + device = torch.device("cuda") + length, heads = 273, 2 + width, state_len = 3 * heads * D, 3 + num_spec + layer = object.__new__(kda.Glm5NextLinearAttention) + torch.nn.Module.__init__(layer) + layer.prefix = "layer.0" + layer.local_num_heads = heads + layer.head_dim = D + layer.conv_size = 4 + layer.local_projection_size = heads * D + layer.kda_safe_gate = True + layer.kda_lower_bound = -5.0 + layer.kda_prefill_backend = "flashkda" + layer._conv_state_dim_first = dim_first + layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1 + layer.q_conv1d = SimpleNamespace(bias=None) + layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1 + layer.dt_bias = torch.randn(heads * D, device=device) * 0.1 + state_shape = (2, heads, D, D) + layer._flashkda_buffer_specs = ( + (state_shape, torch.float32), + (state_shape, torch.float32), + ( + (torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),), + torch.uint8, + ), + ((1, 2 * length, heads, D), torch.bfloat16), + ) + layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3) + workspace = WorkspaceManager(device) + monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace) + conv_shape = (width, state_len) if dim_first else (state_len, width) + num_slots = 5 + (num_spec + 1 if num_spec else 0) + conv_storage = torch.zeros( + num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16 + ) + conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape) + recurrent_storage = torch.zeros( + num_slots, heads * D * D + 8, device=device, dtype=torch.float32 + ) + recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D) + layer.kv_cache = (conv, recurrent) + projected = torch.randn( + 2 * length, width + heads, device=device, dtype=torch.bfloat16 + ) + raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0) + raw_before = raw_qkv.clone() + gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16) + + def run(qkv, g, b, slots, has_initial, checkpoint=None): + n = qkv.shape[0] // 2 + cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32) + nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata( + cu_cpu, device=device + ) + metadata = GDNAttentionMetadata( + num_prefills=2, + num_prefill_tokens=2 * n, + num_decodes=0, + num_decode_tokens=0, + num_spec_decodes=0, + num_spec_decode_tokens=0, + num_actual_tokens=2 * n, + has_initial_state=torch.tensor(has_initial, device=device), + non_spec_query_start_loc=cu_cpu.to(device), + non_spec_state_indices_tensor=torch.tensor( + slots, dtype=torch.int32, device=device + ), + checkpoint=checkpoint, + nums_dict=nums, + batch_ptr=batch_ptr, + token_chunk_offset_ptr=chunk_offsets, + ) + monkeypatch.setattr( + kda, + "get_forward_context", + lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}), + ) + spec_tokens = num_spec + 1 if num_spec else 0 + if spec_tokens: + qkv = torch.cat([qkv[:spec_tokens], qkv]) + g = torch.cat([g[:, :spec_tokens], g], dim=1) + b = torch.cat([b[:, :spec_tokens], b], dim=1) + metadata.num_spec_decodes = 1 + metadata.num_spec_decode_tokens = spec_tokens + metadata.num_actual_tokens += spec_tokens + metadata.spec_sequence_masks = torch.tensor( + [True, False, False], device=device + ) + metadata.spec_query_start_loc = torch.tensor( + [0, spec_tokens], dtype=torch.int32, device=device + ) + metadata.spec_state_indices_tensor = torch.arange( + 5, num_slots, dtype=torch.int32, device=device + ).unsqueeze(0) + metadata.spec_token_indx = torch.arange(spec_tokens, device=device) + metadata.non_spec_token_indx = torch.arange( + spec_tokens, qkv.shape[0], device=device + ) + metadata.num_accepted_tokens = torch.ones( + 1, dtype=torch.int32, device=device + ) + out = torch.empty( + 1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16 + ) + layer._forward(qkv, g, b, out) + return out[:, spec_tokens:] + + checkpoint = MambaPrefillCheckpointMetadata( + torch.tensor([offset, 0], dtype=torch.int32, device=device), + torch.tensor([3, 0], dtype=torch.int32, device=device), + ) + expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint) + expected_state = recurrent[1].clone() + saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone() + torch.testing.assert_close(raw_qkv, raw_before) + conv_view = conv if dim_first else conv.transpose(-1, -2) + torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T) + assert torch.count_nonzero(conv_view[3, :, 3:]) == 0 + assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0 + + run( + torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]), + torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1), + torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1), + [1, 2], + [False, False], + ) + torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03) + + suffix = slice(offset, length) + qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]]) + g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1) + b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1) + conv[4].copy_(saved_conv) + recurrent[4].copy_(saved_recurrent) + actual = run(qkv, g, b, [3, 4], [True, True]) + torch.testing.assert_close( + actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03 + ) + torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index 44dee49a4e..0256a13141 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -367,3 +367,66 @@ def test_cudagraph_capture_batch_stays_decode_only(): assert staged is not None assert staged.data_ptr() == builder.non_spec_state_indices_tensor.data_ptr() torch.testing.assert_close(staged, common_attn_metadata.block_table_tensor[:, 0]) + + +@pytest.mark.parametrize("num_spec", [0, 3]) +@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)]) +def test_checkpoint_metadata_preserves_non_spec_order( + num_spec, prefix_match_unit, expected_offset +): + """Only eligible non-spec rows get a checkpoint in their reserved page.""" + from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + + vllm_config = create_vllm_config( + model_name="Qwen/Qwen3.5-0.8B", + block_size=BLOCK_SIZE, + ) + if num_spec > 0: + vllm_config.speculative_config = SpeculativeConfig( + method="ngram", + num_speculative_tokens=num_spec, + ) + vllm_config.cache_config.mamba_cache_mode = "align" + vllm_config.cache_config.prefix_match_unit = prefix_match_unit + mamba_spec = MambaSpec( + block_size=64, + shapes=((16, 64),), + dtypes=(torch.float16,), + mamba_cache_mode="align", + num_speculative_blocks=num_spec, + num_prefill_checkpoint_blocks=1, + prefill_checkpoint_alignment=16, + ) + builder = GDNAttentionMetadataBuilder( + kv_cache_spec=mamba_spec, + layer_names=["layer.0"], + vllm_config=vllm_config, + device=DEVICE, + ) + batch = BatchSpec( + seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99] + ) + common = create_common_attn_metadata(batch, 64, DEVICE, arange_block_indices=True) + table = common.block_table_tensor + 1 + if num_spec: + table = torch.cat([table, torch.zeros(3, num_spec, dtype=table.dtype)], dim=1) + common = common.replace(block_table_tensor=table) + kwargs = {} + if num_spec: + kwargs = dict( + num_decode_draft_tokens_cpu=torch.tensor([2, -1, -1], dtype=torch.int32), + num_accepted_tokens=torch.ones(3, dtype=torch.int32), + ) + + actual = builder.build(0, common, **kwargs) + assert actual.checkpoint is not None + offsets = [expected_offset, 0] if num_spec else [0, expected_offset, 0] + slots = [common.block_table_tensor[1, 0].item(), NULL_BLOCK_ID] + if not num_spec: + slots.insert(0, NULL_BLOCK_ID) + torch.testing.assert_close( + actual.checkpoint.checkpoint_offsets, torch.tensor(offsets, dtype=torch.int32) + ) + torch.testing.assert_close( + actual.checkpoint.state_indices, torch.tensor(slots, dtype=torch.int32) + ) diff --git a/vllm/model_executor/layers/mamba/checkpoint.py b/vllm/model_executor/layers/mamba/checkpoint.py index 3ba4062079..1f4bf9dc37 100644 --- a/vllm/model_executor/layers/mamba/checkpoint.py +++ b/vllm/model_executor/layers/mamba/checkpoint.py @@ -76,6 +76,8 @@ class MambaPrefillCheckpointBuilder: m: CommonAttentionMetadata, request_rows: list[int], ) -> MambaPrefillCheckpointMetadata | None: + if self.vllm_config.cache_config.mamba_cache_mode != "align": + return None if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0: return None assert m.seq_lens_cpu_upper_bound is not None diff --git a/vllm/models/glm5next/common/kda.py b/vllm/models/glm5next/common/kda.py index 436b9685cb..c67ac5b012 100644 --- a/vllm/models/glm5next/common/kda.py +++ b/vllm/models/glm5next/common/kda.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """GLM-5.3-Flash KDA layer with separate convolutions and a bounded safe gate.""" +from dataclasses import replace + import torch from torch import nn @@ -14,7 +16,14 @@ from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, RowParallelLinear, ) +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, +) from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention +from vllm.model_executor.layers.mamba.kda_checkpoint import ( + FlashKDAPrefillCheckpointExporter, + kda_prefill_checkpoint_alignment, +) from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, MambaStateShapeCalculator, @@ -37,6 +46,7 @@ from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata +from vllm.v1.kv_cache_interface import MambaSpec from vllm.v1.worker.workspace import current_workspace_manager if current_platform.is_rocm(): @@ -155,6 +165,16 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): num_heads: int conv_size: int + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec: + spec = super().get_kv_cache_spec(vllm_config) + assert isinstance(spec, MambaSpec) + alignment = kda_prefill_checkpoint_alignment(self.kda_prefill_backend) + return replace( + spec, + num_prefill_checkpoint_blocks=int(alignment is not None), + prefill_checkpoint_alignment=alignment, + ) + def get_state_dtype( self, ) -> tuple[torch.dtype, torch.dtype]: @@ -330,6 +350,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): self._flashkda_buffer_specs: ( tuple[tuple[tuple[int, ...], torch.dtype], ...] | None ) = None + self._checkpoint_exporter: FlashKDAPrefillCheckpointExporter | None = None if self.kda_prefill_backend == "flashkda": import vllm._flashkda_C # noqa: F401 @@ -339,6 +360,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): max_tokens, self.local_num_heads, max_seqs ) self._flashkda_buffer_specs = ( + ( + (max_seqs, self.local_num_heads, self.head_dim, self.head_dim), + self.get_state_dtype()[1], + ), ( (max_seqs, self.local_num_heads, self.head_dim, self.head_dim), self.get_state_dtype()[1], @@ -351,6 +376,9 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): vllm_config.model_config.dtype, ), ) + self._checkpoint_exporter = FlashKDAPrefillCheckpointExporter( + state_len=self.conv_size - 1 + ) def _flashkda_prefill( self, @@ -362,6 +390,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): initial_state: torch.Tensor, cu_seqlens: torch.Tensor, out: torch.Tensor | None, + checkpoint: MambaPrefillCheckpointMetadata | None = None, + raw_qkv: torch.Tensor | None = None, + conv_state: torch.Tensor | None = None, + recurrent_state: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits ``g`` and raw ``beta`` logits, l2-normalizes q/k in-kernel and applies the @@ -370,10 +402,15 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): attention output into ``out`` (a workspace buffer when ``None``) and returns ``(out, final_state)``.""" assert self._flashkda_buffer_specs is not None - final_state, workspace, workspace_out = ( + final_state, checkpoint_state, workspace, workspace_out = ( current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs) ) final_state = final_state[: initial_state.shape[0]] + checkpoint_state = ( + checkpoint_state[: initial_state.shape[0]] + if checkpoint is not None + else None + ) if out is None: out = workspace_out[:, : q.shape[1]] # FlashKDA hardcodes dense q/k/v/g strides; beta may be row-strided. @@ -392,9 +429,23 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): initial_state.contiguous(), final_state, cu_seqlens.contiguous(), - None, - None, + checkpoint_state, + checkpoint.checkpoint_offsets.contiguous() + if checkpoint is not None + else None, ) + if checkpoint is not None: + assert self._checkpoint_exporter is not None + assert raw_qkv is not None and conv_state is not None + assert recurrent_state is not None + self._checkpoint_exporter.export( + checkpoint, + raw_qkv=raw_qkv, + conv_state=conv_state, + recurrent_checkpoint=checkpoint_state, + recurrent_state=recurrent_state, + cu_seqlens=cu_seqlens, + ) return out, final_state def forward( @@ -565,6 +616,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1) # --- causal conv1d: non-spec path (prefill or plain decode) --- + raw_qkv_ns = qkv_ns q_ns = k_ns = v_ns = None if attn_metadata_narrowed.num_prefills > 0: assert qkv_ns is not None @@ -662,6 +714,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): initial_state=initial_state, cu_seqlens=non_spec_query_start_loc, out=ns_out, + checkpoint=attn_metadata_narrowed.checkpoint, + raw_qkv=raw_qkv_ns, + conv_state=conv_state, + recurrent_state=recurrent_state, ) else: ( diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index 2562171109..fe50bd4f53 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -18,10 +18,6 @@ from typing import TYPE_CHECKING import torch from vllm.config import VllmConfig -from vllm.model_executor.layers.mamba.checkpoint import ( - MambaPrefillCheckpointBuilder, - MambaPrefillCheckpointMetadata, -) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import async_tensor_h2d @@ -271,7 +267,6 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata): recoverssm_context: "KDARecoverSSMCommitContext | None" = field( default=None, repr=False, compare=False ) - checkpoint: MambaPrefillCheckpointMetadata | None = None def commit_recoverssm_state( self, num_accepted_tokens: torch.Tensor @@ -319,9 +314,6 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): device: torch.device, ) -> None: super().__init__(kv_cache_spec, layer_names, vllm_config, device) - self.checkpoint_builder = MambaPrefillCheckpointBuilder( - vllm_config, kv_cache_spec - ) additional_config = vllm_config.additional_config self.use_flashinfer_prefill = ( isinstance(additional_config, dict) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index ae8a9437d5..eb616644f3 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -8,6 +8,10 @@ from typing import Literal import torch from vllm.config import VllmConfig +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointBuilder, + MambaPrefillCheckpointMetadata, +) from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, @@ -48,6 +52,7 @@ class GDNAttentionMetadata: num_spec_decode_tokens: int num_actual_tokens: int + checkpoint: MambaPrefillCheckpointMetadata | None = None has_initial_state: torch.Tensor | None = None spec_query_start_loc: torch.Tensor | None = None # shape: [num_spec_decodes + 1,] @@ -97,6 +102,9 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] super().__init__(kv_cache_spec, layer_names, vllm_config, device) self.compilation_config = vllm_config.compilation_config self.speculative_config = vllm_config.speculative_config + self.checkpoint_builder = MambaPrefillCheckpointBuilder( + vllm_config, kv_cache_spec + ) from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( _resolve_gdn_prefill_backend, ) @@ -466,6 +474,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] else: has_initial_state = None + checkpoint: MambaPrefillCheckpointMetadata | None = None + if num_prefills > 0: + request_rows = list(range(m.num_reqs)) + if spec_sequence_masks_cpu is not None: + request_rows = (~spec_sequence_masks_cpu).nonzero().flatten().tolist() + checkpoint = self.checkpoint_builder.build(m, request_rows) + # Function code counted on either presency non-spec decode or spec decode, # but not both. assert not (num_decodes > 0 and num_spec_decodes > 0), ( @@ -543,6 +558,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] num_spec_decodes=num_spec_decodes, num_spec_decode_tokens=num_spec_decode_tokens, num_actual_tokens=m.num_actual_tokens, + checkpoint=checkpoint, has_initial_state=has_initial_state, chunk_indices=chunk_indices, chunk_offsets=chunk_offsets, From 4861833ae280cf56e7802e6f2adb797031b51449 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Tue, 29 Sep 2026 17:23:32 +0800 Subject: [PATCH 019/248] [Bugfix][Core] Allow AuxOutput reset after abort without running requests (#59060) Signed-off-by: aoshen02 Co-authored-by: OpenAI Codex --- tests/v1/core/test_scheduler.py | 8 +------- vllm/v1/core/sched/scheduler.py | 10 ---------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 91c93ed68f..d439a197e9 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -1562,13 +1562,7 @@ def test_scheduler_reset_prefix_cache(): assert not scheduler.reset_prefix_cache() scheduler.aux_output_connector.reset.assert_not_called() - with pytest.raises(RuntimeError, match=r"pause\(mode='keep'\)"): - scheduler.reset_prefix_cache(reset_running_requests=True) - - # pause(mode="keep") also waits for scheduled model outputs to drain. - scheduler.set_pause_state(PauseState.PAUSED_ALL) - with pytest.raises(RuntimeError, match="model output is in flight"): - scheduler.reset_prefix_cache(reset_running_requests=True) + # Pause completes pending model outputs before the caller resets the scheduler. for request in requests: request.num_in_flight_tokens = 0 diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index e4af907154..24ecc4534d 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -2781,16 +2781,6 @@ class Scheduler(SchedulerInterface): Otherwise, this method will only reset the KV prefix cache when there is no running requests taking KV cache. """ - if reset_running_requests and self.aux_output_connector is not None: - if self._pause_state != PauseState.PAUSED_ALL: - raise RuntimeError( - "AuxOutput Connector only supports resetting running requests " - "after pause(mode='keep')." - ) - if any(request.num_in_flight_tokens for request in self.requests.values()): - raise RuntimeError( - "AuxOutput Connector cannot reset while model output is in flight." - ) if reset_running_requests: # For logging. timestamp = time.monotonic() From 77e52645e9baba15d6b9cd7e09a12e70628b8237 Mon Sep 17 00:00:00 2001 From: "yhcheong (vllmellm)" Date: Tue, 29 Sep 2026 17:37:20 +0800 Subject: [PATCH 020/248] [Bugfix][Model] MiMo: keep fused fp8 qkv_proj pairing state across weight-loading calls (#58142) Signed-off-by: vllmellm --- vllm/model_executor/models/mimo_v2.py | 8 ++++++-- vllm/model_executor/models/mimo_v2_mtp.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index dfbed25655..6267a37dc0 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -742,8 +742,12 @@ class MiMoV2Model(nn.Module, EagleModelMixin): expert_params_mapping = self.get_expert_mapping() # Pro-format fused qkv_proj arrives as two tensors (weight and # weight_scale_inv). Store them per-layer so that they can be - # sharded together. - pending_fp8_qkv_proj: dict[str, dict[str, torch.Tensor]] = {} + # sharded together. The state must outlive this call: AutoWeightsLoader + # delegates per contiguous group of names, so a pair can straddle two + # calls and would otherwise be dropped silently. + pending_fp8_qkv_proj = getattr(self, "_pending_fp8_qkv_proj", None) + if pending_fp8_qkv_proj is None: + self._pending_fp8_qkv_proj = pending_fp8_qkv_proj = {} for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: continue diff --git a/vllm/model_executor/models/mimo_v2_mtp.py b/vllm/model_executor/models/mimo_v2_mtp.py index a4e46977ec..c413b65ba9 100644 --- a/vllm/model_executor/models/mimo_v2_mtp.py +++ b/vllm/model_executor/models/mimo_v2_mtp.py @@ -266,7 +266,12 @@ class MiMoV2MTP(nn.Module): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() - pending_qkv_proj: dict[str, dict[str, torch.Tensor]] = {} + # The pairing state must outlive this call: AutoWeightsLoader delegates + # per contiguous group of names, so a pair can straddle two calls and + # would otherwise be dropped silently. + pending_qkv_proj = getattr(self, "_pending_qkv_proj", None) + if pending_qkv_proj is None: + self._pending_qkv_proj = pending_qkv_proj = {} for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: From af5b4857e1353c01fd6bf41bc3cb9f84dc82dd89 Mon Sep 17 00:00:00 2001 From: siyu Date: Tue, 29 Sep 2026 17:37:43 +0800 Subject: [PATCH 021/248] [Fast Start] Support PP (#55477) Signed-off-by: Isotr0py Co-authored-by: Isotr0py Co-authored-by: Xun Sun --- .../model_loader/test_weight_cache.py | 36 +++++--- vllm/entrypoints/cli/preload.py | 22 +++-- .../model_loader/weight_cache/daemon.py | 84 +++++++++++-------- .../model_loader/weight_cache/ipc_loader.py | 4 + .../model_loader/weight_cache/protocol.py | 6 ++ 5 files changed, 100 insertions(+), 52 deletions(-) diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index e4ac86e0eb..7706522f6e 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -250,12 +250,12 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus(): from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1) - assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)] + assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)] dp_third_node = _parallel( data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8 ) - assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)] + assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)] dp_tp = _parallel( tensor_parallel_size=2, @@ -263,29 +263,43 @@ def test_daemon_places_tp_and_dp_ranks_on_local_gpus(): data_parallel_size_local=2, data_parallel_rank=2, ) - assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)] + assert plan_local_ranks(dp_tp) == [ + (0, 2, 0, 0), + (1, 2, 0, 1), + (2, 3, 0, 0), + (3, 3, 0, 1), + ] # DP + nnodes: node-local TP replicas dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2) - assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)] + assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)] dp_tp_node1 = _parallel( tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1 ) - assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)] + assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)] # DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes) tp_span_node1 = _parallel( tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1 ) - assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)] + assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)] tp_span_node2 = _parallel( tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2 ) - assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)] + assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)] # DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes) dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2) - assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)] + assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)] + + # PP adds one daemon placement per stage while preserving TP rank order. + pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2) + assert plan_local_ranks(pp_tp) == [ + (0, 0, 0, 0), + (1, 0, 0, 1), + (2, 0, 1, 0), + (3, 0, 1, 1), + ] def test_daemon_rejects_unmappable_parallelism(): @@ -302,8 +316,7 @@ def test_daemon_rejects_unmappable_parallelism(): _reject_unsupported_parallelism( _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2) ) - with pytest.raises(ValueError, match="pipeline"): - _reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2)) + _reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2)) with pytest.raises(ValueError, match="evenly divide"): _reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2)) with pytest.raises(ValueError, match="evenly divide"): @@ -335,6 +348,9 @@ def test_weight_cache_key_distinguishes_dp_ranks(): vllm_version="v", dp_size=16, dp_rank=3, + pp_size=2, + pp_rank=1, ) assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"] assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"] + assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"] diff --git a/vllm/entrypoints/cli/preload.py b/vllm/entrypoints/cli/preload.py index 50734a052b..9eab7421f1 100644 --- a/vllm/entrypoints/cli/preload.py +++ b/vllm/entrypoints/cli/preload.py @@ -170,6 +170,7 @@ class PreloadSubcommand(CLISubcommand): parallel_config = vllm_config.parallel_config _reject_unsupported_parallelism(parallel_config) tp_size = parallel_config.tensor_parallel_size + pp_size = parallel_config.pipeline_parallel_size dp_size = parallel_config.data_parallel_size placements = plan_local_ranks(parallel_config) @@ -230,16 +231,21 @@ class PreloadSubcommand(CLISubcommand): ctx = multiprocessing.get_context("spawn") ready_queue: multiprocessing.Queue[tuple[str, int]] = ctx.Queue() - # Local index == device index; global rank enumerates DP then TP. + # Local index == device index; global rank enumerates DP, PP, then TP. expected_ready = { - (format_daemon_role(is_draft), dp_rank * tp_size + tp_rank) - for (is_draft, _, _), (_, dp_rank, tp_rank) in product(groups, placements) + ( + format_daemon_role(is_draft), + dp_rank * pp_size * tp_size + pp_rank * tp_size + tp_rank, + ) + for (is_draft, _, _), (_, dp_rank, pp_rank, tp_rank) in product( + groups, placements + ) } procs = [ ctx.Process( target=_run_daemon, args=( - tp_rank, + dp_rank * pp_size * tp_size + pp_rank * tp_size + tp_rank, local_rank, config, init_method, @@ -247,13 +253,17 @@ class PreloadSubcommand(CLISubcommand): ready_queue, is_draft, dp_rank, + pp_rank, + ), + name=( + f"vllm-weight-cache-{format_daemon_role(is_draft)}-" + f"{(dp_rank * pp_size * tp_size + pp_rank * tp_size + tp_rank)}" ), - name=f"vllm-weight-cache-{format_daemon_role(is_draft)}-" - f"{dp_rank * tp_size + tp_rank}", ) for (is_draft, config, init_method), ( local_rank, dp_rank, + pp_rank, tp_rank, ) in product(groups, placements) ] diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index 561f693ae7..b56cbbea8f 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -17,8 +17,7 @@ Engines then load from the daemons with: vllm serve /path/to/model --tensor-parallel-size 4 \\ --load-format ipc_cache -Tensor, expert and data parallelism are supported; pipeline parallelism is -rejected at launch. +Tensor, expert, data and pipeline parallelism are supported. For multi-node tensor parallelism, run one launcher per node with a shared rendezvous so the global TP group forms across nodes (CUDA IPC handles are @@ -40,7 +39,8 @@ node-local, so each node serves only its local GPUs' shards). Reuse the same The global TP rank of local GPU ``i`` on node ``r`` is ``r * (tp_size // nnodes) + i``, matching vLLM's contiguous per-node rank assignment, so each engine worker maps its shard from the daemon on its own -node. +node. With pipeline parallelism each node additionally runs one daemon per PP +stage; the global rank is ``pp_rank * tp_size + tp_rank``. For data parallelism (e.g. a TP1 x DP16 x EP decode fleet) run one launcher per node with the engine's DP placement flags. Local GPU ``i`` serves DP rank @@ -168,27 +168,30 @@ def export_entries( class WeightCacheDaemon: - """Per-GPU process that loads one TP shard and serves CUDA IPC handles.""" + """Per-GPU process that loads one TP/PP shard and serves CUDA IPC handles.""" def __init__( self, vllm_config: VllmConfig, - tp_rank: int, + global_rank: int, local_rank: int, distributed_init_method: str, socket_dir: str | None = None, is_draft: bool = False, dp_rank: int = 0, + pp_rank: int = 0, ): parallel_config = vllm_config.parallel_config - if parallel_config.data_parallel_size > 1: + self.tp_size = parallel_config.tensor_parallel_size + self.pp_size = parallel_config.pipeline_parallel_size + self.dp_size = parallel_config.data_parallel_size + if self.dp_size > 1: # The engine sets these on each DP rank's workers; the MoE/EP # layers read them to place their expert shards. - tp_size = parallel_config.tensor_parallel_size if parallel_config.nnodes > 1: - world_size = parallel_config.data_parallel_size * tp_size + world_size = self.dp_size * self.tp_size local_world = world_size // parallel_config.nnodes - start_dp_rank = parallel_config.node_rank * local_world // tp_size + start_dp_rank = parallel_config.node_rank * local_world // self.tp_size else: start_dp_rank = parallel_config.data_parallel_rank parallel_config = replace( @@ -206,12 +209,12 @@ class WeightCacheDaemon: self.model_config = spec.draft_model_config else: self.model_config = vllm_config.model_config - self.tp_rank = tp_rank + self.global_rank = global_rank + rank_in_dp = global_rank - dp_rank * self.pp_size * self.tp_size + self.tp_rank = rank_in_dp % self.tp_size + self.pp_rank = rank_in_dp // self.tp_size self.dp_rank = dp_rank - self.tp_size = parallel_config.tensor_parallel_size - self.dp_size = parallel_config.data_parallel_size - self.global_rank = dp_rank * self.tp_size + tp_rank - self.world_size = self.dp_size * self.tp_size + self.world_size = self.dp_size * self.pp_size * self.tp_size self.local_rank = local_rank self.distributed_init_method = distributed_init_method self.socket_dir = socket_dir @@ -223,7 +226,9 @@ class WeightCacheDaemon: self.cache_config = WeightCacheKey.from_model_config( self.model_config, tp_size=self.tp_size, - tp_rank=tp_rank, + tp_rank=self.tp_rank, + pp_size=self.pp_size, + pp_rank=self.pp_rank, dp_size=self.dp_size, dp_rank=dp_rank, is_draft=is_draft, @@ -242,7 +247,7 @@ class WeightCacheDaemon: backend=current_platform.dist_backend, ) with set_current_vllm_config(self.vllm_config): - ensure_model_parallel_initialized(self.tp_size, 1) + ensure_model_parallel_initialized(self.tp_size, self.pp_size) self.model = self.get_model() logger.info( "Weight cache %s daemon rank %d loaded model", @@ -414,7 +419,7 @@ class WeightCacheDaemon: def _run_daemon( - tp_rank: int, + global_rank: int, local_rank: int, vllm_config: VllmConfig, distributed_init_method: str, @@ -422,15 +427,17 @@ def _run_daemon( ready_queue: "multiprocessing.Queue[tuple[str, int]]", is_draft: bool = False, dp_rank: int = 0, + pp_rank: int = 0, ) -> None: daemon = WeightCacheDaemon( vllm_config, - tp_rank, + global_rank, local_rank, distributed_init_method, socket_dir, is_draft, dp_rank, + pp_rank, ) daemon.load_model() daemon.serve_forever( @@ -438,26 +445,34 @@ def _run_daemon( ) -def plan_local_ranks(parallel_config: ParallelConfig) -> list[tuple[int, int, int]]: - """``(local_rank, dp_rank, tp_rank)`` for every GPU this launcher serves. +def plan_local_ranks( + parallel_config: ParallelConfig, +) -> list[tuple[int, int, int, int]]: + """``(local_rank, dp_rank, pp_rank, tp_rank)`` for every local daemon. - Global ranks enumerate DP then TP: ``global = dp_rank * tp_size + tp_rank``. + Global ranks enumerate DP, PP, then TP: ``global = dp_rank * pp_size * + tp_size + pp_rank * tp_size + tp_rank``. With ``--nnodes`` the engine hands each node a contiguous block of global ranks, so node ``r`` serves ``node_rank * local + i``; without it a launcher's block starts at ``--data-parallel-start-rank * tp_size``. """ tp_size = parallel_config.tensor_parallel_size - world_size = parallel_config.data_parallel_size * tp_size + pp_size = parallel_config.pipeline_parallel_size + world_size = parallel_config.data_parallel_size * pp_size * tp_size if parallel_config.nnodes > 1: local_world_size = world_size // parallel_config.nnodes base = parallel_config.node_rank * local_world_size else: - local_world_size = parallel_config.data_parallel_size_local * tp_size - base = parallel_config.data_parallel_rank * tp_size - return [ - (i, (base + i) // tp_size, (base + i) % tp_size) - for i in range(local_world_size) - ] + local_world_size = parallel_config.data_parallel_size_local * pp_size * tp_size + base = parallel_config.data_parallel_rank * pp_size * tp_size + placements = [] + for local_rank in range(local_world_size): + global_rank = base + local_rank + rank_in_dp = global_rank % (pp_size * tp_size) + dp_rank = global_rank // (pp_size * tp_size) + pp_rank, tp_rank = divmod(rank_in_dp, tp_size) + placements.append((local_rank, dp_rank, pp_rank, tp_rank)) + return placements def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None: @@ -469,21 +484,18 @@ def get_draft_daemon_config(vllm_config: VllmConfig) -> VllmConfig | None: def _reject_unsupported_parallelism(parallel_config: ParallelConfig) -> None: - """Reject pipeline parallelism and placements the daemon cannot map.""" - if parallel_config.pipeline_parallel_size > 1: - raise ValueError( - "The weight cache daemon only supports tensor, expert and data " - "parallelism; pipeline parallelism is not supported" - ) + """Reject placements the daemon cannot map.""" dp_size = parallel_config.data_parallel_size tp_size = parallel_config.tensor_parallel_size + pp_size = parallel_config.pipeline_parallel_size if parallel_config.nnodes > 1: - world_size = dp_size * tp_size + world_size = dp_size * pp_size * tp_size if world_size % parallel_config.nnodes != 0: raise ValueError( f"--nnodes ({parallel_config.nnodes}) must evenly divide the " f"daemon world size ({world_size} = data-parallel-size " - f"{dp_size} x tensor-parallel-size {tp_size})" + f"{dp_size} x pipeline-parallel-size {pp_size} x " + f"tensor-parallel-size {tp_size})" ) return if dp_size == 1: diff --git a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py index 508e95c83d..185c11b83a 100644 --- a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py +++ b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py @@ -15,6 +15,7 @@ from vllm.config import ModelConfig, VllmConfig from vllm.config.load import LoadConfig from vllm.distributed import ( get_dp_group, + get_pp_group, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, ) @@ -289,10 +290,13 @@ class IpcModelLoader(BaseModelLoader): def _fetch_entries(self, model_config: ModelConfig) -> WeightCacheState: dp_group = get_dp_group() + pp_group = get_pp_group() cache_config = WeightCacheKey.from_model_config( model_config, tp_size=get_tensor_model_parallel_world_size(), tp_rank=get_tensor_model_parallel_rank(), + pp_size=pp_group.world_size, + pp_rank=pp_group.rank_in_group, dp_size=dp_group.world_size, dp_rank=dp_group.rank_in_group, is_draft=self.is_draft, diff --git a/vllm/model_executor/model_loader/weight_cache/protocol.py b/vllm/model_executor/model_loader/weight_cache/protocol.py index 6b5bbc69e5..72053805cd 100644 --- a/vllm/model_executor/model_loader/weight_cache/protocol.py +++ b/vllm/model_executor/model_loader/weight_cache/protocol.py @@ -289,6 +289,8 @@ class WeightCacheKey: quant_config_hash: str revision: str | None vllm_version: str + pp_size: int = 1 + pp_rank: int = 0 is_draft: bool = False """Daemon group the weights come from; False is the target model.""" dp_size: int = 1 @@ -301,6 +303,8 @@ class WeightCacheKey: tp_size: int, tp_rank: int, *, + pp_size: int = 1, + pp_rank: int = 0, is_draft: bool = False, dp_size: int = 1, dp_rank: int = 0, @@ -325,6 +329,8 @@ class WeightCacheKey: model_arch=arch, tp_size=tp_size, tp_rank=tp_rank, + pp_size=pp_size, + pp_rank=pp_rank, dtype=str(model_config.dtype), quantization=model_config.quantization, quant_config_hash=_hash_quant_config(quant_config), From 208bb38c80984e29c03282d295d50325f5895338 Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Tue, 29 Sep 2026 03:11:53 -0700 Subject: [PATCH 022/248] [Bugfix][Rust Frontend] Fix startup with config-only model (#58643) Signed-off-by: louie-tsai --- .../test_api_server_process_manager.py | 46 +++++++++++++++++++ vllm/v1/utils.py | 10 ++++ 2 files changed, 56 insertions(+) diff --git a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py index 668f115448..80a2577dba 100644 --- a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py +++ b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py @@ -489,3 +489,49 @@ def test_rust_frontend_launch_log_redacts_credentials(monkeypatch, caplog): assert hf_token not in message assert api_key not in message assert '"hf_token": "***"' in message + + +def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_path): + """Config-only model selection must be forwarded to the Rust frontend.""" + import subprocess as subprocess_mod + + from vllm.entrypoints.launchers.cli_args import make_arg_parser + from vllm.utils.argparse_utils import FlexibleArgumentParser + from vllm.v1.utils import RustFrontendProcessManager + + config_path = tmp_path / "config.yml" + config_path.write_text("model: org/model\n") + args = make_arg_parser(FlexibleArgumentParser()).parse_args( + ["--config", str(config_path)] + ) + assert args.model == "org/model" + assert args.model_tag is None + + class _FakeProc: + pid = 4321 + returncode = 0 + + def poll(self): + return 0 + + def wait(self, timeout=None): + return 0 + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + monkeypatch.setattr(subprocess_mod, "Popen", lambda *a, **kw: _FakeProc()) + with caplog.at_level("INFO", logger="vllm.v1.utils"): + RustFrontendProcessManager( + binary_path="/nonexistent/vllm-rs", + sock=sock, + args=args, + input_address="ipc:///tmp/in", + output_address="ipc:///tmp/out", + engine_start_index=0, + engine_count=1, + data_parallel_size=1, + ) + finally: + sock.close() + + assert '"model_tag": "org/model"' in caplog.text diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index 81f4eaf842..f3bd97784e 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -391,6 +391,16 @@ class RustFrontendProcessManager: "data_parallel_hybrid_lb", }, ) + + # `model_tag` is the positional `vllm serve` model argument. When the + # model is supplied only through `--config`, argparse populates `model` + # while leaving `model_tag` unset. The Rust frontend requires + # `model_tag` in its JSON bootstrap payload, so use the resolved model + # as a fallback. + model_tag = getattr(args, "model_tag", None) or getattr(args, "model", None) + if model_tag is not None: + args_dict["model_tag"] = model_tag + # The Rust `frontend` subcommand parses --args-json via serde_json, # which bypasses clap and therefore ignores any `#[arg(env = ...)]` # declarations on SharedRuntimeArgs fields. Forward the env-driven From 3f3fbe286f25ead10f793dea0c549a99c196ad11 Mon Sep 17 00:00:00 2001 From: Reid <61492567+reidliu41@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:24:28 +0800 Subject: [PATCH 023/248] [Bugfix][Rust Frontend] Account for new requests in DP routing (#58956) Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: reidliu41 Signed-off-by: Bugen Zhao --- .../engine-core-client/src/client/state.rs | 155 +++++++++++++++++- 1 file changed, 148 insertions(+), 7 deletions(-) diff --git a/rust/src/engine-core-client/src/client/state.rs b/rust/src/engine-core-client/src/client/state.rs index 724ae5b64c..a8f0fa3c24 100644 --- a/rust/src/engine-core-client/src/client/state.rs +++ b/rust/src/engine-core-client/src/client/state.rs @@ -67,25 +67,40 @@ struct EngineRoutingState { inflight: usize, /// The latest real scheduler snapshot received from this engine, if any. last_scheduler_stats: Option, + /// Requests admitted since the last scheduler snapshot was received. + /// + /// Added to the snapshot so each admission raises the routing score before + /// the next snapshot arrives. Only tracked once a snapshot exists. + admitted_since_stats: usize, } impl EngineRoutingState { /// Compute the routing score used to pick the least-loaded engine. /// - /// Scheduler stats can raise the load estimate above the frontend-local - /// view, but they should not lower it below requests this frontend has - /// already admitted. + /// Scheduler stats, plus admissions not yet reflected in them, can raise + /// the load estimate above the frontend-local view, but they should not + /// lower it below requests this frontend has already admitted. fn routing_score(&self) -> usize { let Some(stats) = self.last_scheduler_stats else { return self.inflight; }; - self.inflight.max(stats.running + stats.waiting) + self.inflight.max(stats.running + stats.waiting + self.admitted_since_stats) } - /// Replace the local routing view with a fresh real scheduler snapshot. + /// Record one request admitted to this engine. + fn record_admission(&mut self) { + self.inflight += 1; + if self.last_scheduler_stats.is_some() { + self.admitted_since_stats += 1; + } + } + + /// Replace the local routing view with a fresh real scheduler snapshot, + /// resetting the admissions counted on top of the previous one. fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) { self.last_scheduler_stats = Some(next); + self.admitted_since_stats = 0; } } @@ -121,7 +136,7 @@ impl RequestRegistry { /// /// When `data_parallel_rank` is provided, the request is routed directly to /// the engine at that rank index, bypassing load balancing. Otherwise - /// the engine with the fewest in-flight requests is chosen. + /// the engine with the lowest routing score is chosen. pub fn register( &mut self, request_id: String, @@ -154,7 +169,7 @@ impl RequestRegistry { .routing_per_engine .get_mut(&engine_id) .expect("request registry must track all known engines"); - state.inflight += 1; + state.record_admission(); Ok((engine_id, rx)) } @@ -736,6 +751,7 @@ mod tests { let state = EngineRoutingState { inflight: 3, last_scheduler_stats: None, + ..Default::default() }; assert_eq!(state.routing_score(), 3); @@ -749,6 +765,7 @@ mod tests { waiting: 0, running: 2, }), + ..Default::default() }; assert_eq!(state.routing_score(), 7); @@ -762,6 +779,7 @@ mod tests { waiting: 3, running: 2, }), + ..Default::default() }; assert_eq!(state.routing_score(), 5); @@ -795,6 +813,129 @@ mod tests { assert_eq!(chosen, engine_1); } + #[test] + fn registry_spreads_bursts_after_cancellation_before_stats_refresh() { + let mut distributions = Vec::new(); + for counts in [[10, 12], [12, 12]] { + let mut registry = RequestRegistry::new( + &[0, 1].map(|rank| connected_engine(EngineId::from_engine_index(rank))), + ); + let mut old_requests = Vec::new(); + for (rank, running) in counts.into_iter().enumerate() { + for index in 0..running { + let request_id = format!("old-{rank}-{index}"); + registry.register(request_id.clone(), None, Some(rank as u32)).unwrap(); + old_requests.push(request_id); + } + registry.apply_scheduler_counts( + rank as u32, + EngineLoadSnapshot { + waiting: 0, + running, + }, + ); + } + drop(registry.abort_many(&old_requests, 0.0)); + + let mut distribution = [0; 2]; + for index in 0..14 { + let (engine, _) = registry.register(format!("new-{index}"), None, None).unwrap(); + distribution[engine.engine_index().unwrap() as usize] += 1; + } + distributions.push(distribution); + } + expect_test::expect!["[[8, 6], [7, 7]]"].assert_eq(&format!("{distributions:?}")); + } + + #[test] + fn registry_preserves_tie_order_across_finished_requests_and_stats_refreshes() { + let ranks = [2, 5, 9]; + let mut registry = RequestRegistry::new( + &ranks.map(|rank| connected_engine(EngineId::from_engine_index(rank))), + ); + let mut chosen = Vec::new(); + for index in 0..6 { + for rank in ranks { + registry.apply_scheduler_counts( + u32::from(rank), + EngineLoadSnapshot { + waiting: 0, + running: 0, + }, + ); + } + let request_id = format!("req-{index}"); + let (engine, _) = registry.register(request_id.clone(), None, None).unwrap(); + chosen.push(engine.engine_index().unwrap()); + drop(registry.finish_many(&[request_id])); + } + expect_test::expect!["[2, 2, 2, 2, 2, 2]"].assert_eq(&format!("{chosen:?}")); + } + + #[test] + fn registry_refreshes_estimates_without_erasing_inflight() { + let mut registry = RequestRegistry::new( + &[0, 1].map(|rank| connected_engine(EngineId::from_engine_index(rank))), + ); + let snapshot = EngineLoadSnapshot { + waiting: 0, + running: 10, + }; + registry.apply_scheduler_counts(0, snapshot); + registry.apply_scheduler_counts( + 1, + EngineLoadSnapshot { + waiting: 0, + running: 12, + }, + ); + for index in 0..4 { + registry.register(format!("pinned-{index}"), None, Some(0)).unwrap(); + } + + // Even unchanged counts replace the optimistic estimate for that engine. + registry.apply_scheduler_counts(0, snapshot); + let (after_refresh, _) = registry.register("after-refresh".into(), None, None).unwrap(); + // An older, empty snapshot must not erase the five admitted requests. + for rank in [0, 1] { + registry.apply_scheduler_counts( + rank, + EngineLoadSnapshot { + waiting: 0, + running: 0, + }, + ); + } + let (after_empty, _) = registry.register("after-empty".into(), None, None).unwrap(); + expect_test::expect!["(Some(0), Some(1))"].assert_eq(&format!( + "{:?}", + (after_refresh.engine_index(), after_empty.engine_index()) + )); + } + + #[test] + fn registry_explicit_admissions_raise_routing_score() { + let mut registry = RequestRegistry::new( + &[2, 5, 9].map(|rank| connected_engine(EngineId::from_engine_index(rank))), + ); + for rank in [2, 5, 9] { + registry.apply_scheduler_counts( + rank, + EngineLoadSnapshot { + waiting: 0, + running: 10, + }, + ); + } + registry.register("pinned".into(), None, Some(9)).unwrap(); + let mut chosen = Vec::new(); + for index in 0..3 { + let (engine, _) = registry.register(format!("auto-{index}"), None, None).unwrap(); + chosen.push(engine.engine_index().unwrap()); + } + expect_test::expect!["[2, 5, 2]"].assert_eq(&format!("{chosen:?}")); + } + #[test] fn register_with_data_parallel_rank_routes_to_specified_engine() { let engine_0 = EngineId::from_engine_index(0); From 05d89636fca80d806732ced636ef505f42e6fe35 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Tue, 29 Sep 2026 05:28:15 -0500 Subject: [PATCH 024/248] [ROCm][CI] Mirror generic GEMM-RS/AR on MI355 (#58433) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/test-amd.yaml | 31 +++- .buildkite/test_areas/distributed.yaml | 28 +++ .buildkite/test_areas/lm_eval.yaml | 4 +- tests/kernels/test_rocm_gemm_rs_ar.py | 230 +++++++++++++++++++++++++ 4 files changed, 289 insertions(+), 4 deletions(-) create mode 100644 tests/kernels/test_rocm_gemm_rs_ar.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 69dd0d8cff..9e4eaceb41 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3419,6 +3419,33 @@ steps: - pytest -v -s distributed/test_sharded_rdt_producer.py - pytest -v -s distributed/test_sharded_rdt_trainer.py +- label: ":amd: (MI355) GEMM-RS-AR" + timeout_in_minutes: 20 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - .buildkite/test-amd.yaml + - tests/kernels/test_gemm_rs_ar.py + - tests/kernels/test_rocm_gemm_rs_ar.py + - tests/conftest.py + - vllm/models/common/ops/sequence_parallel.py + - vllm/model_executor/layers/linear.py + - vllm/model_executor/layers/utils.py + - vllm/model_executor/layers/quantization/online/ + - vllm/model_executor/layers/quantization/utils/mxfp8_utils.py + - vllm/model_executor/kernels/linear/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - csrc/rocm/skinny_gemms.cu + - vllm/platforms/rocm.py + commands: + - pytest -v -s tests/kernels/test_rocm_gemm_rs_ar.py + - label: ":amd: (MI355) Distributed DP Basic" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -4533,7 +4560,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-r1-dp.txt -- label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP" +- label: ":amd: (MI355) LM Eval DeepSeek V3-2 TP" timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false @@ -4566,7 +4593,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi355-deepseek-v32-tp.txt -- label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP" +- label: ":amd: (MI355) LM Eval DeepSeek V3-2 DP" timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index a7f65417a6..7138e1e065 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -473,6 +473,34 @@ steps: - tests/kernels/test_gemm_rs_ar.py commands: - pytest -v -s tests/kernels/test_gemm_rs_ar.py + mirror: + amd: + label: ":amd: (MI355) GEMM-RS-AR" + dind: false + device: mi355_4 + num_devices: 4 + timeout_in_minutes: 20 + working_dir: /vllm-workspace/ + depends_on: + - image-build-amd + source_file_dependencies: + - .buildkite/test_areas/distributed.yaml + - tests/kernels/test_gemm_rs_ar.py + - tests/kernels/test_rocm_gemm_rs_ar.py + - tests/conftest.py + - vllm/models/common/ops/sequence_parallel.py + - vllm/model_executor/layers/linear.py + - vllm/model_executor/layers/utils.py + - vllm/model_executor/layers/quantization/online/ + - vllm/model_executor/layers/quantization/utils/mxfp8_utils.py + - vllm/model_executor/kernels/linear/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - vllm/_custom_ops.py + - csrc/rocm/skinny_gemms.cu + - vllm/platforms/rocm.py + commands: + - pytest -v -s tests/kernels/test_rocm_gemm_rs_ar.py - label: ":nvidia: (L4) Distributed 2-Node" key: 2-node-test-4-gpus diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index ce74f3a447..a23fcd0a98 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -578,7 +578,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-tp.txt mirror: amd: - label: ":amd: (MI355) LM Eval DeepSeek V3.2 TP" + label: ":amd: (MI355) LM Eval DeepSeek V3-2 TP" dind: false device: mi355_4 num_devices: 4 @@ -621,7 +621,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-dp.txt mirror: amd: - label: ":amd: (MI355) LM Eval DeepSeek V3.2 DP" + label: ":amd: (MI355) LM Eval DeepSeek V3-2 DP" dind: false device: mi355_4 num_devices: 4 diff --git a/tests/kernels/test_rocm_gemm_rs_ar.py b/tests/kernels/test_rocm_gemm_rs_ar.py new file mode 100644 index 0000000000..0a1540e842 --- /dev/null +++ b/tests/kernels/test_rocm_gemm_rs_ar.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ROCm projection/collective contract corresponding to the SM100 GEMM-RS/AR. + +Exercise the production RowParallelLinear and sequence-parallel reduction +paths: native BF16/MXFP8 GEMM, AITER all-reduce, and RCCL reduce-scatter. This +checks the same observable contract, not NVIDIA's fused GEMM/communication +implementation or its performance. +""" + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from tests.kernels.test_gemm_rs_ar import _N, _SHAPES +from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config +from vllm.config.quantization import QuantizationConfigArgs +from vllm.distributed import cleanup_dist_env_and_memory, get_tp_group +from vllm.distributed.parallel_state import ( + graph_capture, + init_distributed_environment, + initialize_model_parallel, +) +from vllm.model_executor.layers.linear import RowParallelLinear +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + _mxfp8_e4m3_quantize_torch, + dequant_mxfp8_to_bf16, +) +from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port +from vllm.utils.system_utils import update_environment_variables + +# Retain the expanded parent's shape transitions, plus fully padded RS ranks. +_ROCM_SHAPES = ((1, 512), (7, 768), *_SHAPES) + + +def _reference( + x: torch.Tensor, + layer: RowParallelLinear, + all_reduce: bool, +) -> torch.Tensor: + weight = layer.weight + if hasattr(layer, "weight_scale"): + # The production ROCm path uses the Triton quantizer and dot_scaled; + # use the independent torch quantizer and FP32 matmul as the oracle. + x_q, x_scale = _mxfp8_e4m3_quantize_torch(x) + x = dequant_mxfp8_to_bf16(x_q, x_scale) + weight = dequant_mxfp8_to_bf16(weight, layer.weight_scale) + partial = (x.float() @ weight.float().T).to(torch.bfloat16).float() + group = get_tp_group() + # Always use FP32 RCCL AR for the oracle, including the RS candidate. + dist.all_reduce(partial, group=group.device_group) + expected = partial.to(torch.bfloat16) + if all_reduce: + return expected + padding = -x.shape[0] % group.world_size + expected = torch.nn.functional.pad(expected, (0, 0, 0, padding)) + return expected.chunk(group.world_size)[group.rank_in_group] + + +def _apply(x: torch.Tensor, layer: RowParallelLinear) -> torch.Tensor: + output, _ = layer(x) + return output if layer.reduce_results else sp_reduce_scatter(output) + + +def _assert_output( + actual: torch.Tensor, expected: torch.Tensor, M: int, all_reduce: bool +) -> None: + assert actual.dtype == torch.bfloat16 + assert actual.shape == expected.shape + assert torch.isfinite(actual).all() + # Match the parent's tolerance at 0.1 weight scale, then bound total error. + # BF16 partial/reduction rounding can dominate near a cancelled output. + torch.testing.assert_close(actual, expected, rtol=5e-2, atol=0.4) + error = (actual.float() - expected.float()).norm() + assert error <= 5e-3 * expected.float().norm() + 1e-6 + if not all_reduce: + rank_start = get_tp_group().rank_in_group * actual.shape[0] + valid_rows = min(max(M - rank_start, 0), actual.shape[0]) + assert not actual[valid_rows:].any(), "RS padding must remain zero" + + +def _run_mode( + device: torch.device, backend: str, all_reduce: bool, world_size: int +) -> None: + group = get_tp_group() + generator = torch.Generator(device=device) + quant_config = ( + OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8")) + if backend == "mxfp8" + else None + ) + layers = {} + for K in sorted({K for _, K in _ROCM_SHAPES}): + with torch.device(device): + layer = RowParallelLinear( + K * world_size, + _N, + bias=False, + params_dtype=torch.bfloat16, + quant_config=quant_config, + reduce_results=all_reduce, + ) + generator.manual_seed(1000 + group.rank_in_group * 100 + K) + weight = ( + torch.randn(_N, K, dtype=torch.bfloat16, device=device, generator=generator) + * 0.1 + ) + layer.weight = torch.nn.Parameter(weight, requires_grad=False) + layer.quant_method.process_weights_after_loading(layer) + if backend == "mxfp8": + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import ( + RocmDotScaledMxfp8LinearKernel, + ) + + assert isinstance(layer.quant_method.kernel, RocmDotScaledMxfp8LinearKernel) + assert layer.weight.dtype == torch.float8_e4m3fn + layers[K] = layer + + def make_input(M: int, K: int, seed: int) -> torch.Tensor: + generator.manual_seed(seed + group.rank_in_group * 100) + return torch.randn( + M, K, dtype=torch.bfloat16, device=device, generator=generator + ) + + comm = group.device_communicator + assert comm is not None + assert comm.use_aiter_allreduce + assert comm.aiter_ar_comm is not None and not comm.aiter_ar_comm.disabled + assert comm.qr_comm is None or comm.qr_comm.disabled + assert comm.pynccl_comm is not None and not comm.pynccl_comm.disabled + for M, K in (*_ROCM_SHAPES, *_ROCM_SHAPES[::-1]): + x = make_input(M, K, 2000 + M + K) + if all_reduce: + probe = torch.empty(M, _N, dtype=x.dtype, device=device) + assert comm.aiter_ar_comm.should_custom_ar(probe) + expected = _reference(x, layers[K], all_reduce) + actual = _apply(x, layers[K]) + _assert_output(actual, expected, M, all_reduce) + + # Retained outputs must survive the next call's collective buffer reuse. + x = make_input(257, 768, 2500) + first = _apply(x, layers[768]) + snapshot = first.clone() + second = _apply(-x, layers[768]) + torch.accelerator.synchronize(device) + assert first.data_ptr() != second.data_ptr() + torch.testing.assert_close(first, snapshot, rtol=0, atol=0) + + M, K = 1025, 4224 + graph_x = make_input(M, K, 3000) + for _ in range(3): + _apply(graph_x, layers[K]) + torch.accelerator.synchronize(device) + dist.barrier(group=group.device_group) + graph = torch.cuda.CUDAGraph() + # vLLM's context registers the AITER IPC addresses used by graph replay. + with ( + graph_capture(device=device) as context, + torch.cuda.graph(graph, stream=context.stream), + ): + graph_output = _apply(graph_x, layers[K]) + torch.cuda.current_stream().wait_stream(context.stream) + dist.barrier(group=group.device_group) + for replay in range(3): + graph_x.copy_(make_input(M, K, 3001 + replay)) + expected = _reference(graph_x, layers[K], all_reduce) + graph.replay() + torch.accelerator.synchronize(device) + _assert_output(graph_output, expected, M, all_reduce) + dist.barrier(group=group.device_group) + + +def _worker(rank: int, world_size: int, port: int, backend: str) -> None: + device = torch.device("cuda", rank) + torch.accelerator.set_device_index(device) + update_environment_variables( + { + "RANK": str(rank), + "LOCAL_RANK": str(rank), + "WORLD_SIZE": str(world_size), + "MASTER_ADDR": "localhost", + "MASTER_PORT": str(port), + } + ) + config = VllmConfig() + config.model_config = ModelConfig(dtype="bfloat16") + try: + with set_current_vllm_config(config), torch.inference_mode(): + init_distributed_environment() + initialize_model_parallel(tensor_model_parallel_size=world_size) + for all_reduce in (False, True): + _run_mode(device, backend, all_reduce, world_size) + finally: + cleanup_dist_env_and_memory() + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm GEMM-RS/AR contract") +@pytest.mark.parametrize("world_size", [2, 4]) +@pytest.mark.parametrize( + "backend", + [ + "bf16", + pytest.param( + "mxfp8", + marks=pytest.mark.skipif( + not current_platform.supports_mx(), reason="native MXFP8 requires CDNA4" + ), + ), + ], +) +@pytest.mark.distributed(num_gpus=4) +def test_rocm_gemm_rs_ar( + monkeypatch: pytest.MonkeyPatch, world_size: int, backend: str +) -> None: + """Preserve rank reduction, tails, shape reuse, lifetime, and graph replay.""" + if torch.accelerator.device_count() < world_size: + pytest.skip(f"GEMM-RS/AR requires {world_size} GPUs") + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1") + monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE") + try: + mp.spawn( + _worker, args=(world_size, get_open_port(), backend), nprocs=world_size + ) + finally: + cleanup_dist_env_and_memory() From 998490cd9f297381288eb631bc98a4dfe1304f57 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:16:33 +0100 Subject: [PATCH 025/248] [Docs] Speed up docs build ~5x (#58705) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- docs/mkdocs/overrides/partials/nav.html | 34 +++++++++++++ mkdocs.yaml | 10 ---- requirements/docs.in | 1 - requirements/docs.txt | 66 +++++++++++-------------- 4 files changed, 63 insertions(+), 48 deletions(-) create mode 100644 docs/mkdocs/overrides/partials/nav.html diff --git a/docs/mkdocs/overrides/partials/nav.html b/docs/mkdocs/overrides/partials/nav.html new file mode 100644 index 0000000000..20a5b1fc58 --- /dev/null +++ b/docs/mkdocs/overrides/partials/nav.html @@ -0,0 +1,34 @@ + +{% import "partials/nav-item.html" as item with context %} +{% set features = features + ["navigation.prune"] %} +{% import "partials/nav-item.html" as pruned_item with context %} +{% set class = "md-nav md-nav--primary" %} +{% if "navigation.tabs" in features %} + {% set class = class ~ " md-nav--lifted" %} +{% endif %} +{% if "toc.integrate" in features %} + {% set class = class ~ " md-nav--integrated" %} +{% endif %} + diff --git a/mkdocs.yaml b/mkdocs.yaml index f3b9deab78..8c712257ee 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -65,13 +65,6 @@ plugins: - awesome-nav - glightbox - git-revision-date-localized - - minify: - minify_html: true - minify_js: true - minify_css: true - cache_safe: true - js_files: [docs/mkdocs/javascript/*.js] - css_files: [docs/mkdocs/stylesheets/*.css] # For API reference generation - api-autonav: modules: ["vllm"] @@ -98,9 +91,6 @@ plugins: summary: true inventories: - https://docs.python.org/3/objects.inv - - https://typing-extensions.readthedocs.io/en/latest/objects.inv - - https://docs.aiohttp.org/en/stable/objects.inv - - https://pillow.readthedocs.io/en/stable/objects.inv - https://numpy.org/doc/stable/objects.inv - https://pytorch.org/docs/stable/objects.inv - redirects: diff --git a/requirements/docs.in b/requirements/docs.in index 952e7c09ba..baabfa0331 100644 --- a/requirements/docs.in +++ b/requirements/docs.in @@ -6,7 +6,6 @@ mkdocs-gen-files mkdocs-awesome-nav mkdocs-glightbox mkdocs-git-revision-date-localized-plugin -mkdocs-minify-plugin mkdocs-redirects regex ruff diff --git a/requirements/docs.txt b/requirements/docs.txt index 5331a3b79b..efc92e94b5 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,40 +1,36 @@ # This file was autogenerated by uv via the following command: # uv pip compile requirements/docs.in -o requirements/docs.txt --python-platform x86_64-manylinux_2_28 --python-version 3.12 -annotated-types==0.7.0 +annotated-types==0.8.0 # via pydantic babel==2.18.0 # via # mkdocs-git-revision-date-localized-plugin # mkdocs-material -backrefs==7.0 +backrefs==8.0 # via mkdocs-material -bracex==2.6 +bracex==3.0.1 # via wcmatch -certifi==2026.4.22 +certifi==2026.7.22 # via requests -charset-normalizer==3.4.7 +charset-normalizer==3.5.1 # via requests -click==8.3.3 +click==8.5.0 # via # mkdocs # properdocs colorama==0.4.6 # via mkdocs-material -csscompressor==0.9.5 - # via mkdocs-minify-plugin ghp-import==2.1.0 # via # mkdocs # properdocs gitdb==4.0.12 # via gitpython -gitpython==3.1.50 +gitpython==3.1.62 # via mkdocs-git-revision-date-localized-plugin -griffelib==2.0.2 +griffelib==2.3.0 # via mkdocstrings-python -htmlmin2==0.1.13 - # via mkdocs-minify-plugin -idna==3.13 +idna==3.20 # via requests jinja2==3.1.6 # via @@ -42,9 +38,7 @@ jinja2==3.1.6 # mkdocs-material # mkdocstrings # properdocs -jsmin==3.0.1 - # via mkdocs-minify-plugin -markdown==3.10.2 +markdown==3.10.3 # via # mkdocs # mkdocs-autorefs @@ -72,7 +66,6 @@ mkdocs==1.6.1 # mkdocs-gen-files # mkdocs-git-revision-date-localized-plugin # mkdocs-material - # mkdocs-minify-plugin # mkdocs-redirects # mkdocstrings mkdocs-api-autonav==0.4.0 @@ -87,21 +80,19 @@ mkdocs-gen-files==0.6.1 # via -r requirements/docs.in mkdocs-get-deps==0.2.2 # via mkdocs -mkdocs-git-revision-date-localized-plugin==1.5.1 +mkdocs-git-revision-date-localized-plugin==1.6.0 # via -r requirements/docs.in mkdocs-glightbox==0.5.2 # via -r requirements/docs.in -mkdocs-material==9.7.6 +mkdocs-material==9.7.7 # via -r requirements/docs.in mkdocs-material-extensions==1.3.1 # via mkdocs-material -mkdocs-minify-plugin==0.8.0 - # via -r requirements/docs.in mkdocs-redirects==1.2.3 # via -r requirements/docs.in -mkdocstrings==1.0.4 +mkdocstrings==1.0.6 # via mkdocstrings-python -mkdocstrings-python==2.0.3 +mkdocstrings-python==2.0.8 # via # -r requirements/docs.in # mkdocs-api-autonav @@ -109,9 +100,10 @@ msgspec==0.21.1 # via -r requirements/docs.in natsort==8.4.0 # via mkdocs-awesome-nav -packaging==26.2 +packaging==26.3 # via # mkdocs + # mkdocs-git-revision-date-localized-plugin # properdocs paginate==0.5.7 # via mkdocs-material @@ -119,7 +111,7 @@ pathspec==1.1.1 # via # mkdocs # properdocs -platformdirs==4.9.6 +platformdirs==4.11.12 # via # mkdocs-get-deps # properdocs @@ -127,15 +119,15 @@ properdocs==1.6.7 # via # mkdocs-gen-files # mkdocs-redirects -pydantic==2.13.4 +pydantic==2.13.5 # via # -r requirements/docs.in # mkdocs-awesome-nav -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic -pygments==2.20.0 +pygments==2.21.0 # via mkdocs-material -pymdown-extensions==10.21.2 +pymdown-extensions==12.0.1 # via # mkdocs-material # mkdocstrings @@ -153,30 +145,30 @@ pyyaml-env-tag==1.1 # via # mkdocs # properdocs -regex==2026.4.4 +regex==2026.9.10 # via -r requirements/docs.in -requests==2.33.1 +requests==2.34.2 # via mkdocs-material -ruff==0.15.12 +ruff==0.16.8 # via -r requirements/docs.in -selectolax==0.4.8 +selectolax==0.4.12 # via mkdocs-glightbox six==1.17.0 # via python-dateutil smmap==5.0.3 # via gitdb -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # pydantic # pydantic-core # typing-inspection -typing-inspection==0.4.2 +typing-inspection==0.4.4 # via pydantic -urllib3==2.6.3 +urllib3==2.8.0 # via requests watchdog==6.0.0 # via # mkdocs # properdocs -wcmatch==10.1 +wcmatch==11.0.1 # via mkdocs-awesome-nav From 79a0307d7463b11938c30e924c606509601b22aa Mon Sep 17 00:00:00 2001 From: jack Date: Tue, 29 Sep 2026 20:42:02 +0800 Subject: [PATCH 026/248] [Config] Remove Engram CUDA-alike device restrictions (#59171) Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> --- tests/test_config.py | 4 ++-- vllm/config/engram.py | 6 +----- vllm/config/vllm.py | 6 +----- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index bd6b312094..39798acb05 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1588,14 +1588,14 @@ def test_engram_dp_shared_memory_config_validation( ("DeepseekV41ForCausalLM", [1], "cuda", True), ("DeepseekV41ForCausalLM", [1], "rocm", True), ("DeepseekV41ForCausalLM", [], "cuda", False), - ("DeepseekV41ForCausalLM", [1], "cpu", False), + ("DeepseekV41ForCausalLM", [1], "cpu", True), ("Qwen4ExpForCausalLM", [1], "cuda", True), ("Qwen4ExpForCausalLM", [1], "rocm", True), ("Qwen4ExpForConditionalGeneration", [1], "cuda", True), ("Qwen4ExpForConditionalGeneration", [1], "rocm", True), ("Qwen4ExpForCausalLM", [], "cuda", False), ("Qwen4ExpForCausalLM", None, "cuda", False), - ("Qwen4ExpForCausalLM", [1], "cpu", False), + ("Qwen4ExpForCausalLM", [1], "cpu", True), ("LlamaForCausalLM", [1], "cuda", False), ("Qwen4ExpMTP", [], "cuda", False), (None, None, "cuda", False), diff --git a/vllm/config/engram.py b/vllm/config/engram.py index 2539475296..fd95bc2acd 100644 --- a/vllm/config/engram.py +++ b/vllm/config/engram.py @@ -67,8 +67,6 @@ class EngramConfig: def verify_model_config(self, model_config: "ModelConfig | None") -> None: """Reject Engram configuration for models without n-gram embeddings.""" - from vllm.platforms import current_platform - field = ( _NGRAM_LAYER_FIELDS.get(model_config.architecture) if model_config is not None @@ -77,13 +75,11 @@ class EngramConfig: if ( model_config is None or field is None - or not current_platform.is_cuda_alike() or not getattr(model_config.hf_text_config, field, None) ): raise ValueError( "EngramConfig requires a model with supported Engram " - "embeddings, non-empty n-gram layer ids, and a CUDA-alike " - "device (CUDA or ROCm)." + "embeddings and non-empty n-gram layer ids." ) def resolve_dp_shared_memory(self, parallel_config: "ParallelConfig") -> None: diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 8022cd0b88..28764e1fcc 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1370,8 +1370,6 @@ class VllmConfig: def _resolve_and_verify_engram_config(self) -> None: """Resolve defaults and validate n-gram embedding settings.""" - from vllm.platforms import current_platform - model_config = self.model_config speculative_config = self.speculative_config # Draft configs inherit the target's communication groups and settings. @@ -1392,9 +1390,7 @@ class VllmConfig: "Disable --enable-dbo and set --ubatch-size to 0." ) if self.engram_config is None: - if not current_platform.is_cuda_alike() or not model_has_engram_layers( - model_config - ): + if not model_has_engram_layers(model_config): return self.engram_config = EngramConfig() self.engram_config.verify_model_config(model_config) From dfc8e0f3e2ad7791ba32e2e88053cda064052208 Mon Sep 17 00:00:00 2001 From: Yixin Dong Date: Tue, 29 Sep 2026 08:47:47 -0400 Subject: [PATCH 027/248] [Frontend] Support strict MiMo-V2.6 tool calling (#58019) Co-authored-by: Codex Co-authored-by: yuchuan Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: Ubospica Signed-off-by: yuchuan Signed-off-by: Bugen Zhao --- docs/features/tool_calling.md | 11 +++ requirements/common.txt | 2 +- requirements/rocm.txt | 2 +- requirements/test/cpu.txt | 2 +- requirements/test/cuda.txt | 2 +- requirements/test/rocm.txt | 4 +- requirements/test/xpu.txt | 2 +- tests/parser/engine/test_qwen3.py | 24 ++++++ tests/parser/engine/trace_builder.py | 42 +++++++++- .../test_structural_tag_registry.py | 79 ++++++++++++++++++- vllm/parser/engine/registered_adapters.py | 6 ++ vllm/parser/mimo.py | 42 ++++++++++ vllm/reasoning/__init__.py | 4 +- .../reasoning/mimo_engine_reasoning_parser.py | 6 ++ vllm/tool_parsers/__init__.py | 4 +- vllm/tool_parsers/mimo_tool_parser.py | 25 ++++++ vllm/tool_parsers/structural_tag_registry.py | 1 + 17 files changed, 243 insertions(+), 15 deletions(-) create mode 100644 vllm/parser/mimo.py create mode 100644 vllm/reasoning/mimo_engine_reasoning_parser.py create mode 100644 vllm/tool_parsers/mimo_tool_parser.py diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index fb70c50248..6527eccd09 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -458,6 +458,17 @@ Supported models: Flags: `--tool-call-parser qwen3_xml` +### MiMo-V2.6 Models (`mimo`) + +Supported models: + +* `XiaomiMiMo/MiMo-V2.6-Pro-RL` +* `XiaomiMiMo/MiMo-V2.6-Flash-RL` + +Use the checkpoint's chat template with `--tool-call-parser mimo --reasoning-parser mimo --enable-auto-tool-choice`. +For schema-constrained tool arguments, set `strict: true` inside each tool's `function`, or use `--tool-strict-level parameter`. +The parser supports `auto`, `required`, named tools, and `parallel_tool_calls=False`, preserving leading/trailing whitespace in string arguments. + ### Olmo 3 Models (`olmo3`) Olmo 3 models output tool calls in a format that is very similar to the one expected by the `pythonic` parser (see below), with a few differences. Each tool call is a pythonic string, but the parallel tool calls are newline-delimited, and the calls are wrapped within XML tags as `..`. In addition, the parser also allows JSON boolean and null literals (`true`, `false`, and `null`) in addition to the pythonic ones (`True`, `False`, and `None`). diff --git a/requirements/common.txt b/requirements/common.txt index ab0060284c..1958173fd7 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -26,7 +26,7 @@ lm-format-enforcer == 0.11.3 llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x" outlines_core == 0.2.14 lark == 1.2.2 -xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" +xgrammar == 0.2.8; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" typing_extensions >= 4.10 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317 partial-json-parser # used for parsing partial JSON outputs diff --git a/requirements/rocm.txt b/requirements/rocm.txt index 824e2e1201..85b196e401 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -24,7 +24,7 @@ timm>=1.0.17 amd-quark==0.12.post1 tilelang==0.1.10 # Required apache-tvm-ffi matching tilelang version -apache-tvm-ffi==0.1.10 +apache-tvm-ffi==0.1.11 # Required for faster safetensors model loading fastsafetensors >= 0.3.3 # Mooncake transfer engine for KV cache offloading/transfer (ROCm build, published on PyPI) diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index ab3e42886c..e8dd282121 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -1319,7 +1319,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.7 +xgrammar==0.2.8 # via -r requirements/test/../common.txt xxhash==3.5.0 # via diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index ffbc639c4c..b7711a10cf 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1429,7 +1429,7 @@ word2number==1.1 # via lm-eval wrapt==1.17.2 # via smart-open -xgrammar==0.2.7 +xgrammar==0.2.8 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 36e1ba182d..3581e5fea9 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -43,7 +43,7 @@ anyio==4.13.0 # sse-starlette # starlette # watchfiles -apache-tvm-ffi==0.1.10 +apache-tvm-ffi==0.1.11 # via # -c requirements/rocm.txt # tilelang @@ -1380,7 +1380,7 @@ word2number==1.1 # via lm-eval wrapt==2.1.2 # via smart-open -xgrammar==0.2.7 +xgrammar==0.2.8 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index e5f898adcc..2bcd3664c1 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -1218,7 +1218,7 @@ werkzeug==3.1.8 # via schemathesis word2number==1.1 # via lm-eval -xgrammar==0.2.7 +xgrammar==0.2.8 # via # -c requirements/common.txt # -r requirements/test/../common.txt diff --git a/tests/parser/engine/test_qwen3.py b/tests/parser/engine/test_qwen3.py index 07a025876a..7bdddb2faf 100644 --- a/tests/parser/engine/test_qwen3.py +++ b/tests/parser/engine/test_qwen3.py @@ -1175,3 +1175,27 @@ class TestNestedSchemaCoercion: assert questions[0]["question"] == "Pick a color" assert questions[0]["multiSelect"] is False assert questions[0]["answer"] is None + + +@pytest.mark.parametrize("value", ["\n北京 &\n", "null", "42", "", "a\nb"]) +@pytest.mark.parametrize("chunk_size", [1, 7, 1000]) +def test_mimo_preserves_verbatim_parameter_values( + mock_tokenizer, mock_request, value, chunk_size +): + from vllm.parser.mimo import MiMoParser + + text = ( + f"{value}" + "" + ) + parser = MiMoParser(mock_tokenizer, chat_template_kwargs={"enable_thinking": False}) + result = parser.extract_tool_calls(text, mock_request) + assert json.loads(result.tool_calls[0].function.arguments) == {"text": value} + parser = MiMoParser(mock_tokenizer, chat_template_kwargs={"enable_thinking": False}) + results = simulate_tool_streaming( + parser, + mock_request, + [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)], + ) + assert collect_function_name(results) == "run" + assert json.loads(collect_tool_arguments(results)) == {"text": value} diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 29b9db3f2b..321ce6b9e0 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -15,6 +15,7 @@ from __future__ import annotations import functools import json +from collections.abc import Callable from dataclasses import dataclass, replace from typing import Any @@ -39,6 +40,7 @@ from vllm.parser.engine.registered_adapters import ( GraniteThinkingParser, InklingParser, KimiK2Parser, + MiMoParser, MinimaxM2Parser, NemotronV3Parser, Plamo3Parser, @@ -356,7 +358,12 @@ def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: ] -def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]: +def _qwen3_segments( + scenario: Scenario, + tool_segments: Callable[[ToolCallSpec], list[tuple[str, bool]]] = ( + _qwen3_tool_segments + ), +) -> list[tuple[str, bool]]: segs: list[tuple[str, bool]] = [] if scenario.reasoning is not None: segs.append((scenario.reasoning, False)) @@ -369,7 +376,7 @@ def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]: segs.append((scenario.content, False)) if scenario.tool_calls: for tc in scenario.tool_calls: - segs.extend(_qwen3_tool_segments(tc)) + segs.extend(tool_segments(tc)) return segs @@ -389,6 +396,9 @@ def _build_qwen3( parser_cls: type = Qwen3Parser, strip_trailing_ws: bool = False, validate: bool = True, + tool_segments: Callable[[ToolCallSpec], list[tuple[str, bool]]] = ( + _qwen3_tool_segments + ), ) -> Sample: expected_reasoning: str | None if scenario.reasoning is not None: @@ -403,7 +413,7 @@ def _build_qwen3( sample_id=f"{name}-{scenario.id}", description=scenario.description, vocab=_QWEN3_VOCAB, - segments=_qwen3_segments(scenario), + segments=_qwen3_segments(scenario, tool_segments), expected_reasoning=expected_reasoning, expected_content=_qwen3_expected_content(scenario), expected_tool_calls=_expected_tc(scenario), @@ -414,6 +424,31 @@ def _build_qwen3( return sample +# ── MiMo (compact Qwen3 XML, no newlines between tags) ─────────────── + + +def _mimo_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: + parts = [f""] + for key, value in tc.arguments.items(): + parts.append(f"{_qwen3_arg_value(value)}") + parts.append("") + return [ + ("", True), + ("".join(parts), False), + ("", True), + ] + + +def _build_mimo(scenario: Scenario, validate: bool = True) -> Sample: + return _build_qwen3( + scenario, + name="mimo", + parser_cls=MiMoParser, + validate=validate, + tool_segments=_mimo_tool_segments, + ) + + # ── MiniMax M2 (XML invoke format, starts in REASONING) ────────────── _MINIMAX_M2_VOCAB: dict[str, int] = { @@ -1232,6 +1267,7 @@ _BUILDERS: dict[str, Any] = { "glm47_moe": _build_glm47_moe, "kimi_k2": _build_kimi_k2, "qwen3": _build_qwen3, + "mimo": _build_mimo, "inkling": _build_inkling, "plamo3": _build_plamo3, } diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 99b18c6736..01bc53968e 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -40,6 +40,7 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser +from vllm.tool_parsers.mimo_tool_parser import MiMoToolParser from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser @@ -665,6 +666,7 @@ def test_get_model_structural_tag_supports_named_tool_choice( (Llama3JsonToolParser, "llama"), (MinimaxM2ToolParser, "minimax"), (Qwen3EngineToolParser, "qwen_3_coder"), + (MiMoToolParser, "mimo"), ], ) def test_tool_parsers_declare_matching_xgrammar_builtin_model(parser_cls, model): @@ -727,7 +729,7 @@ def test_get_structural_tag_disables_reasoning( @pytest.mark.parametrize( - "parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser] + "parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser, MiMoToolParser] ) def test_unified_parser_get_structural_tag_disables_reasoning( parser_cls, @@ -1167,3 +1169,78 @@ def test_tool_strict_level_from_name(): ToolStrictLevel.from_name("strict") with pytest.raises(ValueError, match="expected one of auto, function, parameter"): ToolStrictLevel.from_name("off") + + +@pytest.mark.parametrize("policy", ["auto", "required", "named"]) +def test_mimo_strict_compact_xml(policy): + tools = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "run", + "strict": True, + "parameters": { + "type": "object", + "properties": {"n": {"type": "integer", "minimum": 1}}, + "required": ["n"], + "additionalProperties": False, + }, + }, + ) + ] + choice = ( + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="run") + ) + if policy == "named" + else policy + ) + tag = get_model_structural_tag("mimo", tools, choice, reasoning=False) + grammar = Grammar.from_structural_tag(tag) + call = "2" + assert _is_grammar_accept_string(grammar, call) + assert _is_grammar_accept_string(grammar, call * 2) == (policy != "named") + assert _is_grammar_accept_string(grammar, "answer") == (policy == "auto") + for invalid in [ + call.replace("function=run", "function=unknown"), + call.replace("2", ""), + call.replace(">2", ">bad"), + call.replace(">2", ">0"), + call.replace("", ""), + call.replace("", "3"), + "plan" + call, + ]: + assert not _is_grammar_accept_string(grammar, invalid) + tools[0].function.strict = False + relaxed = Grammar.from_structural_tag( + get_model_structural_tag("mimo", tools, "required", reasoning=False) + ) + assert _is_grammar_accept_string( + relaxed, call.replace("", "") + ) + + +@pytest.mark.parametrize("policy", ["auto", "required"]) +def test_mimo_request_limits_parallel_calls(sample_tools_strict, policy): + from vllm.parser.parser_manager import ParserManager + + request = ChatCompletionRequest( + model="mimo", + messages=[], + tools=[t.model_dump() for t in sample_tools_strict], + tool_choice=policy, + parallel_tool_calls=False, + ) + parser_cls = ParserManager.get_parser( + tool_parser_name="mimo", enable_auto_tools=True + ) + parser = parser_cls(MagicMock(), tools=sample_tools_strict) + adjusted = parser.adjust_request(request) + grammar = Grammar.from_structural_tag(adjusted.structured_outputs.structural_tag) + call = ( + "北京" + "" + ) + assert _is_grammar_accept_string(grammar, call) + assert not _is_grammar_accept_string(grammar, call * 2) + assert not _is_grammar_accept_string(grammar, call + "extra text") diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 7ce123c787..3c445c44c9 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -17,6 +17,7 @@ from vllm.parser.granite import GraniteParser from vllm.parser.granite_thinking import GraniteThinkingParser from vllm.parser.inkling import InklingParser from vllm.parser.kimi_k2 import KimiK2Parser +from vllm.parser.mimo import MiMoParser from vllm.parser.minimax_m2 import MinimaxM2Parser from vllm.parser.mistral import MistralParser from vllm.parser.nemotron_v3 import NemotronV3Parser @@ -69,6 +70,11 @@ from vllm.parser.seed_oss import SeedOssParser Qwen3ParserToolAdapter, ) = make_adapters(Qwen3Parser) +( + MiMoParserReasoningAdapter, + MiMoParserToolAdapter, +) = make_adapters(MiMoParser) + ( SeedOssParserReasoningAdapter, SeedOssParserToolAdapter, diff --git a/vllm/parser/mimo.py b/vllm/parser/mimo.py new file mode 100644 index 0000000000..b1c37dd746 --- /dev/null +++ b/vllm/parser/mimo.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""MiMo's compact XML parameters contain verbatim string values.""" + +import json +from dataclasses import replace + +from vllm.parser.qwen3 import ( + _PARAM_RE, + _PARTIAL_PARAM_RE, + CHATML_TURN_BOUNDARIES, + Qwen3Parser, + qwen3_config, +) + + +def _mimo_arg_converter(raw_args: str, partial: bool) -> str: + params = {m.group(1): m.group(2) for m in _PARAM_RE.finditer(raw_args)} + if partial: + match = _PARTIAL_PARAM_RE.search(_PARAM_RE.sub("", raw_args)) + if match and match.group(1): + params[match.group(1)] = match.group(2) + return json.dumps(params, ensure_ascii=False) + + +class MiMoParser(Qwen3Parser): + def __init__(self, tokenizer, tools=None, **kwargs): + thinking = (kwargs.get("chat_template_kwargs") or {}).get( + "enable_thinking", True + ) + kwargs.setdefault( + "parser_engine_config", + replace( + qwen3_config( + thinking=thinking, + name="mimo", + turn_boundary_tokens=CHATML_TURN_BOUNDARIES, + ), + arg_converter=_mimo_arg_converter, + ), + ) + super().__init__(tokenizer, tools, **kwargs) diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index f1ab60b6b7..736a4fcdeb 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -109,8 +109,8 @@ _REASONING_PARSERS_TO_REGISTER = { "K2HorizonReasoningParser", ), "mimo": ( - "qwen3_engine_reasoning_parser", - "Qwen3ParserReasoningAdapter", + "mimo_engine_reasoning_parser", + "MiMoParserReasoningAdapter", ), "minimax_m2": ( "minimax_m2_reasoning_parser", diff --git a/vllm/reasoning/mimo_engine_reasoning_parser.py b/vllm/reasoning/mimo_engine_reasoning_parser.py new file mode 100644 index 0000000000..d07cde9278 --- /dev/null +++ b/vllm/reasoning/mimo_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import MiMoParserReasoningAdapter + +__all__ = ["MiMoParserReasoningAdapter"] diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 90a724858a..5647a75291 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -143,8 +143,8 @@ _TOOL_PARSERS_TO_REGISTER = { "LongcatFlashToolParser", ), "mimo": ( - "qwen3_engine_tool_parser", - "Qwen3EngineToolParser", + "mimo_tool_parser", + "MiMoToolParser", ), "minimax_m2": ( "minimax_m2_tool_parser", diff --git a/vllm/tool_parsers/mimo_tool_parser.py b/vllm/tool_parsers/mimo_tool_parser.py new file mode 100644 index 0000000000..cca77eb56b --- /dev/null +++ b/vllm/tool_parsers/mimo_tool_parser.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from xgrammar.structural_tag import TriggeredTagsFormat + +from vllm.parser.engine.registered_adapters import MiMoParserToolAdapter +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel + + +class MiMoToolParser(MiMoParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = "mimo" + + def get_structural_tag( + self, request, *, reasoning=False, strict_level=ToolStrictLevel.AUTO + ): + tag = super().get_structural_tag( + request, reasoning=reasoning, strict_level=strict_level + ) + if ( + tag is not None + and request.parallel_tool_calls is False + and isinstance(tag.format, TriggeredTagsFormat) + ): + tag.format.stop_after_first = True + return tag diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index fdb4e4f92a..bc8fab8fe6 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -66,6 +66,7 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( "deepseek_v3_1", "qwen_3_5", "qwen_3_coder", + "mimo", "qwen_3", "deepseek_v3_2", "glm_4_7", From d95d1dcfb975240d125b327f1eaef1cbd56ef56c Mon Sep 17 00:00:00 2001 From: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:26:34 +0800 Subject: [PATCH 028/248] [Bugfix][Frontend] Force reasoning mode for GLM-5.3 chat templates in the GLM MoE parser (#56994) Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Chauncey --- .../test_anthropic_messages_conversion.py | 2 +- .../test_glm4_moe_reasoning_parser.py | 80 +++++++++++++++++++ vllm/parser/glm47_moe.py | 30 +++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index aa68b6bff0..06efefa099 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -1445,7 +1445,7 @@ class TestDetectMergeInlineSystem: AnthropicServingMessages._should_merge_inline_system(online_renderer) is expected ) - assert mock_logger.warning.called is expected + assert mock_logger.warning_once.called is expected # ====================================================================== diff --git a/tests/reasoning/test_glm4_moe_reasoning_parser.py b/tests/reasoning/test_glm4_moe_reasoning_parser.py index 3d6f21b5e1..dcfba7e55d 100644 --- a/tests/reasoning/test_glm4_moe_reasoning_parser.py +++ b/tests/reasoning/test_glm4_moe_reasoning_parser.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import copy + import pytest from transformers import AutoTokenizer @@ -227,3 +229,81 @@ def test_is_reasoning_end_full_prompt( token_ids = glm45_tokenizer.convert_tokens_to_ids(tokens) check_is_reasoning_end = parser.is_reasoning_end(token_ids) assert check_is_reasoning_end == is_reasoning_end + + +GLM53_TEMPLATE = ( + "[gMASK]\n" + "{%- set effective_reasoning_effort = reasoning_effort if reasoning_effort is" + " defined and reasoning_effort in ['low', 'high'] else 'max' -%}\n" + "<|system|>Reasoning Effort: {{ effective_reasoning_effort | capitalize }}\n" + "{% for tc in m.tool_calls %}\n" + "{{- '' + tc.name -}}\n" + "{% set _args = tc.arguments %}" + "{% for k, v in _args.items() %}" + "{{ k }}{{ v }}" + "{% endfor %}\n" + "{% endfor %}\n" + "<|assistant|>{{- '' -}}" +) + +GLM53_LEAK = { + "output": "Simple question.2 + 2 = **4**", + "reasoning": "Simple question.", + "content": "2 + 2 = **4**", + "is_reasoning_end": True, +} + + +@pytest.fixture() +def glm53_style_tokenizer(glm45_tokenizer): + tokenizer = copy.copy(glm45_tokenizer) + tokenizer.chat_template = GLM53_TEMPLATE + return tokenizer + + +@pytest.mark.parametrize( + "disable_kwargs", [{"enable_thinking": False}, {"thinking": False}] +) +def test_glm53_template_forces_reasoning(disable_kwargs: dict, glm53_style_tokenizer): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + parser = parser_cls(glm53_style_tokenizer, chat_template_kwargs=disable_kwargs) + assert parser._parser_engine.thinking_enabled + + output = glm53_style_tokenizer.tokenize(GLM53_LEAK["output"]) + output_tokens: list[str] = [ + glm53_style_tokenizer.convert_tokens_to_string([token]) for token in output + ] + reasoning, content = run_reasoning_extraction(parser, output_tokens) + assert reasoning == GLM53_LEAK["reasoning"] + assert content == GLM53_LEAK["content"] + + output_ids = glm53_style_tokenizer.convert_tokens_to_ids(output) + assert parser.is_reasoning_end(output_ids) == GLM53_LEAK["is_reasoning_end"] + + +def test_glm53_template_forces_reasoning_streaming(glm53_style_tokenizer): + parser = ReasoningParserManager.get_reasoning_parser(parser_name)( + glm53_style_tokenizer, chat_template_kwargs={"enable_thinking": False} + ) + output = glm53_style_tokenizer.tokenize(GLM53_LEAK["output"]) + output_tokens: list[str] = [ + glm53_style_tokenizer.convert_tokens_to_string([token]) for token in output + ] + reasoning, content = run_reasoning_extraction(parser, output_tokens, streaming=True) + assert reasoning == GLM53_LEAK["reasoning"] + assert content == GLM53_LEAK["content"] + + +def test_glm47_template_honors_thinking_disable(glm45_tokenizer): + parser = ReasoningParserManager.get_reasoning_parser(parser_name)( + glm45_tokenizer, chat_template_kwargs={"enable_thinking": False} + ) + assert not parser._parser_engine.thinking_enabled + + output = glm45_tokenizer.tokenize(GLM53_LEAK["output"]) + output_tokens: list[str] = [ + glm45_tokenizer.convert_tokens_to_string([token]) for token in output + ] + reasoning, content = run_reasoning_extraction(parser, output_tokens) + assert reasoning is None + assert content == GLM53_LEAK["output"] diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py index 00e517f2ea..9a18228fb0 100644 --- a/vllm/parser/glm47_moe.py +++ b/vllm/parser/glm47_moe.py @@ -20,6 +20,7 @@ import regex as re from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.logger import init_logger from vllm.parser.engine.events import EventType from vllm.parser.engine.parser_engine import ParserEngine from vllm.parser.engine.parser_engine_config import ( @@ -28,6 +29,8 @@ from vllm.parser.engine.parser_engine_config import ( Transition, ) +logger = init_logger(__name__) + if TYPE_CHECKING: from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool @@ -77,6 +80,23 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str: return json.dumps(params, ensure_ascii=False) +def _glm53_always_thinks(tokenizer: TokenizerLike) -> bool: + # GLM-5.3 keeps the GLM-4.x prompt and tool-call format but replaces the + # enable_thinking toggle with an always-on "Reasoning Effort:" header, so + # its generation prompt always opens and the model output carries + # no opening tag. + template = getattr(tokenizer, "chat_template", None) + return ( + isinstance(template, str) + and "[gMASK]" in template + and "Reasoning Effort:" in template + and "enable_thinking" not in template + and "" in template + and "" in template + and "" in template + ) + + @functools.cache def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: arg_tag_transitions = { @@ -192,6 +212,16 @@ class Glm47MoeParser(ParserEngine): chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} thinking = chat_kwargs.get("thinking", None) enable_thinking = chat_kwargs.get("enable_thinking", None) + if (thinking is False or enable_thinking is False) and _glm53_always_thinks( + tokenizer + ): + logger.warning_once( + "Ignoring enable_thinking/thinking: the GLM-5.3 chat template " + "has no thinking switch, so reasoning is always on and " + "disabling extraction would leak it into the content." + ) + thinking = None + enable_thinking = None self.thinking_enabled = ( True if thinking is None and enable_thinking is None From cd947f5c629f16fb1ac8ac243481ce82bfa6f5a2 Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Tue, 29 Sep 2026 15:27:03 +0200 Subject: [PATCH 029/248] [MM] Fix compiled ViT attention output layouts (#58182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Zakrzewski Co-authored-by: OpenAI Codex Co-authored-by: Nicolò Lucchesi --- tests/v1/attention/test_vit_attn_wrappers.py | 87 ++++++++++++++++++++ vllm/v1/attention/ops/vit_attn_wrappers.py | 44 ++++++---- 2 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 tests/v1/attention/test_vit_attn_wrappers.py diff --git a/tests/v1/attention/test_vit_attn_wrappers.py b/tests/v1/attention/test_vit_attn_wrappers.py new file mode 100644 index 0000000000..5d6cb323ba --- /dev/null +++ b/tests/v1/attention/test_vit_attn_wrappers.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import sys +from types import ModuleType, SimpleNamespace + +import pytest +import torch +from torch._subclasses.fake_tensor import FakeTensorMode + +from vllm.platforms import current_platform +from vllm.v1.attention.ops import vit_attn_wrappers + + +def _noncontiguous_bshd(device: torch.device | None = None) -> torch.Tensor: + return ( + torch.arange(120, dtype=torch.float32, device=device) + .reshape(1, 3, 5, 8) + .transpose(1, 2) + ) + + +def test_registered_attention_wrapper_fake_layouts(): + with FakeTensorMode(): + q = _noncontiguous_bshd() + cu_seqlens = torch.tensor([0, 5], dtype=torch.int32) + max_seqlen = torch.tensor(5, dtype=torch.int32) + flashinfer_cu_seqlens = torch.tensor([0, 5, 0, 5], dtype=torch.int32) + sequence_lengths = torch.tensor([5], dtype=torch.int32) + + outputs = ( + torch.ops.vllm.flash_attn_maxseqlen_wrapper( + q, q, q, 1, False, None, None, cu_seqlens, max_seqlen + ), + torch.ops.vllm.triton_attn_wrapper( + q, q, q, 1, None, cu_seqlens, max_seqlen + ), + torch.ops.vllm.torch_sdpa_wrapper(q, q, q, None, None, False), + torch.ops.vllm.flashinfer_wrapper( + q, + q, + q, + 1.0, + torch.empty(1, dtype=torch.uint8), + flashinfer_cu_seqlens, + max_seqlen, + sequence_lengths, + None, + None, + None, + None, + ), + ) + + assert all(output.is_contiguous() for output in outputs) + + +def test_aiter_flash_attn_omits_out_and_returns_contiguous_output( + monkeypatch: pytest.MonkeyPatch, +): + aiter = ModuleType("aiter") + + def flash_attn_varlen_func(q, k, v, **kwargs): + assert "out" not in kwargs + return q + + aiter.__dict__["flash_attn_varlen_func"] = flash_attn_varlen_func + monkeypatch.setitem(sys.modules, "aiter", aiter) + monkeypatch.setattr( + vit_attn_wrappers, "current_platform", SimpleNamespace(is_rocm=lambda: True) + ) + q = _noncontiguous_bshd() + assert not q.is_contiguous() + + output = vit_attn_wrappers.flash_attn_maxseqlen_wrapper(q, q, q, 1, True, None) + + assert torch.equal(output, q) + assert output.is_contiguous() + + +@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires GPU") +def test_sdpa_output_layout_without_cu_seqlens(): + q = _noncontiguous_bshd(torch.device("cuda")) + + output = vit_attn_wrappers.vit_torch_sdpa_wrapper(q, q, q) + + assert output.is_contiguous() diff --git a/vllm/v1/attention/ops/vit_attn_wrappers.py b/vllm/v1/attention/ops/vit_attn_wrappers.py index 36f7de393c..bef9ddb4d0 100644 --- a/vllm/v1/attention/ops/vit_attn_wrappers.py +++ b/vllm/v1/attention/ops/vit_attn_wrappers.py @@ -58,13 +58,15 @@ def flash_attn_maxseqlen_wrapper( max_seqlen = max_seqlen.item() q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) - - if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing(): - # workaround for encoder CUDA-graph replay paddings with NaNs edge case - kwargs["out"] = torch.zeros( - q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device - ) - + supports_out = not current_platform.is_rocm() + if supports_out: + if current_platform.is_cuda() and torch.cuda.is_current_stream_capturing(): + # workaround for encoder CUDA-graph replay paddings with NaNs edge case + kwargs["out"] = torch.zeros( + q.shape[0], q.shape[1], v.shape[-1], dtype=q.dtype, device=q.device + ) + else: + kwargs["out"] = torch.empty_like(q, memory_format=torch.contiguous_format) output = flash_attn_varlen_func( q, k, @@ -78,6 +80,8 @@ def flash_attn_maxseqlen_wrapper( softmax_scale=scale, **kwargs, ) + if not supports_out: + output = output.contiguous() context_layer = einops.rearrange(output, "(b s) h d -> b s h d", b=batch_size) return context_layer @@ -93,7 +97,7 @@ def flash_attn_maxseqlen_wrapper_fake( cu_seqlens: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None, ) -> torch.Tensor: - return torch.empty_like(q) + return torch.empty_like(q, memory_format=torch.contiguous_format) direct_register_custom_op( @@ -179,7 +183,7 @@ def triton_attn_wrapper( max_seqlen = max_seqlen.item() q, k, v = (einops.rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]) - output = torch.empty_like(q) + output = torch.empty_like(q, memory_format=torch.contiguous_format) context_attention_fwd( q, k, @@ -207,7 +211,7 @@ def triton_attn_wrapper_fake( cu_seqlens: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None, ) -> torch.Tensor: - return torch.empty_like(q) + return torch.empty_like(q, memory_format=torch.contiguous_format) direct_register_custom_op( @@ -281,7 +285,7 @@ def torch_sdpa_wrapper( v = v.contiguous() if cu_seqlens is None: - return apply_sdpa(q, k, v, scale=scale, enable_gqa=enable_gqa) + return apply_sdpa(q, k, v, scale=scale, enable_gqa=enable_gqa).contiguous() outputs = [] @@ -302,11 +306,11 @@ def torch_sdpa_wrapper_fake( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, - scale: float | None, - cu_seqlens: torch.Tensor | None, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, enable_gqa: bool = False, ) -> torch.Tensor: - return torch.empty_like(q) + return torch.empty_like(q, memory_format=torch.contiguous_format) direct_register_custom_op( @@ -368,6 +372,11 @@ def flashinfer_wrapper( with gpu_sync_allowed(): max_seqlen = max_seqlen.item() + output_buffer = torch.empty_like( + q, + dtype=o_data_type, + memory_format=torch.contiguous_format, + ) output, _ = cudnn_batch_prefill_with_kv_cache( q, k, @@ -388,6 +397,7 @@ def flashinfer_wrapper( k_scale=k_scale, v_scale=v_scale, o_data_type=o_data_type, + out=output_buffer, ) if is_reshaped: @@ -410,7 +420,11 @@ def vit_flashinfer_wrapper_fake( v_scale: torch.Tensor | None = None, o_data_type: torch.dtype | None = None, ) -> torch.Tensor: - return torch.empty_like(q, dtype=o_data_type or q.dtype) + return torch.empty_like( + q, + dtype=o_data_type, + memory_format=torch.contiguous_format, + ) direct_register_custom_op( From 741edeebeec3cedbe938d831b6d87641ed6191ef Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Tue, 29 Sep 2026 15:27:34 +0200 Subject: [PATCH 030/248] [Bugfix][CI] Assert the logger call the Anthropic merge warning makes (#59202) Signed-off-by: Stefan Koncarevic From ac7f3e11ea88fff26e322f529c2e563dac2f663a Mon Sep 17 00:00:00 2001 From: ahmed xijiaat <52128022+xijiaat@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:21:27 +0900 Subject: [PATCH 031/248] [Bugfix] Profile maximum DeepSeek V4.1 vision features (#57898) Signed-off-by: ahmed xijiaat <52128022+xijiaat@users.noreply.github.com> Co-authored-by: OpenAI Codex Co-authored-by: Isotr0py --- .../deepseek_v41/common/mm_preprocess.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/vllm/models/deepseek_v41/common/mm_preprocess.py b/vllm/models/deepseek_v41/common/mm_preprocess.py index 7710edb43f..017f5f87a2 100644 --- a/vllm/models/deepseek_v41/common/mm_preprocess.py +++ b/vllm/models/deepseek_v41/common/mm_preprocess.py @@ -275,15 +275,28 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo): hf_config = self.get_hf_config() patch_size = hf_config.vision_patch_size downsample_ratio = hf_config.vision_downsample_ratio - # A square maximizes the ViT patch count (area) within the token - # budget; solve the budget-derived size directly to keep the dummy - # image small. budget = hf_config.vision_max_n_token - side = budget * patch_size * downsample_ratio - best_h, best_w = solve_resize_ratio( - side, side, patch_size, downsample_ratio, budget - ) - return ImageSize(width=best_w, height=best_h) + max_wh_ratio = hf_config.vision_max_wh_ratio + best_h = best_w = 0 + # Each LLM row costs an extra newline token. Maximize the ViT grid, + # including the patch rounding after the optional aspect-ratio clamp. + for n_llm_h in range(1, (budget - 2) // 2 + 1): + n_vit_h = n_llm_h * downsample_ratio + n_vit_w = ((budget - 2) // n_llm_h - 1) * downsample_ratio + if max_wh_ratio is not None: + n_vit_w = min(n_vit_w, math.ceil(n_vit_h * max_wh_ratio)) + if n_vit_h * n_vit_w > best_h * best_w: + best_h, best_w = n_vit_h, n_vit_w + if best_h == 0: + raise ValueError( + "Image token budget and aspect ratio allow no image patches" + ) + height, width = best_h * patch_size, best_w * patch_size + # Avoid the min-pixels upscaler perturbing the selected aspect ratio. + scale = 1 + while height * width * scale * scale < hf_config.vision_min_pixels: + scale *= 2 + return ImageSize(width=width * scale, height=height * scale) class DeepseekV4VLDummyInputsBuilder( From 8aaeef343a103bea13aadbbd55978df2c272744e Mon Sep 17 00:00:00 2001 From: Giulio De Pasquale Date: Tue, 29 Sep 2026 16:33:01 +0100 Subject: [PATCH 032/248] [Bugfix][MiMo] Declare embedding_fields so an EPD pair can serve images (#58938) Signed-off-by: Giulio De Pasquale --- vllm/model_executor/models/mimo_v2_omni.py | 24 ++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index 8c838c7cfb..390af45739 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -69,7 +69,10 @@ from .qwen2_5_vl import ( Qwen2_5_VLVideoInputs, Qwen2_5_VLVideoPixelInputs, ) -from .qwen2_vl import _create_qwen2vl_field_factory +from .qwen2_vl import ( + Qwen2VLMultiModalDataParser, + _create_qwen2vl_field_factory, +) from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix @@ -670,6 +673,14 @@ class MiMoVisionTransformer(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) +class MiMoV2OmniMultiModalDataParser(Qwen2VLMultiModalDataParser): + # Video stays raw: get_video_replacement also reads second_per_grid_ts and + # video_start_times, which a metadata-only rewrite would not carry. + embedding_fields = { + "image": Qwen2VLMultiModalDataParser.embedding_fields["image"], + } + + class MiMoV2OmniProcessingInfo(BaseProcessingInfo): def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"audio": None, "image": None, "video": None} @@ -689,9 +700,14 @@ class MiMoV2OmniProcessingInfo(BaseProcessingInfo): return self.get_hf_processor(**kwargs).image_processor def get_data_parser(self): - from vllm.multimodal.parse import MultiModalDataParser - - return MultiModalDataParser(target_sr=24000.0) + # Without embedding_fields the EC producer publishes no metadata and + # the EPD proxy never rewrites the media item. + return MiMoV2OmniMultiModalDataParser( + self.get_hf_config().vision_config.spatial_merge_size, + target_sr=24000.0, + expected_hidden_size=self._get_expected_hidden_size(), + allow_missing_mm_embeddings=self.allow_missing_mm_embeddings, + ) def get_mm_max_tokens_per_item( self, From f4917dadc8b44b75e880d870588f9ddbd874f412 Mon Sep 17 00:00:00 2001 From: AlanFokCo <49893220+AlanFokCo@users.noreply.github.com> Date: Tue, 29 Sep 2026 23:34:15 +0800 Subject: [PATCH 033/248] [Bugfix][Quantization] Stop sleep(level=2) from zeroing compressed-tensors KV scales (#57163) Signed-off-by: AlanFokCo Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 Co-authored-by: aoshen02 --- .../memory/sleep_mode/test_sleep_mode.py | 54 +++++++++++++++++++ .../compressed_tensors/compressed_tensors.py | 7 +-- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py b/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py index 903fc7d3cb..3ed4703c8b 100644 --- a/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py +++ b/tests/basic_correctness/memory/sleep_mode/test_sleep_mode.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import math import pytest import torch @@ -9,6 +10,7 @@ import torch import vllm.device_allocator.cumem as cumem import vllm.envs as envs from vllm import LLM, AsyncEngineArgs, AsyncLLMEngine, SamplingParams +from vllm.model_executor.model_loader import get_model_loader from vllm.platforms import current_platform from vllm.utils.mem_constants import GiB_bytes @@ -232,6 +234,58 @@ def test_deep_sleep_async(gpu_memory_cleared): asyncio.run(test()) +def _load_weights(worker) -> None: + # Not reload_weights(): its layerwise finalize re-derives the KV scales. + model = worker.model_runner.get_model() + loader = get_model_loader(worker.load_config) + model.load_weights(loader.get_all_weights(worker.model_config, model)) + + +def _attn_kv_scales(model) -> dict[str, list[float]]: + return { + f"{name}.{attr}": getattr(module, attr).flatten().tolist() + for name, module in model.named_modules() + for attr in ("_q_scale", "_k_scale", "_v_scale") + if isinstance(getattr(module, attr, None), torch.Tensor) + } + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Segfaults on ROCm.") +@requires_fp8 +@create_new_process_for_each_test() +def test_deep_sleep_compressed_tensors_kv_scales( + monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared +): + """KV scales must survive level-2 sleep plus a weights reload (else NaN).""" + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + llm = LLM( + "nm-testing/TinyLlama-1.1B-Chat-v1.0-kvcache-fp8-tensor", + enable_sleep_mode=True, + kv_cache_dtype="fp8", + enforce_eager=True, + gpu_memory_utilization=0.6, + ) + sampling_params = SamplingParams(temperature=0, max_tokens=10, logprobs=0) + scales = llm.apply_model(_attn_kv_scales)[0] + assert any(v != 1.0 for values in scales.values() for v in values) + expected = llm.generate("How are you?", sampling_params)[0].outputs[0] + + llm.sleep(level=2) + llm.wake_up(tags=["weights"]) + llm.collective_rpc(_load_weights) + llm.wake_up(tags=["kv_cache"]) + + assert llm.apply_model(_attn_kv_scales)[0] == scales + actual = llm.generate("How are you?", sampling_params)[0].outputs[0] + assert actual.token_ids == expected.token_ids + logprobs = [ + [lp[t].logprob for lp, t in zip(o.logprobs, o.token_ids)] + for o in (expected, actual) + ] + assert all(math.isfinite(x) for x in logprobs[1]) + assert logprobs[0] == logprobs[1] + + @requires_fp8 def test_deep_sleep_fp8_kvcache_mrv1( monkeypatch: pytest.MonkeyPatch, gpu_memory_cleared diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index cb69a1d665..a3fc11be37 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -1208,9 +1208,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod): """Override the default vLLM placeholder scales with the llm-compressor loaded scales. Zero points are not used as only symmetric quantization is supported. """ - layer._k_scale = layer.k_scale - layer._v_scale = layer.v_scale - layer._q_scale = layer.q_scale + # Buffers (not Parameters) so sleep snapshots them; ATTN_HEAD changes shape. + layer.register_buffer("_k_scale", layer.k_scale.detach()) + layer.register_buffer("_v_scale", layer.v_scale.detach()) + layer.register_buffer("_q_scale", layer.q_scale.detach()) # Set the _float variants that the attention backend uses. def _to_scalar(tensor: torch.Tensor) -> float: From 2454b5a4f0c4c65109fff40e9c60e94bc0245bd2 Mon Sep 17 00:00:00 2001 From: Ashraf Bhuiyan Date: Tue, 29 Sep 2026 12:10:28 -0400 Subject: [PATCH 034/248] [Mypy] Fix mypy typing for Whisper models (#58254) Signed-off-by: Ashraf Bhuiyan Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- tools/pre_commit/mypy.py | 1 - vllm/model_executor/models/interfaces.py | 5 ++-- vllm/model_executor/models/whisper.py | 16 +++++++++---- vllm/model_executor/models/whisper_causal.py | 24 +++++++++++++------- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 69b36d43b2..5ad37e138e 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[wW]", r"vllm/model_executor/models/[zZ]", ] diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 7901a136f5..e28cc9b806 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -61,6 +61,7 @@ if TYPE_CHECKING: from vllm.multimodal.registry import _ProcessorFactories from vllm.sequence import IntermediateTensors from vllm.tasks import ScoreType + from vllm.tokenizers import TokenizerLike from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphCaptureInputs, @@ -1449,7 +1450,7 @@ class SupportsTranscription(Protocol): def parse_language_detection_output( cls, token_ids: list[int], - tokenizer: object, + tokenizer: "TokenizerLike", ) -> str: """Parse the detected language from model output token IDs. @@ -1461,7 +1462,7 @@ class SupportsTranscription(Protocol): @classmethod def get_language_token_ids( cls, - tokenizer: object, + tokenizer: "TokenizerLike", ) -> list[int] | None: """Return token IDs that represent valid language tokens. diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index f67e3f0cb9..09f5f85453 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -63,6 +63,7 @@ from vllm.multimodal.processing import ( ) from vllm.multimodal.processing.processor import HFMultiModalInputs from vllm.renderers import TokenizeParams +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.jsontree import json_map_leaves from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -325,7 +326,7 @@ class WhisperCrossAttention(WhisperAttention): prefix=f"{prefix}.kv_proj", ) - def forward( + def forward( # type: ignore[override] self, hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor | None, @@ -872,7 +873,7 @@ class WhisperForConditionalGeneration( @classmethod def get_language_token_ids( cls, - tokenizer: object, + tokenizer: TokenizerLike, ) -> list[int]: """Return token IDs for all supported language tokens. @@ -908,8 +909,8 @@ class WhisperForConditionalGeneration( def parse_language_detection_output( cls, token_ids: list[int], - tokenizer: object, - ) -> str | None: + tokenizer: TokenizerLike, + ) -> str: """Parse the language token predicted by Whisper. Decodes the first token ID and extracts the language code from the @@ -1004,6 +1005,7 @@ class WhisperForConditionalGeneration( audio_input = self._parse_and_validate_audio_input(**kwargs) # Split concatenated encoder outputs into one tensor per audio input enc_output = self.model.get_encoder_outputs(audio_input["input_features"]) + assert enc_output is not None # The assumption is we can only process whole mm items (audios) return enc_output.unbind(dim=0) @@ -1021,8 +1023,12 @@ class WhisperForConditionalGeneration( def _parse_and_validate_audio_input(self, **kwargs: object) -> WhisperAudioInputs: input_features = kwargs.pop("input_features", None) + def to_dtype(value: object) -> torch.Tensor: + assert isinstance(value, torch.Tensor) + return value.to(self.dtype) + if input_features is not None: - input_features = json_map_leaves(lambda x: x.to(self.dtype), input_features) + input_features = json_map_leaves(to_dtype, input_features) return WhisperAudioInputs(input_features=input_features) diff --git a/vllm/model_executor/models/whisper_causal.py b/vllm/model_executor/models/whisper_causal.py index ec84cae8cf..745945c2ea 100644 --- a/vllm/model_executor/models/whisper_causal.py +++ b/vllm/model_executor/models/whisper_causal.py @@ -6,7 +6,7 @@ import math from dataclasses import replace from fractions import Fraction from functools import partial -from typing import ClassVar +from typing import ClassVar, Protocol import torch import torch.nn.functional as F @@ -35,11 +35,6 @@ from vllm.v1.attention.backend import ( ) from vllm.v1.attention.backends.cpu_attn import CPUAttentionBackend from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend - -try: - from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend -except ImportError: - AiterFlashAttentionBackend = None from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend from vllm.v1.attention.selector import get_attn_backend @@ -47,11 +42,24 @@ from vllm.v1.kv_cache_interface import AttentionSpec, SlidingWindowSpec from .utils import make_layers +AiterFlashAttentionBackend: type[AttentionBackend] | None = None +try: + from vllm.v1.attention.backends import rocm_aiter_fa + + AiterFlashAttentionBackend = rocm_aiter_fa.AiterFlashAttentionBackend +except ImportError: + pass + logger = logging.getLogger(__name__) CausalRMSNorm = partial(RMSNorm, eps=1e-5) +class _BlockPoolingAttentionMetadata(Protocol): + @property + def slot_mapping(self) -> torch.Tensor: ... + + def _pad1d( x: torch.Tensor, paddings: tuple[int, int], @@ -115,7 +123,7 @@ class WhisperCausalConv1d(nn.Conv1d): @functools.lru_cache def create_whisper_attention_backend_with_block_pooling( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], block_pool_size: int, sliding_window: int | None = None, ) -> type[AttentionBackend]: @@ -270,7 +278,7 @@ def create_whisper_attention_backend_with_block_pooling( key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, - attn_metadata: AttentionMetadata, + attn_metadata: _BlockPoolingAttentionMetadata | None, output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, From c3a4a36c201f8da384e43d14600ffaa2802930c6 Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Tue, 29 Sep 2026 18:32:04 +0200 Subject: [PATCH 035/248] [CI][ROCm] Fix stale basic_correctness path in Model Runner V2 Distributed (#59201) Signed-off-by: Stefan Koncarevic Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 9e4eaceb41..b39939b235 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1960,14 +1960,14 @@ steps: source_file_dependencies: - vllm/v1/worker/gpu/ - vllm/v1/worker/gpu_worker.py - - tests/basic_correctness/test_basic_correctness.py + - tests/basic_correctness/models/test_basic_correctness.py - tests/v1/distributed/test_async_llm_dp.py - tests/v1/distributed/test_eagle_dp.py - vllm/platforms/rocm.py commands: - set -x - export VLLM_USE_V2_MODEL_RUNNER=1 - - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True" + - TARGET_TEST_SUITE=MI300 pytest -v -s basic_correctness/models/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py From 31876d8ea166e9fbeb6782466954151fd7e06a91 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Wed, 30 Sep 2026 00:34:48 +0800 Subject: [PATCH 036/248] [Bugfix] Fix TorchCodec audio IO correctness (#58364) Signed-off-by: Isotr0py --- docs/features/multimodal_inputs.md | 19 +++++---- tests/multimodal/media/test_audio.py | 23 +++++++++-- vllm/multimodal/media/audio.py | 61 +++++++++++++++++----------- 3 files changed, 67 insertions(+), 36 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 64dca49d28..40ceb8158b 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1210,7 +1210,7 @@ vLLM decodes audio bytes into waveforms using a selectable decoding backend: | Backend | Description | | --- | --- | -| `auto` (default) | soundfile, falling back to torchcodec, then PyAV | +| `auto` (default) | torchcodec, falling back to soundfile, then PyAV | | `soundfile` | libsndfile only, no fallback | | `pyav` | PyAV (FFmpeg) only, no fallback | | `torchcodec` | TorchCodec (PyTorch-native) only, no fallback | @@ -1226,17 +1226,18 @@ vllm serve mistralai/Voxtral-Mini-3B-2507 \ `pyav` drives FFmpeg through a per-frame Python generator, so under concurrency the Python/C crossings contend on the GIL. `torchcodec` decodes each stream in a single call that releases the GIL for its whole - duration. Select it explicitly for concurrent decoding workloads. - `auto` prefers soundfile for supported formats to preserve their existing - decoding behavior, including encoder padding. Audio extracted from formats - that soundfile cannot read, such as video containers, can use torchcodec - through the fallback chain. + duration, which is why `auto` prefers it when many requests decode audio + concurrently, such as when audio tracks are extracted from video. + vLLM normalizes codec padding on this path (e.g. it trims trailing Vorbis + padding that older system FFmpeg versions (< 5.0) fail to trim), so the + decoded waveform matches the soundfile reference length. !!! note `torchcodec` ships as a requirement on CUDA, CPU and XPU builds. On other - platforms (e.g. ROCm, TPU), install it manually to enable that backend. - torchcodec also links against a system FFmpeg installation. If the package - or FFmpeg is unavailable, `auto` uses the soundfile → PyAV chain. + platforms (e.g. ROCm, TPU) `auto` falls back to the soundfile → PyAV + chain unless you install it manually. torchcodec also links against a + system FFmpeg installation; if FFmpeg is missing, `auto` falls back the + same way. ### Embedding Inputs diff --git a/tests/multimodal/media/test_audio.py b/tests/multimodal/media/test_audio.py index 0404535e1b..a75c20979e 100644 --- a/tests/multimodal/media/test_audio.py +++ b/tests/multimodal/media/test_audio.py @@ -185,9 +185,15 @@ def test_load_audio_backend_matches_default(backend, dummy_audio_bytes): ref_audio, ref_sr = load_audio(BytesIO(dummy_audio_bytes), sr=None) audio, sr = load_audio(BytesIO(dummy_audio_bytes), sr=None, backend=backend) assert sr == ref_sr - # Decoders disagree only on codec encoder-delay/padding, so torchcodec may - # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare the - # overlapping region, which must agree to float32 precision. + if backend == "torchcodec": + # Vorbis encoder padding is trimmed, so torchcodec must match the + # soundfile reference exactly, including the waveform length. + assert audio.shape == ref_audio.shape + np.testing.assert_allclose(ref_audio, audio, atol=1e-4) + return + # Decoders disagree only on codec encoder-delay/padding, so PyAV may + # emit a few extra trailing samples (e.g. ~192 for Ogg Vorbis). Compare + # the overlapping region, which must agree to float32 precision. n = min(ref_audio.shape[-1], audio.shape[-1]) assert n > 0 np.testing.assert_allclose(ref_audio[:n], audio[:n], atol=1e-4) @@ -201,6 +207,17 @@ def test_load_audio_default_preserves_vorbis_length(dummy_audio_bytes): assert audio.shape == expected.shape +def test_load_audio_torchcodec_preserves_vorbis_length(dummy_audio_bytes): + """Torchcodec must trim Vorbis trailing padding even on FFmpeg < 5.0, + matching the soundfile reference exactly.""" + pytest.importorskip("torchcodec") + expected, expected_sr = load_audio_soundfile(BytesIO(dummy_audio_bytes), sr=None) + audio, sr = load_audio_torchcodec(BytesIO(dummy_audio_bytes), sr=None) + assert sr == expected_sr + assert audio.shape == expected.shape + np.testing.assert_allclose(audio, expected, atol=1e-4) + + def test_load_audio_unknown_backend_rejected(dummy_audio_bytes): """An unknown backend must fail loudly instead of silently degrading.""" with pytest.raises(ValueError, match="Unknown audio backend"): diff --git a/vllm/multimodal/media/audio.py b/vllm/multimodal/media/audio.py index 1f1cda8dcc..c05f4e4dcc 100644 --- a/vllm/multimodal/media/audio.py +++ b/vllm/multimodal/media/audio.py @@ -15,6 +15,7 @@ from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.multimodal.audio import resample_audio_pyav from vllm.utils.import_utils import PlaceholderModule +from vllm.utils.math_utils import cdiv from vllm.utils.mem_constants import MiB_bytes from vllm.utils.serial_utils import tensor2base64 from vllm.utils.sparse_utils import ( @@ -60,11 +61,11 @@ _BAD_SF_CODES = {0, 1, 3, 4} # Audio decoding backends selectable via `load_audio(backend=...)` or # `--media-io-kwargs '{"audio": {"audio_backend": ...}}'`. -# "auto" tries soundfile, then torchcodec, then PyAV. +# "auto" tries torchcodec, then soundfile, then PyAV. AUDIO_BACKENDS = ("auto", "soundfile", "pyav", "torchcodec") # Raised as ImportError when the torchcodec backend cannot be used, so -# `load_audio(backend="auto")` can continue to PyAV. +# `load_audio(backend="auto")` falls back to the soundfile → PyAV chain. _TORCHCODEC_UNAVAILABLE_MSG = ( "torchcodec audio backend is unavailable (requires the torchcodec " "package and a system ffmpeg installation)" @@ -74,6 +75,9 @@ _TORCHCODEC_UNAVAILABLE_MSG = ( # over-long stream is rejected rather than truncated to the limit. _DURATION_GUARD_MARGIN_S = 1.0 +# Largest Vorbis block size in native-rate samples +_VORBIS_MAX_BLOCK_SAMPLES = 8192 + def load_audio_pyav( path: BytesIO | Path | str, @@ -364,6 +368,16 @@ def load_audio_torchcodec( "audio or video file (e.g. a complete WAV, MP3, or MP4)." ) from e + # FFmpeg < 5.0 misses Ogg end_trimming, leaving up to one Vorbis block of + # padding at the tail; we need to trim for valid audio length. + if metadata.codec == "vorbis" and metadata.duration_seconds is not None: + expected_samples = round(metadata.duration_seconds * samples.sample_rate) + padding = samples.data.shape[-1] - expected_samples + native_sr = metadata.sample_rate or samples.sample_rate + max_padding = cdiv(_VORBIS_MAX_BLOCK_SAMPLES * samples.sample_rate, native_sr) + if 0 < padding <= max_padding: + samples.data = samples.data[..., :expected_samples] + # Authoritative post-decode checks: the metadata estimates above can be # bypassed by a container that lies about its duration or size. if max_duration_s is not None: @@ -425,8 +439,8 @@ def load_audio( max_decode_bytes: Reject audio that would decode to more than this many bytes. backend: One of ``AUDIO_BACKENDS``. ``None`` (default) selects - ``"auto"``, which tries soundfile, then torchcodec, then PyAV; - the other values select a single + ``"auto"``, which tries torchcodec, then falls back to the + soundfile → PyAV chain; the other values select a single backend with no fallback. """ @@ -446,11 +460,25 @@ def load_audio( max_decode_bytes=max_decode_bytes, ) - # Keep soundfile first to preserve decoding and padding of supported formats. - # "auto": soundfile → torchcodec → PyAV. Each backend is called by its - # bare name so tests that monkeypatch it take effect. Only an ImportError - # (backend missing) or, for soundfile, an unsupported-format error defers - # to the next; any other error (decode failure, guard ValueError) raises. + # "auto": torchcodec → soundfile → PyAV, each called by its bare name so + # monkeypatched tests take effect. Only ImportError (or, for soundfile, an + # unsupported-format error) defers to the next backend; anything else raises. + if isinstance(path, BytesIO): + path.seek(0) + try: + return load_audio_torchcodec( + path, + sr=sr, + mono=mono, + max_duration_s=max_duration_s, + max_decode_bytes=max_decode_bytes, + ) + except ImportError as exc: + # Decode errors don't defer: FFmpeg-based PyAV would fail the same way. + logger.warning( + "torchcodec unavailable (%r); falling back to soundfile/PyAV.", exc + ) + if isinstance(path, BytesIO): path.seek(0) try: @@ -474,21 +502,6 @@ def load_audio( if exc.code not in _BAD_SF_CODES: raise - if isinstance(path, BytesIO): - path.seek(0) - try: - return load_audio_torchcodec( - path, - sr=sr, - mono=mono, - max_duration_s=max_duration_s, - max_decode_bytes=max_decode_bytes, - ) - except ImportError as exc: - # Decode errors don't defer: torchcodec is FFmpeg-based like PyAV, so - # retrying would just fail the same way. - logger.warning("torchcodec unavailable (%r); falling back to PyAV.", exc) - # PyAV is terminal: nothing left to fall back to. Normalize an FFmpeg # failure to ValueError; let a missing soundfile/PyAV surface its # "install vllm[audio]" ImportError. From cd94b21264a76c484a567566658995fd3a84db65 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 30 Sep 2026 00:46:56 +0800 Subject: [PATCH 037/248] [Bugfix][Frontend] Return 400 for malformed RL dev route bodies (#59236) Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- .../test_dev_route_request_bodies.py | 96 +++++++++++++++++++ vllm/entrypoints/serve/dev/rlhf/api_router.py | 37 +++---- vllm/entrypoints/serve/dev/rpc/api_router.py | 5 + 3 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 tests/entrypoints/unit_tests/test_dev_route_request_bodies.py diff --git a/tests/entrypoints/unit_tests/test_dev_route_request_bodies.py b/tests/entrypoints/unit_tests/test_dev_route_request_bodies.py new file mode 100644 index 0000000000..78fe5eef5d --- /dev/null +++ b/tests/entrypoints/unit_tests/test_dev_route_request_bodies.py @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Malformed bodies on RL dev routes are 400s that never reach the engine.""" + +import json +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from vllm.entrypoints.serve.dev.rlhf.api_router import router as rlhf_router +from vllm.entrypoints.serve.dev.rpc.api_router import router as rpc_router +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler + +pytestmark = pytest.mark.cpu_test + + +class Engine: + def __init__(self): + self.calls: list[str] = [] + + def __getattr__(self, name): + async def call(*args, **kwargs): + self.calls.append(name) + + return call + + +@pytest.fixture +def client(): + app = FastAPI() + app.include_router(rlhf_router) + app.include_router(rpc_router) + init_exception_handler(app) + app.state.engine_client = Engine() + app.state.args = SimpleNamespace(log_error_stack=False) + return TestClient(app) + + +def post(client, path, body): + return client.post( + path, content=json.dumps(body), headers={"content-type": "application/json"} + ) + + +@pytest.mark.parametrize( + "path", + [ + "/abort_requests", + "/init_weight_transfer_engine", + "/update_weights", + "/collective_rpc", + ], +) +@pytest.mark.parametrize("body", [[], 1, "x", None]) +def test_non_object_body_is_rejected(client, path, body): + assert post(client, path, body).status_code == 400 + assert client.app.state.engine_client.calls == [] + + +@pytest.mark.parametrize( + "path,body", + [ + ("/init_weight_transfer_engine", {}), + ("/init_weight_transfer_engine", {"init_info": []}), + ("/init_weight_transfer_engine", {"init_info": 1}), + ("/update_weights", {}), + ("/update_weights", {"update_info": 1}), + ("/update_weights", {"update_info": [1, 2]}), + ("/update_weights", {"update_info": "x"}), + ], +) +def test_invalid_field_is_rejected_before_the_engine(client, path, body): + # Reaching the engine would also abort an in-progress weight update. + assert post(client, path, body).status_code == 400 + assert client.app.state.engine_client.calls == [] + + +@pytest.mark.parametrize( + "path,body,engine_call", + [ + ("/abort_requests", {"request_ids": ["a"]}, "abort"), + ( + "/init_weight_transfer_engine", + {"init_info": {}}, + "init_weight_transfer_engine", + ), + ("/update_weights", {"update_info": {"names": []}}, "update_weights"), + ("/update_weights", {"update_info": [{}, {}]}, "update_weights"), + ("/collective_rpc", {"method": "m"}, "collective_rpc"), + ], +) +def test_valid_body_reaches_the_engine(client, path, body, engine_call): + assert post(client, path, body).status_code == 200 + assert client.app.state.engine_client.calls == [engine_call] diff --git a/vllm/entrypoints/serve/dev/rlhf/api_router.py b/vllm/entrypoints/serve/dev/rlhf/api_router.py index 603fdf785c..7af2ead22c 100644 --- a/vllm/entrypoints/serve/dev/rlhf/api_router.py +++ b/vllm/entrypoints/serve/dev/rlhf/api_router.py @@ -3,13 +3,14 @@ import json from http import HTTPStatus -from typing import Annotated +from typing import Annotated, Any from fastapi import APIRouter, Body, FastAPI, HTTPException, Query, Request from fastapi.responses import JSONResponse from vllm.distributed.weight_transfer.base import ( WeightTransferInitRequest, + WeightTransferUpdatePayload, WeightTransferUpdateRequest, ) from vllm.engine.protocol import EngineClient @@ -104,6 +105,10 @@ async def abort_requests(raw_request: Request) -> JSONResponse: body = await raw_request.json() except json.JSONDecodeError as e: raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904 + if not isinstance(body, dict): + raise HTTPException( + status_code=400, detail="Request body must be a JSON object" + ) request_ids = body.get("request_ids") @@ -154,17 +159,10 @@ async def is_paused(raw_request: Request) -> JSONResponse: @router.post("/init_weight_transfer_engine") -async def init_weight_transfer_engine(raw_request: Request): - try: - body = await raw_request.json() - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904 - init_info = body.get("init_info") - if init_info is None: - raise HTTPException( - status_code=HTTPStatus.BAD_REQUEST.value, - detail="Missing 'init_info' in request body", - ) +async def init_weight_transfer_engine( + raw_request: Request, + init_info: Annotated[dict[str, Any], Body(embed=True)], +): await engine_client(raw_request).init_weight_transfer_engine( WeightTransferInitRequest(init_info=init_info) ) @@ -184,17 +182,10 @@ async def start_draft_weight_update(raw_request: Request): @router.post("/update_weights") -async def update_weights(raw_request: Request): - try: - body = await raw_request.json() - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail="Invalid JSON format") from e # noqa: B904 - update_info = body.get("update_info") - if update_info is None: - raise HTTPException( - status_code=HTTPStatus.BAD_REQUEST.value, - detail="Missing 'update_info' in request body", - ) +async def update_weights( + raw_request: Request, + update_info: Annotated[WeightTransferUpdatePayload, Body(embed=True)], +): await engine_client(raw_request).update_weights( request=WeightTransferUpdateRequest(update_info=update_info) ) diff --git a/vllm/entrypoints/serve/dev/rpc/api_router.py b/vllm/entrypoints/serve/dev/rpc/api_router.py index 99b904c2f6..468e876508 100644 --- a/vllm/entrypoints/serve/dev/rpc/api_router.py +++ b/vllm/entrypoints/serve/dev/rpc/api_router.py @@ -29,6 +29,11 @@ async def collective_rpc(raw_request: Request): status_code=HTTPStatus.BAD_REQUEST.value, detail=f"JSON decode error: {e}", ) from e + if not isinstance(body, dict): + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="Request body must be a JSON object", + ) method = body.get("method") if method is None: raise HTTPException( From cb4f016c5d02c9d483e6e3f1702d51da239b1a9b Mon Sep 17 00:00:00 2001 From: Jung Jiyu <142137932+jungjiyu@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:53:20 +0900 Subject: [PATCH 038/248] [Bugfix] Fix standalone torch.compile cache loading after relocation (#52142) Signed-off-by: jungjiyu Co-authored-by: Misha Goin --- tests/compile/test_compiler_interface.py | 48 ++++++++++++++++++++++++ vllm/compilation/compiler_interface.py | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests/compile/test_compiler_interface.py diff --git a/tests/compile/test_compiler_interface.py b/tests/compile/test_compiler_interface.py new file mode 100644 index 0000000000..a54f2a1056 --- /dev/null +++ b/tests/compile/test_compiler_interface.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from vllm.compilation.compiler_interface import InductorStandaloneAdaptor + + +@pytest.mark.parametrize("save_format", ["binary", "unpacked"]) +def test_inductor_standalone_load_uses_current_cache_dir( + tmp_path: Path, + save_format: str, +): + old_cache_dir = tmp_path / "old" + new_cache_dir = tmp_path / "new" + key = "artifact_shape_None_subgraph_0" + + adaptor = InductorStandaloneAdaptor(save_format=save_format) + adaptor.initialize_cache(str(new_cache_dir)) + + # Simulate a handle persisted before the cache directory was relocated. + handle = (key, str(old_cache_dir / key)) + + with ( + patch( + "torch._inductor.CompiledArtifact.load", + return_value=MagicMock(), + ) as load_mock, + patch( + "torch._inductor.compile_fx.graph_returns_tuple", + return_value=True, + ), + ): + adaptor.load( + handle=handle, + graph=MagicMock(), + example_inputs=[], + graph_index=0, + compile_range=MagicMock(), + ) + + load_mock.assert_called_once_with( + path=str(new_cache_dir / key), + format=save_format, + ) diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index 394799ab87..a297b0ed95 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -418,7 +418,7 @@ class InductorStandaloneAdaptor(CompilerInterface): assert isinstance(handle, tuple) assert isinstance(handle[0], str) assert isinstance(handle[1], str) - path = handle[1] + path = os.path.join(self.cache_dir, handle[0]) inductor_compiled_graph = torch._inductor.CompiledArtifact.load( path=path, format=self.save_format ) From f5ed3d9c03ec634fbfe670baaf234569bbdecf96 Mon Sep 17 00:00:00 2001 From: siyu Date: Wed, 30 Sep 2026 01:58:35 +0800 Subject: [PATCH 039/248] [Fast Start] Charge daemon-held weights against `gpu_memory_utilization` (#57298) Signed-off-by: liusy58 Signed-off-by: Isotr0py Signed-off-by: Xun Sun Co-authored-by: Isotr0py Co-authored-by: Xun Sun --- .../model_loader/test_weight_cache.py | 15 ++++ tests/v1/worker/test_utils.py | 38 +++++++++- .../model_loader/base_loader.py | 5 ++ .../model_loader/weight_cache/daemon.py | 19 ++++- .../model_loader/weight_cache/ipc_loader.py | 76 ++++++++++++++++--- vllm/v1/worker/gpu_worker.py | 9 ++- vllm/v1/worker/utils.py | 39 +++++++++- 7 files changed, 187 insertions(+), 14 deletions(-) diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index 7706522f6e..106b9690ac 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -354,3 +354,18 @@ def test_weight_cache_key_distinguishes_dp_ranks(): assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"] assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"] assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"] + + +def test_ipc_loader_copy_mode_reports_no_external_weight_memory(): + """Copy mode clones the weights into the engine, so nothing is external + (vllm_config is never touched).""" + from vllm.config import LoadConfig + from vllm.model_executor.model_loader.weight_cache.ipc_loader import ( + IpcModelLoader, + ) + + copy_mode = LoadConfig( + load_format="ipc_cache", model_loader_extra_config={"mode": "copy"} + ) + loader = IpcModelLoader(copy_mode) + assert loader.get_external_weight_memory(None) == 0 # type: ignore[arg-type] diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 9a345e926c..40ac8b56ee 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -13,7 +13,7 @@ import torch import vllm.v1.attention.backends.mla.index_group as index_group_module import vllm.v1.hisparse.runtime as hisparse_runtime_module -from vllm.config import CUDAGraphMode +from vllm.config import CacheConfig, CUDAGraphMode from vllm.config.mamba import MambaBackendEnum, MambaConfig from vllm.distributed.kv_transfer.kv_connector.v1.hisparse import ( worker as hisparse_worker_module, @@ -25,12 +25,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import ( _SlotMappingStaging, ) from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 +from vllm.utils.mem_constants import GiB_bytes +from vllm.utils.mem_utils import MemorySnapshot from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror from vllm.v1.worker.utils import ( bind_kv_cache, bind_kv_cache_to_layers, copy_kv_cache_blocks_inplace, + request_memory, ) @@ -1390,3 +1393,36 @@ def test_bind_kv_cache_draft_model(default_vllm_config): assert runner_kv_caches[1] is kv_cache["draft_model.layers.0.attn"] assert runner_kv_caches[2] is kv_cache["model.layers.1.attn"] assert runner_kv_caches[3] is kv_cache["draft_model.layers.1.attn"] + + +def _memory_snapshot(total_gib: int, free_gib: int) -> MemorySnapshot: + return MemorySnapshot( + free_memory=free_gib * GiB_bytes, + total_memory=total_gib * GiB_bytes, + device="cpu", + auto_measure=False, + ) + + +def test_request_memory_charges_external_weights(): + """Externally held weights are charged against the utilization budget; + the engine is granted only the remainder.""" + cache_config = CacheConfig(gpu_memory_utilization=0.9) + # 70 GiB of a 100 GiB device is held externally before the worker starts. + snapshot = _memory_snapshot(total_gib=100, free_gib=30) + + with pytest.raises(ValueError, match="less than desired"): + request_memory(snapshot, cache_config) + + # 90 GiB budget - 70 GiB external = 20 GiB for the engine. + assert request_memory(snapshot, cache_config, 70 * GiB_bytes) == 20 * GiB_bytes + + # The external weights alone exceed the utilization budget. + with pytest.raises(ValueError, match="exceed the desired"): + request_memory(snapshot, cache_config, 95 * GiB_bytes) + + # Other tenants squeeze free memory below the engine's remainder. + with pytest.raises(ValueError, match="less than the engine's budget"): + request_memory( + _memory_snapshot(total_gib=100, free_gib=10), cache_config, 70 * GiB_bytes + ) diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index 6f35646458..7712efbfd1 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -28,6 +28,11 @@ class BaseModelLoader(ABC): def __init__(self, load_config: LoadConfig): self.load_config = load_config + def get_external_weight_memory(self, vllm_config: VllmConfig) -> int: + """Get weights memory from external process; + 0 when the weights are not external.""" + return 0 + @abstractmethod def download_model(self, model_config: ModelConfig) -> None: """Download a model so that it can be immediately loaded.""" diff --git a/vllm/model_executor/model_loader/weight_cache/daemon.py b/vllm/model_executor/model_loader/weight_cache/daemon.py index b56cbbea8f..2c6944f1eb 100644 --- a/vllm/model_executor/model_loader/weight_cache/daemon.py +++ b/vllm/model_executor/model_loader/weight_cache/daemon.py @@ -78,6 +78,7 @@ cached and keep loading from disk in the engine. import contextlib import fcntl +import gc import multiprocessing import os import socket @@ -116,6 +117,7 @@ from vllm.model_executor.model_loader.weight_cache.utils import ( is_draft_model_cacheable, ) from vllm.platforms import current_platform +from vllm.utils.mem_utils import format_gib from vllm.utils.torch_utils import set_default_torch_dtype from vllm.v1.worker.workspace import init_workspace_manager @@ -249,10 +251,17 @@ class WeightCacheDaemon: with set_current_vllm_config(self.vllm_config): ensure_model_parallel_initialized(self.tp_size, self.pp_size) self.model = self.get_model() + # Loading and post-processing leave freed transients in the caching + # allocator; return them so engines sharing the GPU can use them. + gc.collect() + torch.accelerator.empty_cache() logger.info( - "Weight cache %s daemon rank %d loaded model", + "Weight cache %s daemon rank %d loaded model (%s GiB allocated, " + "%s GiB reserved)", self.role, self.global_rank, + format_gib(torch.accelerator.memory_allocated()), + format_gib(torch.accelerator.memory_reserved()), ) def get_model(self) -> torch.nn.Module: @@ -369,6 +378,14 @@ class WeightCacheDaemon: cmd = request.get("cmd") if cmd == "get_state": self._handle_get_state(conn, request) + elif cmd == "get_memory": + send_msg( + conn, + { + "status": "ok", + "memory_bytes": torch.accelerator.memory_allocated(), + }, + ) elif cmd == "release": self._handle_release(conn) else: diff --git a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py index 185c11b83a..5fd755bf3f 100644 --- a/vllm/model_executor/model_loader/weight_cache/ipc_loader.py +++ b/vllm/model_executor/model_loader/weight_cache/ipc_loader.py @@ -6,7 +6,9 @@ daemon via CUDA IPC instead of loading from disk.""" import dataclasses import socket import time +from collections.abc import Callable from copy import copy +from typing import TypeVar import torch import torch.nn as nn @@ -23,6 +25,7 @@ from vllm.logger import init_logger from vllm.model_executor.model_loader.base_loader import BaseModelLoader from vllm.model_executor.model_loader.default_loader import DefaultModelLoader from vllm.model_executor.model_loader.utils import ( + get_draft_load_config, initialize_model, process_weights_after_loading, ) @@ -40,6 +43,9 @@ from vllm.model_executor.model_loader.weight_cache.protocol import ( send_msg, verify_socket_owner, ) +from vllm.model_executor.model_loader.weight_cache.utils import ( + is_draft_model_cacheable, +) from vllm.model_executor.utils import weights_already_processed from vllm.tracing import instrument from vllm.utils.torch_utils import set_default_torch_dtype @@ -50,6 +56,8 @@ _CONNECT_TIMEOUT_S = 5.0 _STATE_TIMEOUT_S = 300.0 _STARTUP_RETRY_INTERVAL_S = 0.5 +_T = TypeVar("_T") + class IpcModelLoader(BaseModelLoader): """Loads a model by mapping the weight cache daemon's tensors via CUDA IPC. @@ -110,6 +118,52 @@ class IpcModelLoader(BaseModelLoader): f"{load_config.load_format}: {sorted(extra_config)}" ) + def get_external_weight_memory(self, vllm_config: VllmConfig) -> int: + # Copy mode clones the weights into this process; nothing external. + if self.mode != "zero_copy": + return 0 + total = self._daemon_memory() + if is_draft_model_cacheable(vllm_config.speculative_config): + # The draft group is queried with the same config the engine + # would load the draft with. + draft_config = get_draft_load_config(vllm_config) + if draft_config.load_format == "ipc_cache": + draft_loader = IpcModelLoader(draft_config) + # Only a zero-copy draft-group daemon holds external weights; + # an explicit draft_load_config without is_draft would resolve + # back to the target socket and double-count it. + if draft_loader.is_draft and draft_loader.mode == "zero_copy": + total += draft_loader._daemon_memory() + return total + + def _daemon_memory(self) -> int: + if not self.fallback: + # The loader waits for the daemon at load time, so the weights + # will be zero-copy mapped for sure; mirror that wait here since + # returning 0 would over-grant the memory budget. + return self._with_startup_wait(self._query_daemon_memory) + # An unreachable daemon means a disk load, i.e. nothing external. + try: + return self._query_daemon_memory() + except (WeightCacheUnavailableError, ConnectionError, OSError) as e: + logger.warning( + "Cannot query weight cache daemon memory (%s); " + "assuming the weights are not externally held", + e, + ) + return 0 + + def _query_daemon_memory(self) -> int: + with self._connect(self.connect_timeout_s) as conn: + send_msg(conn, {"cmd": "get_memory"}) + response = recv_msg(conn) + if response.get("status") != "ok": + raise WeightCacheUnavailableError( + "Weight cache daemon rejected the memory query: " + f"{response.get('message')}" + ) + return int(response.get("memory_bytes", 0)) + def download_model(self, model_config: ModelConfig) -> None: DefaultModelLoader(self._fallback_load_config()).download_model(model_config) @@ -308,23 +362,25 @@ class IpcModelLoader(BaseModelLoader): def _request_state_with_startup_wait( self, cache_config: WeightCacheKey ) -> WeightCacheState: + return self._with_startup_wait(lambda: self._request_state(cache_config)) + + def _with_startup_wait(self, op: Callable[[], _T]) -> _T: + """Retry op until the daemon answers or the state timeout elapses; + the daemon may still be loading the model when the engine starts.""" deadline = time.monotonic() + self.state_timeout_s - waiting_logged = False while True: try: - return self._request_state(cache_config) - except WeightCacheUnavailableError as e: + return op() + except (WeightCacheUnavailableError, ConnectionError, OSError) as e: if time.monotonic() >= deadline: raise WeightCacheUnavailableError( "Weight cache daemon did not become ready within " - f"{self.state_timeout_s:.1f}s" + f"{self.state_timeout_s:.1f}s: {e}" ) from e - if not waiting_logged: - logger.info( - "Waiting up to %.1fs for the weight cache daemon to start", - self.state_timeout_s, - ) - waiting_logged = True + logger.info_once( + "Waiting up to %.1fs for the weight cache daemon to start", + self.state_timeout_s, + ) time.sleep( max( 0.0, diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 778d71bdbc..c6da90b077 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -57,6 +57,7 @@ from vllm.distributed.weight_transfer import ( ) from vllm.logger import init_logger from vllm.lora.request import LoRARequest +from vllm.model_executor.model_loader import get_model_loader from vllm.model_executor.warmup.kernel_warmup import kernel_warmup from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory from vllm.platforms import current_platform @@ -487,7 +488,13 @@ class Worker(WorkerBase): # take current memory snapshot self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device) - self.requested_memory = request_memory(init_snapshot, self.cache_config) + # Weights from external model loader process + external_weight_memory = get_model_loader( + self.load_config + ).get_external_weight_memory(self.vllm_config) + self.requested_memory = request_memory( + init_snapshot, self.cache_config, external_weight_memory + ) logger.debug("worker init memory snapshot: %r", self.init_snapshot) logger.debug( "worker requested memory: %sGiB", format_gib(self.requested_memory) diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index a6ef84bbd3..422ba8885f 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -536,7 +536,11 @@ def sanity_check_mm_encoder_outputs( ) -def request_memory(init_snapshot: MemorySnapshot, cache_config: CacheConfig) -> int: +def request_memory( + init_snapshot: MemorySnapshot, + cache_config: CacheConfig, + external_weight_memory: int = 0, +) -> int: """Calculate the amount of memory required by vLLM, then validate that the current amount of free memory is sufficient for that. """ @@ -544,6 +548,39 @@ def request_memory(init_snapshot: MemorySnapshot, cache_config: CacheConfig) -> init_snapshot.total_memory * cache_config.gpu_memory_utilization ) + if external_weight_memory > 0: + engine_memory = requested_memory - external_weight_memory + if engine_memory <= 0: + raise ValueError( + f"Externally held weights on device {init_snapshot.device_} " + f"({format_gib(external_weight_memory)}/" + f"{format_gib(init_snapshot.total_memory)} GiB) exceed the " + "desired GPU memory utilization " + f"({cache_config.gpu_memory_utilization}, " + f"{format_gib(requested_memory)} GiB). Increase GPU memory " + "utilization or reduce GPU memory used by other processes." + ) + if init_snapshot.free_memory < engine_memory: + raise ValueError( + f"Free memory on device {init_snapshot.device_} " + f"({format_gib(init_snapshot.free_memory)}/" + f"{format_gib(init_snapshot.total_memory)} GiB) on startup " + "is less than the engine's budget after excluding " + "external process's weights " + f"({cache_config.gpu_memory_utilization}, " + f"{format_gib(engine_memory)} GiB). Decrease GPU memory " + "utilization or reduce GPU memory used by other processes." + ) + logger.info_once( + "Weights are held outside this process: of the %s GiB " + "utilization budget, %s GiB is externally held and " + "%s GiB remains for the engine's own allocations.", + format_gib(requested_memory), + format_gib(external_weight_memory), + format_gib(engine_memory), + ) + return engine_memory + if init_snapshot.free_memory < requested_memory: raise ValueError( f"Free memory on device {init_snapshot.device_} " From afac509a333db6d4ffa1f35b13c27c9e0c1f33aa Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 30 Sep 2026 02:01:54 +0800 Subject: [PATCH 040/248] [Bugfix][Rust Frontend] Skip engine-derived metrics under `--disable-log-stats` (#59205) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/src/bench/src/mm_processor.rs | 1 + .../examples/external_engine_chat_qwen.rs | 1 + rust/src/cmd/src/cli.rs | 4 +- .../examples/external_engine_logprobs.rs | 1 + .../examples/external_engine_utility_call.rs | 1 + rust/src/engine-core-client/src/client.rs | 18 +++ rust/src/engine-core-client/src/client/imp.rs | 28 +++- .../engine-core-client/src/tests/client.rs | 2 + .../src/llm/examples/external_engine_smoke.rs | 1 + rust/src/llm/src/lib.rs | 1 + rust/src/llm/src/request_metrics.rs | 77 +++++++--- rust/src/llm/tests/generate.rs | 140 +++++++++++++++++- rust/src/mock-engine/src/tests.rs | 1 + rust/src/server/src/config.rs | 4 +- rust/src/server/src/grpc/tests.rs | 1 + rust/src/server/src/lib.rs | 1 + 16 files changed, 248 insertions(+), 34 deletions(-) diff --git a/rust/src/bench/src/mm_processor.rs b/rust/src/bench/src/mm_processor.rs index 5b26181f16..3454dc7993 100644 --- a/rust/src/bench/src/mm_processor.rs +++ b/rust/src/bench/src/mm_processor.rs @@ -230,6 +230,7 @@ async fn run_mm_processor_with_engine( coordinator_mode: None, model_name: args.model.clone(), client_index: 0, + engine_stats_enabled: true, }) .await .context("failed to connect to engine core")?; diff --git a/rust/src/chat/examples/external_engine_chat_qwen.rs b/rust/src/chat/examples/external_engine_chat_qwen.rs index 4f62982036..1ba4d8a71a 100644 --- a/rust/src/chat/examples/external_engine_chat_qwen.rs +++ b/rust/src/chat/examples/external_engine_chat_qwen.rs @@ -70,6 +70,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: CLIENT_INDEX, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index c5a7c75baf..f57d43a75f 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -380,7 +380,9 @@ pub struct SharedRuntimeArgs { pub api_key: Vec, /// Disable periodic logging of engine statistics (throughput, queue depth, - /// cache usage). + /// cache usage). Engines also stop recording stats, so metrics derived from + /// engine-reported scheduler stats and request lifecycle events are not + /// exported. #[arg(long)] #[serde(default)] pub disable_log_stats: bool, diff --git a/rust/src/engine-core-client/examples/external_engine_logprobs.rs b/rust/src/engine-core-client/examples/external_engine_logprobs.rs index 891598c9f7..d4ec116f17 100644 --- a/rust/src/engine-core-client/examples/external_engine_logprobs.rs +++ b/rust/src/engine-core-client/examples/external_engine_logprobs.rs @@ -118,6 +118,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: args.client_index, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/engine-core-client/examples/external_engine_utility_call.rs b/rust/src/engine-core-client/examples/external_engine_utility_call.rs index b1826bb76e..8b73a12ce2 100644 --- a/rust/src/engine-core-client/examples/external_engine_utility_call.rs +++ b/rust/src/engine-core-client/examples/external_engine_utility_call.rs @@ -68,6 +68,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: args.client_index, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index aa6aacf0e0..0514292b90 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -155,6 +155,11 @@ pub struct EngineCoreClientConfig { pub model_name: String, /// Frontend client index stamped onto every request. pub client_index: u32, + /// Whether the connected engines record stats, i.e. were not started with + /// `--disable-log-stats`. When `false`, engines emit neither + /// `SchedulerStats` nor request lifecycle events, so frontend metrics + /// derived from them are not exported. + pub engine_stats_enabled: bool, } impl EngineCoreClientConfig { @@ -173,6 +178,7 @@ impl EngineCoreClientConfig { coordinator_mode: None, model_name: String::new(), client_index: 0, + engine_stats_enabled: true, } } @@ -193,6 +199,12 @@ impl EngineCoreClientConfig { self } + /// Set whether the connected engines record stats. + pub fn with_engine_stats_enabled(mut self, engine_stats_enabled: bool) -> Self { + self.engine_stats_enabled = engine_stats_enabled; + self + } + /// Override the optional coordinator mode for this client config. pub fn with_coordinator_mode(mut self, coordinator_mode: Option) -> Self { self.coordinator_mode = coordinator_mode; @@ -361,6 +373,7 @@ impl EngineCoreClient { connected.input_send, runtime.handle().clone(), config.model_name.clone(), + config.engine_stats_enabled, &engines, )); let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop( @@ -546,6 +559,11 @@ impl EngineCoreClient { self.inner.model_name() } + /// Return whether the connected engines record stats. + pub fn engine_stats_enabled(&self) -> bool { + self.config.engine_stats_enabled + } + /// Return whether the client still considers the engine healthy. pub fn is_healthy(&self) -> bool { self.inner.is_healthy() diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index 7318fa01b8..c8abb95bbd 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -46,7 +46,12 @@ pub(crate) struct ClientInner { /// Per-tensor byte threshold loaded from env variable /// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created. msgpack_zero_copy_threshold: usize, - scheduler_stats_recorder: SchedulerStatsRecorder, + /// Whether the connected engines record stats. See + /// [`crate::EngineCoreClientConfig::engine_stats_enabled`]. + engine_stats_enabled: bool, + /// Scheduler-stats metric handles, present only when engines record stats + /// so that gauges are not exported with values no engine ever reports. + scheduler_stats_recorder: Option, iteration_metrics: BTreeMap, request_reg: Mutex, utility_reg: Mutex, @@ -61,10 +66,11 @@ impl ClientInner { input_send: RouterSendHalf, handle: Handle, model_name: String, + engine_stats_enabled: bool, engines: &[ConnectedEngine], ) -> Self { - let scheduler_stats_recorder = - SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines); + let scheduler_stats_recorder = engine_stats_enabled + .then(|| SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines)); let iteration_metrics = engines .iter() .filter_map(|engine| { @@ -78,6 +84,7 @@ impl ClientInner { handle, model_name, msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(), + engine_stats_enabled, scheduler_stats_recorder, iteration_metrics, request_reg: Mutex::new(RequestRegistry::new(engines)), @@ -412,7 +419,9 @@ pub(crate) async fn run_output_dispatcher_loop( inner: Arc, mut output_rx: mpsc::Receiver>, ) { - let mut lora_info = LoraInfoExporter::default(); + // LoRA phases come from request lifecycle events, which engines only emit + // when they record stats. + let mut lora_info = inner.engine_stats_enabled.then(LoraInfoExporter::default); let result: Result<()> = async { loop { @@ -475,14 +484,18 @@ pub(crate) async fn run_output_dispatcher_loop( "dropping scheduler stats for unknown engine" ); } - inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats); + if let Some(recorder) = &inner.scheduler_stats_recorder { + recorder.record(batch.engine_index, scheduler_stats); + } } // The engine's scheduler stats never carry adapter names; // the gauge is derived from the registry's frontend-side // request tracking instead. - let (running, waiting) = inner.lora_adapter_states(); - lora_info.update(&METRICS.scheduler, running, waiting); + if let Some(lora_info) = &mut lora_info { + let (running, waiting) = inner.lora_adapter_states(); + lora_info.update(&METRICS.scheduler, running, waiting); + } } EngineCoreOutputs::Utility(utility) => { let call_id = utility.output.call_id; @@ -530,6 +543,7 @@ mod tests { send, Handle::current(), "test-model".to_string(), + true, &[ConnectedEngine { engine_id: EngineId::from(b"engine-0"), ready_response: default_ready_response(), diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index cdc3f718bd..5f2dc9c81e 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -299,6 +299,7 @@ fn handshake_test_config( coordinator_mode, model_name: model_name.to_string(), client_index, + engine_stats_enabled: true, } } @@ -322,6 +323,7 @@ fn bootstrapped_test_config( coordinator_mode, model_name: "test-model".to_string(), client_index, + engine_stats_enabled: true, } } diff --git a/rust/src/llm/examples/external_engine_smoke.rs b/rust/src/llm/examples/external_engine_smoke.rs index a28639e912..c965ea3038 100644 --- a/rust/src/llm/examples/external_engine_smoke.rs +++ b/rust/src/llm/examples/external_engine_smoke.rs @@ -122,6 +122,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: args.client_index, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/llm/src/lib.rs b/rust/src/llm/src/lib.rs index 5aa2479193..015624b409 100644 --- a/rust/src/llm/src/lib.rs +++ b/rust/src/llm/src/lib.rs @@ -106,6 +106,7 @@ impl Llm { prompt_len, max_tokens_param, 1, + self.client.engine_stats_enabled(), ); let guard = self.inflight.track(external_request_id, internal_request_id); diff --git a/rust/src/llm/src/request_metrics.rs b/rust/src/llm/src/request_metrics.rs index 43802eaf4a..98f9cadd1a 100644 --- a/rust/src/llm/src/request_metrics.rs +++ b/rust/src/llm/src/request_metrics.rs @@ -55,7 +55,6 @@ struct RequestMetricHandles { labels: EngineLabels, // Request-derived counters. - num_preemptions: U64Counter, prompt_tokens: U64Counter, prompt_tokens_local_compute: U64Counter, prompt_tokens_local_cache_hit: U64Counter, @@ -70,16 +69,27 @@ struct RequestMetricHandles { request_max_num_generation_tokens: HistogramMetric, request_params_max_tokens: HistogramMetric, request_params_n: HistogramMetric, - request_num_preemptions: HistogramMetric, request_prefill_kv_computed_tokens: HistogramMetric, time_to_first_token_seconds: HistogramMetric, inter_token_latency_seconds: HistogramMetric, e2e_request_latency_seconds: HistogramMetric, + request_decode_time_seconds: HistogramMetric, + request_time_per_output_token_seconds: HistogramMetric, + + /// Present only when engines record stats. + events: Option, +} + +/// Cached handles for request metrics derived from engine-core lifecycle +/// events (`QUEUED`, `SCHEDULED`, `PREEMPTED`), which engines only emit when +/// they record stats. +#[derive(Clone)] +struct EventMetricHandles { + num_preemptions: U64Counter, + request_num_preemptions: HistogramMetric, request_queue_time_seconds: HistogramMetric, request_prefill_time_seconds: HistogramMetric, - request_decode_time_seconds: HistogramMetric, request_inference_time_seconds: HistogramMetric, - request_time_per_output_token_seconds: HistogramMetric, } impl RequestMetricsTracker { @@ -92,9 +102,14 @@ impl RequestMetricsTracker { prompt_len: u32, max_tokens_param: Option, n_param: u32, + engine_stats_enabled: bool, ) -> Self { Self { - handles: resolve_request_metric_handles(&model_name, engine_index), + handles: resolve_request_metric_handles( + &model_name, + engine_index, + engine_stats_enabled, + ), arrival_time, prompt_len, max_tokens_param, @@ -163,10 +178,7 @@ impl RequestMetricsTracker { let prefill_kv_computed_tokens = self.prompt_len.saturating_sub(self.latest_num_cached_tokens); let e2e_latency_seconds = received_at - self.arrival_time; - let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts); - let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts); let decode_time_seconds = diff_or_zero(self.last_token_ts, self.first_token_ts); - let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts); let time_per_output_token_seconds = if self.num_generation_tokens > 1 { diff_or_zero(self.last_token_ts, self.first_token_ts) / (self.num_generation_tokens - 1) as f64 @@ -187,18 +199,25 @@ impl RequestMetricsTracker { self.handles.request_params_max_tokens.observe(max_tokens_param as f64); } self.handles.request_params_n.observe(self.n_param as f64); - self.handles.request_num_preemptions.observe(self.num_preemptions as f64); self.handles .request_prefill_kv_computed_tokens .observe(prefill_kv_computed_tokens as f64); self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds); - self.handles.request_queue_time_seconds.observe(queue_time_seconds); - self.handles.request_prefill_time_seconds.observe(prefill_time_seconds); self.handles.request_decode_time_seconds.observe(decode_time_seconds); - self.handles.request_inference_time_seconds.observe(inference_time_seconds); self.handles .request_time_per_output_token_seconds .observe(time_per_output_token_seconds); + + if let Some(events) = &self.handles.events { + let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts); + let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts); + let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts); + + events.request_num_preemptions.observe(self.num_preemptions as f64); + events.request_queue_time_seconds.observe(queue_time_seconds); + events.request_prefill_time_seconds.observe(prefill_time_seconds); + events.request_inference_time_seconds.observe(inference_time_seconds); + } } /// Record prompt token counters through cached metric handles. @@ -228,7 +247,9 @@ impl RequestMetricsTracker { } EngineCoreEventType::Preempted => { self.num_preemptions += 1; - self.handles.num_preemptions.inc(); + if let Some(events) = &self.handles.events { + events.num_preemptions.inc(); + } } } } @@ -248,7 +269,11 @@ impl RequestMetricsTracker { } /// Resolve fixed request metric handles for one model and engine index. -fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles { +fn resolve_request_metric_handles( + model_name: &str, + engine: u32, + engine_stats_enabled: bool, +) -> RequestMetricHandles { let metrics = &METRICS.request; let labels = EngineLabels { model_name: model_name.to_string(), @@ -256,7 +281,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri }; RequestMetricHandles { - num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels), prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels), prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned( &prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE), @@ -281,7 +305,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri .get_or_create_owned(&labels), request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels), request_params_n: metrics.request_params_n.get_or_create_owned(&labels), - request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels), request_prefill_kv_computed_tokens: metrics .request_prefill_kv_computed_tokens .get_or_create_owned(&labels), @@ -294,19 +317,25 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri e2e_request_latency_seconds: metrics .e2e_request_latency_seconds .get_or_create_owned(&labels), - request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels), - request_prefill_time_seconds: metrics - .request_prefill_time_seconds - .get_or_create_owned(&labels), request_decode_time_seconds: metrics .request_decode_time_seconds .get_or_create_owned(&labels), - request_inference_time_seconds: metrics - .request_inference_time_seconds - .get_or_create_owned(&labels), request_time_per_output_token_seconds: metrics .request_time_per_output_token_seconds .get_or_create_owned(&labels), + events: engine_stats_enabled.then(|| EventMetricHandles { + num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels), + request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels), + request_queue_time_seconds: metrics + .request_queue_time_seconds + .get_or_create_owned(&labels), + request_prefill_time_seconds: metrics + .request_prefill_time_seconds + .get_or_create_owned(&labels), + request_inference_time_seconds: metrics + .request_inference_time_seconds + .get_or_create_owned(&labels), + }), labels, } } @@ -356,7 +385,7 @@ mod tests { #[test] fn tracker_updates_timing_state_across_prefill_decode_and_finish() { let mut tracker = - RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1); + RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1, true); tracker.observe_output( 10.0, diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index efd48eba77..c596c89ba3 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -6,12 +6,16 @@ use std::sync::{Arc, Once}; use std::time::Duration; use futures::StreamExt as _; +use tokio::sync::oneshot; use tokio::time::timeout; use tracing_subscriber::EnvFilter; use uuid::Uuid; +use vllm_engine_core_client::mock_engine::default_ready_response; +use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse; use vllm_engine_core_client::protocol::logprobs::{ Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob, }; +use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::output::{ EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, @@ -20,7 +24,9 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; use vllm_engine_core_client::protocol::sampling_mask::{MaybeWireSamplingMask, SamplingMask}; use vllm_engine_core_client::protocol::stats::PrefillStats; -use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; +use vllm_engine_core_client::test_utils::{ + IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready, +}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::{ Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm, @@ -868,6 +874,138 @@ async fn generate_records_request_metrics_in_prometheus_output() { assert!(rendered.contains(&format!( "vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1" ))); + assert!(rendered.contains(&format!( + "vllm:request_queue_time_seconds_sum{{model_name=\"{model_name}\",engine=\"4\"}} 1.0" + ))); + + let _ = shutdown_tx.send(()); + engine_task.await.unwrap(); + llm.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn generate_without_engine_stats_exports_only_frontend_observed_metrics() { + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + let engine_id = EngineId::from_engine_index(6); + let model_name = request_metrics_model_name("metrics-no-log-stats-model"); + let lora_name = request_metrics_model_name("metrics-no-log-stats-lora"); + let (finish_tx, finish_rx) = oneshot::channel::<()>(); + + let (shutdown_tx, engine_task) = spawn_mock_engine_task_with_ready( + handshake_address.clone(), + engine_id, + EngineCoreReadyResponse { + supports_lora: true, + max_loras: 1, + ..default_ready_response() + }, + move |dealer, push| { + Box::pin(async move { + let add = recv_engine_message(dealer).await; + assert_eq!(add[0].as_ref(), &[0x00]); + let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap(); + + // Engines started with `--disable-log-stats` attach neither + // lifecycle events nor scheduler stats to their outputs. + for token in [1, 2] { + send_outputs( + push, + RequestBatchOutputs { + engine_index: 6, + timestamp: 10.0 + f64::from(token), + outputs: vec![request_output(&request.request_id, vec![token], None)], + ..Default::default() + } + .into(), + ) + .await; + } + + finish_rx.await.unwrap(); + send_outputs( + push, + RequestBatchOutputs { + engine_index: 6, + timestamp: 13.0, + outputs: vec![request_output( + &request.request_id, + vec![3], + Some(EngineCoreFinishReason::Length), + )], + finished_requests: Some(BTreeSet::from([request.request_id])), + ..Default::default() + } + .into(), + ) + .await; + }) + }, + ); + + let client = EngineCoreClient::connect( + EngineCoreClientConfig::new_single(handshake_address) + .with_model_name(&model_name) + .with_engine_stats_enabled(false) + .with_local_input_output_addresses( + Some(ipc.input_endpoint()), + Some(ipc.output_endpoint()), + ), + ) + .await + .unwrap(); + let llm = Llm::new(client); + let mut request = sample_generate_request("req-no-log-stats", 8); + request.arrival_time = None; + request.lora_request = Some(LoraRequest { + lora_name: lora_name.clone(), + lora_int_id: 1, + lora_path: "/unused".to_string(), + base_model_name: None, + tensorizer_config_dict: None, + load_inplace: false, + is_3d_lora_weight: false, + }); + let mut stream = llm.generate(request).await.unwrap(); + + assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]); + // The dispatcher handles the second batch only after finishing the first + // one, including its LoRA info update. + assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![2]); + assert!(!METRICS.render().unwrap().contains(&lora_name)); + + finish_tx.send(()).unwrap(); + let final_output = stream.next().await.unwrap().unwrap(); + assert_eq!(final_output.finish_reason, Some(FinishReason::Length)); + assert!(stream.next().await.is_none()); + + let rendered = METRICS.render().unwrap(); + for metric in [ + "vllm:time_to_first_token_seconds_count", + "vllm:e2e_request_latency_seconds_count", + "vllm:request_decode_time_seconds_count", + ] { + assert!( + rendered.contains(&format!( + "{metric}{{model_name=\"{model_name}\",engine=\"6\"}} 1" + )), + "{metric} should be exported" + ); + } + for metric in [ + "vllm:num_requests_running", + "vllm:kv_cache_usage_perc", + "vllm:num_preemptions_total", + "vllm:request_num_preemptions_count", + "vllm:request_queue_time_seconds_count", + "vllm:request_prefill_time_seconds_count", + "vllm:request_inference_time_seconds_count", + ] { + assert!( + !rendered.contains(&format!("{metric}{{model_name=\"{model_name}\"")), + "{metric} should not be exported" + ); + } let _ = shutdown_tx.send(()); engine_task.await.unwrap(); diff --git a/rust/src/mock-engine/src/tests.rs b/rust/src/mock-engine/src/tests.rs index c1cfbb49cb..71b4286033 100644 --- a/rust/src/mock-engine/src/tests.rs +++ b/rust/src/mock-engine/src/tests.rs @@ -29,6 +29,7 @@ fn client_config(handshake_address: String, engine_count: usize) -> EngineCoreCl coordinator_mode: None, model_name: "mock-model".to_string(), client_index: 0, + engine_stats_enabled: true, } } diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index c1c580e75a..a9a77a9a9e 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -268,7 +268,9 @@ pub struct Config { #[educe(Debug(method(fmt_redacted_api_keys)))] pub api_keys: Vec, /// When `true`, suppress periodic stats logging (throughput, queue depth, - /// cache usage). + /// cache usage). Engines also stop recording stats, so metrics derived from + /// engine-reported scheduler stats and request lifecycle events are not + /// exported. pub disable_log_stats: bool, /// TCP port for the gRPC Inference service. When `None`, no gRPC server is /// started. diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 18b6e6eb29..21b240c4b1 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -2234,6 +2234,7 @@ async fn control_aggregates_multi_engine_capacity() { coordinator_mode: None, model_name: "test-model".to_string(), client_index: 0, + engine_stats_enabled: true, }; let client_task = tokio::spawn(EngineCoreClient::connect(client_config)); let mut engine_sockets = Vec::new(); diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 91564db73c..d5f9e1ea55 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -130,6 +130,7 @@ async fn build_state(config: &Config) -> Result> { coordinator_mode, model_name: metrics_model_name, client_index: 0, + engine_stats_enabled: !config.disable_log_stats, }) .await .context("failed to connect to engine core")?; From 467d81d9a098c5f6695c67832a372a7ad8cd835a Mon Sep 17 00:00:00 2001 From: Chaitanya Sri Krishna Lolla Date: Tue, 29 Sep 2026 23:37:31 +0530 Subject: [PATCH 041/248] [ROCm] Upgrade MoRI version on rocm dockers required for WideEP DP16 DI CI enablement and fixes for combine API (#56073) Signed-off-by: Shiksha Patel Signed-off-by: lcskrishna Co-authored-by: shikamd123 Co-authored-by: Andreas Karatzas --- docker/Dockerfile.rock_base | 2 +- docker/Dockerfile.rocm_base | 2 +- .../kernels/moe/test_mori_prepare_finalize.py | 128 ++++++++++++++++++ .../layers/fused_moe/prepare_finalize/mori.py | 13 +- 4 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 tests/kernels/moe/test_mori_prepare_finalize.py diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index d3ba0bc842..54025dd508 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -10,7 +10,7 @@ ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" ARG AITER_BRANCH="v0.1.23" ARG AITER_REPO="https://github.com/ROCm/aiter.git" -ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0 +ARG MORI_BRANCH="v1.2.3.post1" ARG MORI_REPO="https://github.com/ROCm/mori.git" ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" ARG ROCM_RUNTIME_COMMIT="25e14349f2606a6b95c4ccd45eb92a1835646cf2" diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index 1c59968005..15e3e2468e 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -11,7 +11,7 @@ ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" ARG AITER_BRANCH="v0.1.23" ARG AITER_REPO="https://github.com/ROCm/aiter.git" -ARG MORI_BRANCH="v1.1.0" +ARG MORI_BRANCH="v1.2.3.post1" ARG MORI_REPO="https://github.com/ROCm/mori.git" ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350" diff --git a/tests/kernels/moe/test_mori_prepare_finalize.py b/tests/kernels/moe/test_mori_prepare_finalize.py new file mode 100644 index 0000000000..c5deffe725 --- /dev/null +++ b/tests/kernels/moe/test_mori_prepare_finalize.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import importlib +import sys +import types +from types import SimpleNamespace + +import torch + + +def _describe(tensor): + if tensor is None: + return "None (the op was never called)" + return f"a different tensor of shape {tuple(tensor.shape)}" + + +def _import_mori_prepare_finalize(monkeypatch): + # This is a contract test for vLLM's adapter; it does not need a real MoRI + # installation or GPU kernels. + fake_mori = types.ModuleType("mori") + fake_mori.ops = SimpleNamespace( # type: ignore[attr-defined] + EpDispatchCombineOp=object + ) + monkeypatch.setitem(sys.modules, "mori", fake_mori) + module_name = "vllm.model_executor.layers.fused_moe.prepare_finalize.mori" + sys.modules.pop(module_name, None) + return importlib.import_module(module_name) + + +class _FakeMoriOp: + def __init__(self): + self.dispatch_indices = None + self.combine_indices = None + + def dispatch(self, a1, weights, scale, indices): + self.dispatch_indices = indices + # Model the global dispatched layout seen with EP=16: the experts need + # these IDs, but passing them to combine violates MoRI's contract. + dispatch_indices = indices.repeat(16, 1) + dispatch_a1 = a1.repeat(16, 1) + dispatch_weights = weights.repeat(16, 1) + recv_token_num = torch.tensor([dispatch_a1.shape[0]]) + return ( + dispatch_a1, + dispatch_weights, + scale, + dispatch_indices, + recv_token_num, + ) + + def combine(self, fused_expert_output, weights, indices): + self.combine_indices = indices + return (fused_expert_output,) + + +def test_mori_combine_uses_original_per_rank_topk_ids(monkeypatch): + mori_module = _import_mori_prepare_finalize(monkeypatch) + mori_op = _FakeMoriOp() + adapter = mori_module.MoriPrepareAndFinalize( + mori_op=mori_op, + max_tokens_per_rank=16, + num_dispatchers=1, + ) + + hidden_states = torch.randn(2, 4) + router_topk_ids = torch.tensor([[0, 1], [2, 3]], dtype=torch.int32) + topk_weights = torch.ones(2, 2) + quant_config = SimpleNamespace( + is_block_quantized=False, + is_per_act_token=False, + ) + + ( + dispatch_a1, + _, + _, + dispatch_ids, + dispatch_weights, + ) = adapter.prepare( + hidden_states, + topk_weights, + router_topk_ids, + num_experts=4, + expert_map=None, + apply_router_weight_on_input=False, + quant_config=quant_config, + ) + + # Read the shapes before the identity asserts below, which narrow these + # names to the fake op's Optional attribute type. + router_shape = tuple(router_topk_ids.shape) + dispatch_shape = tuple(dispatch_ids.shape) + expected_rows = router_shape[0] * 16 + + assert dispatch_shape[0] == expected_rows, ( + "prepare() must return the ids dispatch() produced: EP=16 expands " + f"{router_shape[0]} tokens to {expected_rows} rows, but prepare() " + f"returned {dispatch_shape[0]} rows" + ) + assert mori_op.dispatch_indices is router_topk_ids, ( + "dispatch() must be handed this rank's own pre-dispatch routing of " + f"shape {router_shape}, but it received " + f"{_describe(mori_op.dispatch_indices)}" + ) + + output = torch.empty_like(hidden_states) + adapter.finalize( + output, + dispatch_a1, + dispatch_weights, + dispatch_ids, + apply_router_weight_on_input=False, + weight_and_reduce_impl=SimpleNamespace(), + ) + + assert mori_op.combine_indices is router_topk_ids, ( + "combine() reduces over this rank's own routing, so it must be handed " + f"the pre-dispatch tensor of shape {router_shape}, but it received " + f"{_describe(mori_op.combine_indices)}. finalize() most likely " + "forwarded its own topk_ids, which the modular kernel has already " + "rebound to the ids dispatch() returned (ROCm/mori#475)" + ) + assert mori_op.combine_indices is not dispatch_ids, ( + f"combine() was handed the dispatched ids of shape {dispatch_shape} " + "instead of this rank's pre-dispatch routing, which silently corrupts " + "the reduction (ROCm/mori#475)" + ) diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py index 75b97c69aa..743235bc7e 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py @@ -27,6 +27,7 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): self.num_dispatchers_ = num_dispatchers self.max_tokens_per_rank = max_tokens_per_rank self.use_fp8_dispatch = use_fp8_dispatch + self._dispatch_topk_ids: torch.Tensor | None = None @property def activation_format(self) -> mk.FusedMoEActivationFormat: @@ -92,6 +93,12 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): # mori expects one scale slot per token; broadcast. scale = scale.expand(a1.shape[0], 1).contiguous() + # mori's combine() reduces over this rank's own [num_tokens, topk] + # routing. The modular kernel rebinds topk_ids to the dispatched ids + # returned below before it calls finalize(), so hold a reference to + # the pre-dispatch tensor (ROCm/mori#475). + self._dispatch_topk_ids = topk_ids + ( dispatch_a1, dispatch_weights, @@ -122,9 +129,13 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): weight_and_reduce_impl: mk.TopKWeightAndReduce, ) -> None: num_token = output.shape[0] + assert self._dispatch_topk_ids is not None, ( + "finalize() called before prepare(); mori combine() needs the " + "routing tensor that dispatch() was given" + ) result = self.mori_op.combine( fused_expert_output, None, - topk_ids, + self._dispatch_topk_ids, )[0] output.copy_(result[:num_token]) From 9a190f142c8acccffd0b19d2393e1eb76f775aea Mon Sep 17 00:00:00 2001 From: Ashraf Bhuiyan Date: Tue, 29 Sep 2026 14:13:52 -0400 Subject: [PATCH 042/248] [Mypy] Fix mypy typing for Zamba2 models (#58255) Signed-off-by: Ashraf Bhuiyan Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tools/pre_commit/mypy.py | 1 - vllm/model_executor/models/zamba2.py | 25 +++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 5ad37e138e..88015c6a53 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[zZ]", ] diff --git a/vllm/model_executor/models/zamba2.py b/vllm/model_executor/models/zamba2.py index 9398d99768..6506c0776e 100644 --- a/vllm/model_executor/models/zamba2.py +++ b/vllm/model_executor/models/zamba2.py @@ -86,16 +86,21 @@ class Zamba2LoRA(nn.Module): ) if isinstance(output_dim, list): - B_class = MergedColumnParallelLinear + self.B: ColumnParallelLinear = MergedColumnParallelLinear( + rank, + output_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.B", + ) else: - B_class = ColumnParallelLinear - self.B = B_class( - rank, - output_dim, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.B", - ) + self.B = ColumnParallelLinear( + rank, + output_dim, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.B", + ) def forward( self, @@ -304,7 +309,7 @@ class Zamba2MLP(nn.Module): self, config: Zamba2Config, bare_block_idx: int, - num_hybrid_layers: dict[int, int], + num_hybrid_layers: int, quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: From 91de237333e5f72b12e4de9f535e5774f9bfb1de Mon Sep 17 00:00:00 2001 From: Chanbin Lim <57864041+beenpow@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:15:13 -0700 Subject: [PATCH 043/248] [Bugfix][DP] add_dp_placement_groups does not require ray[default] (#57648) Signed-off-by: beenpow <57864041+beenpow@users.noreply.github.com> --- .../engine/test_core_engine_actor_manager.py | 109 ++++++++++++++++++ vllm/v1/engine/utils.py | 61 ++++++---- 2 files changed, 149 insertions(+), 21 deletions(-) diff --git a/tests/v1/engine/test_core_engine_actor_manager.py b/tests/v1/engine/test_core_engine_actor_manager.py index 69f52e2a7f..f1a67ad9ba 100644 --- a/tests/v1/engine/test_core_engine_actor_manager.py +++ b/tests/v1/engine/test_core_engine_actor_manager.py @@ -20,6 +20,8 @@ from vllm.v1.engine.core_client import BackgroundResources from vllm.v1.engine.utils import ( CoreEngineActorManager, EngineZmqAddresses, + _dp_nodes_master_first, + _node_ip_from_resources, get_engine_zmq_addresses, launch_core_engines, ) @@ -363,3 +365,110 @@ def test_ray_dp_addresses_resolved_before_actor_creation( "time they DEALER-connect. See PR #42585 / Ray-DP " "multi-API-server regression." ) + + +def test_dp_nodes_master_first_orders_and_validates(): + master = {"GPU": 2.0, "node:10.0.0.1": 1.0, "node:__internal_head__": 1.0} + worker = {"GPU": 2.0, "node:10.0.0.2": 1.0} + + nodes = _dp_nodes_master_first({"w": worker, "m": master}, "10.0.0.1") + assert [node_id for node_id, _ in nodes] == ["m", "w"] + + with pytest.raises(AssertionError, match="DP master node"): + _dp_nodes_master_first({"w": worker}, "10.0.0.1") + with pytest.raises(AssertionError, match="one head node"): + _dp_nodes_master_first({"a": master, "b": dict(master)}, "10.0.0.1") + + +@pytest.fixture +def ray_2gpu_node(monkeypatch: pytest.MonkeyPatch): + """Real single-node Ray with 2 virtual GPUs and no dashboard. + + Resource maps and placement groups are real. Without the dashboard any + ``ray.util.state.list_nodes()`` call fails, the production condition this + path must survive. Only the platform device key is patched ("" on CPU). + """ + from ray._private.state import available_resources_per_node + + if ray.is_initialized(): + ray.shutdown() + ray.init(num_cpus=4, num_gpus=2, include_dashboard=False, log_to_driver=False) + monkeypatch.setattr("vllm.v1.engine.utils.current_platform.ray_device_key", "GPU") + + (node_resources,) = available_resources_per_node().values() + master_ip = _node_ip_from_resources(node_resources) + assert master_ip is not None + created = [] + + def hold_gpus(num_gpus: int): + """Occupy GPUs so the node looks like it already runs engines.""" + pg = ray.util.placement_group([{"GPU": 1.0}] * num_gpus) + ray.get(pg.ready(), timeout=60) + created.append(pg) + + yield SimpleNamespace(master_ip=master_ip, hold_gpus=hold_gpus, created=created) + + for pg in created: + ray.util.remove_placement_group(pg) + ray.shutdown() + + +def _elastic_ep_config(dp_size: int, world_size: int, master_ip: str): + return SimpleNamespace( + parallel_config=SimpleNamespace( + data_parallel_size=dp_size, + data_parallel_master_ip=master_ip, + world_size=world_size, + ) + ) + + +@pytest.mark.timeout(120) +def test_add_dp_placement_groups_does_not_require_ray_default(ray_2gpu_node): + """DP 1 -> 2 on an idle node: one schedulable group pinned to the master, + found without the dashboard (``list_nodes`` would fail here).""" + ip = ray_2gpu_node.master_ip + + pgs, local_ranks = CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=1, world_size=1, master_ip=ip), 2 + ) + ray_2gpu_node.created.extend(pgs) + + assert [pg.bundle_specs for pg in pgs] == [ + [{"GPU": 1.0, f"node:{ip}": 0.001}, {"CPU": 1.0}] + ] + assert local_ranks == [0] + ray.get(pgs[0].ready(), timeout=60) + + +@pytest.mark.timeout(120) +def test_add_dp_placement_groups_counts_engines_already_on_node(ray_2gpu_node): + """One GPU already held -> the new engine gets local rank 1.""" + ray_2gpu_node.hold_gpus(1) + + pgs, local_ranks = CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=1, world_size=1, master_ip=ray_2gpu_node.master_ip), + 2, + ) + ray_2gpu_node.created.extend(pgs) + + assert len(pgs) == 1 + assert local_ranks == [1] + ray.get(pgs[0].ready(), timeout=60) + + +@pytest.mark.timeout(120) +def test_add_dp_placement_groups_respects_world_size(ray_2gpu_node): + """With TP=2 a node with one idle GPU cannot host a new engine.""" + ray_2gpu_node.hold_gpus(1) + + assert CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=1, world_size=2, master_ip=ray_2gpu_node.master_ip), + 2, + ) == ([], []) + + +def test_add_dp_placement_groups_noop_without_growth(): + assert CoreEngineActorManager.add_dp_placement_groups( + _elastic_ep_config(dp_size=2, world_size=1, master_ip="10.0.0.1"), 2 + ) == ([], []) diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index 02b49819fd..e786cf2c5b 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -124,6 +124,31 @@ def _get_bundle_node_ip(bundle: dict[str, float]) -> str: raise ValueError(f"Missing node affinity in placement bundle: {bundle}") +def _dp_nodes_master_first( + available_resources: dict[str, dict], dp_master_ip: str +) -> list[tuple[str, dict]]: + """Live Ray nodes as ``(node_id, resources)`` with the DP master first. + + The master is found through the ``node:`` resource Ray sets on every + node, so this works with the resource maps from ``ray._private.state``. + Do not replace it with ``ray.util.state.list_nodes()``: that API goes + through the dashboard HTTP server and only exists with ``ray[default]`` + (see #23822). + """ + master_key = f"node:{dp_master_ip}" + nodes = sorted( + available_resources.items(), key=lambda item: master_key not in item[1] + ) + assert len(nodes) > 0, "No nodes with resources found in Ray cluster." + assert master_key in nodes[0][1], ( + f"The DP master node (ip: {dp_master_ip}) is missing or dead" + ) + assert len(nodes) == 1 or master_key not in nodes[1][1], ( + "There can only be one head node" + ) + return nodes + + def _node_ip_from_resources(node_resources: dict) -> str | None: """Return the node IP encoded in a Ray per-node resource dict, or None. @@ -561,14 +586,12 @@ class CoreEngineActorManager: placement_groups: list[PlacementGroup] = [] local_dp_ranks: list[int] = [] - dp_master_ip_key = f"node:{dp_master_ip}" - nodes = sorted( - available_resources.values(), key=lambda x: dp_master_ip_key not in x - ) - assert len(nodes) > 0, "No nodes with resources found in Ray cluster." - assert dp_master_ip_key in nodes[0], ( - f"The DP master node (ip: {dp_master_ip}) is missing or dead" - ) + nodes = [ + node_resources + for _, node_resources in _dp_nodes_master_first( + available_resources, dp_master_ip + ) + ] # optionally restrict DP placement to a caller-provided node set. requested_node_ips = { @@ -767,7 +790,6 @@ class CoreEngineActorManager: available_resources_per_node, total_resources_per_node, ) - from ray.util.state import list_nodes old_dp_size = old_vllm_config.parallel_config.data_parallel_size num_pg_to_create = new_data_parallel_size - old_dp_size @@ -778,30 +800,27 @@ class CoreEngineActorManager: dp_master_ip = old_vllm_config.parallel_config.data_parallel_master_ip world_size = old_vllm_config.parallel_config.world_size - nodes = list_nodes() - nodes = sorted(nodes, key=lambda node: node.node_ip != dp_master_ip) - assert nodes[0].node_ip == dp_master_ip, "The first node must be the head node" - assert len(nodes) == 1 or nodes[1].node_ip != dp_master_ip, ( - "There can only be one head node" - ) - + # Both maps are keyed by node id and only contain live nodes. available_resources = available_resources_per_node() total_resources = total_resources_per_node() + nodes = _dp_nodes_master_first(available_resources, dp_master_ip) placement_groups = [] local_dp_ranks = [] num_pg_created = 0 device_str = current_platform.ray_device_key - for node in nodes: + for node_id, node_resources in nodes: if num_pg_created >= num_pg_to_create: break - node_ip = node.node_ip - node_id = node.node_id - if device_str not in available_resources[node_id]: + node_ip = _node_ip_from_resources(node_resources) + assert node_ip is not None, ( + f"No node IP key found in node resources: {node_resources}" + ) + if device_str not in node_resources: continue - available_gpus = int(available_resources[node_id][device_str]) + available_gpus = int(node_resources[device_str]) # Get total GPUs on this node from the node's resources # Ray stores node resources with node ID as key From df4dbe46e7521e073a63c68247245323d86de9d5 Mon Sep 17 00:00:00 2001 From: djramic Date: Tue, 29 Sep 2026 20:25:49 +0200 Subject: [PATCH 044/248] [CI][ROCm] Increase timeouts for AMD MI300 jobs near their limits (#59227) Signed-off-by: Djordje Ramic Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 12 ++++++------ .buildkite/test_areas/models_multimodal.yaml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index b39939b235..a6e2001aef 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -773,7 +773,7 @@ steps: - bash standalone_tests/pytorch_nightly_dependency.sh - label: ":amd: (MI300) Distributed Compile (Inductor Partition)" # TBD - timeout_in_minutes: 65 + timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 @@ -1124,7 +1124,7 @@ steps: - pytest -v -s v1/worker/test_worker_memory_snapshot.py - label: ":amd: (MI300) Distributed Compile + Comm" # TBD - timeout_in_minutes: 45 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -2035,7 +2035,7 @@ steps: #----------------------------------------------------- mi300 · models / language -----------------------------------------------------# - label: ":amd: (MI300) Language Models (MTEB)" # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2062,7 +2062,7 @@ steps: - pytest -v -s models/language/pooling -m 'not core_model' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: ":amd: (MI300) Language Models (Extended Generation)" # TBD - timeout_in_minutes: 95 + timeout_in_minutes: 105 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2079,7 +2079,7 @@ steps: #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" # TBD - timeout_in_minutes: 90 + timeout_in_minutes: 100 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -2168,7 +2168,7 @@ steps: #----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------# - label: ":amd: (MI300) Quantized Models" # TBD - timeout_in_minutes: 80 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index e07f38a7bd..82c77f58ef 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -176,7 +176,7 @@ steps: label: ":amd: (MI300) Multimodal Models (Extended Generation 1)" dind: false device: mi300_1 - timeout_in_minutes: 90 + timeout_in_minutes: 100 depends_on: - image-build-amd From 4dc57d44c4a790a210ad17cde2e1af98146f8bca Mon Sep 17 00:00:00 2001 From: Shimi Bandiel Date: Tue, 29 Sep 2026 11:43:44 -0700 Subject: [PATCH 045/248] [Frontend] Attach resolved logprobs to streaming derender chunks (#55029) Signed-off-by: Shimi Bandiel Co-authored-by: Claude --- docs/serving/online_serving/derenderer.md | 11 +- .../derender/test_derender_stream.py | 127 ++++++++++++++++++ .../scale_out/token_in_token_out/protocol.py | 11 ++ vllm/renderers/online_derenderer.py | 83 +++++++++++- 4 files changed, 224 insertions(+), 8 deletions(-) diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index 0e865796f8..e6d1221fd4 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -98,7 +98,7 @@ The server keeps no state between calls. Everything the next call needs is in `s - **Send the same context on every call.** `chat_request` and `prompt_token_ids` aren't kept between calls, so they go with every chunk including the usage chunk. Both are required when a tool or reasoning parser is configured. `prompt_token_ids` is the `token_ids` of the `GenerateRequest` returned by `/render`. - **Don't forward `[DONE]`.** It marks the end of the generate stream and isn't a chunk. -Streaming chunks don't carry logprobs yet, so `logprobs` on a generate chunk are dropped. The non-streaming endpoints do resolve them, including `token_id:N` placeholders. +Streaming derender resolves `logprobs` on the plain detokenization path, including `token_id:N` placeholders, same as the non-streaming endpoints (see [Streaming state and logprobs](#streaming-state-and-logprobs)). The parser path doesn't resolve them yet and drops them. ## Streaming cost @@ -115,6 +115,15 @@ When a tool or reasoning parser is configured, parser internal state (buffered m `output_token_ids` and `prompt_token_ids` are both bounded by `max_model_len` but callers streaming long reasoning traces through a parser configured model should expect materially more state transport and CPU cost than the plain detokenization path. +## Streaming state and logprobs + +Streaming derender is stateless on the server side: all mutable state lives in the client-carried `stream_state` (`DerenderStreamState`), passed back on every per-chunk call. Besides the bounded incremental detokenization window (`prev_tokens`, `prefix_offset`, `read_offset`), `role_sent`, and the parser path's replay fields (see [Streaming cost](#streaming-cost)), the state carries two fields for logprob handling: + +- `logprob_context_token_ids`: the trailing sampled token IDs (at most 4) from previous chunks, used to seed byte-fallback (U+FFFD) correction so multi-byte characters whose tokens split across chunk boundaries still resolve to real strings +- `logprob_text_offset`: the cumulative emitted text length, so `text_offset` in completion streaming logprobs stays absolute across chunks instead of restarting at 0 + +When a streamed `GenerateResponseStreamChoice` carries `logprobs`, the `token_id:N` placeholders are resolved per chunk and the resolved logprobs are attached to the corresponding streamed choice — for chat as `ChatCompletionLogProbs`, for completions converted to the flat `CompletionLogProbs` lists. Chunks without `logprobs` produce choices with `logprobs: null`. + ## Example The example below drives the full `render → generate → derender` round trip for a chat request against a GPU less render server (`/render`, `/derender`) and a token-in / token-out engine (`/inference/v1/generate`). diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 8510d63c4b..63ee9bcad8 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -169,6 +169,7 @@ def _make_stream_chunk( request_id: str = "test-req", usage: dict | None = None, metrics: PerRequestMetrics | None = None, + logprobs: dict | None = None, ) -> GenerateStreamResponse: """Build a GenerateStreamResponse SSE chunk.""" return GenerateStreamResponse( @@ -178,6 +179,7 @@ def _make_stream_chunk( index=index, token_ids=token_ids, finish_reason=finish_reason, + logprobs=logprobs, ) ], usage=UsageInfo(**usage) if usage else None, @@ -185,6 +187,22 @@ def _make_stream_chunk( ) +def _placeholder_logprobs(token_ids: list[int]) -> dict: + """Per-token logprob entries using token_id:N placeholders, as sent by + the generate worker.""" + return { + "content": [ + { + "token": f"token_id:{tid}", + "logprob": -0.5, + "bytes": None, + "top_logprobs": [], + } + for tid in token_ids + ] + } + + def _make_usage_chunk( completion_tokens: int, prompt_tokens: int = 0, @@ -536,6 +554,115 @@ class TestDerenderCompletionStream: assert chunk.choices[0].finish_reason == "length" +class TestStreamLogprobs: + """Streaming derender must carry per-chunk logprobs with placeholders + resolved, matching what the generate streaming path emits.""" + + @pytest.mark.asyncio + async def test_chat_stream_logprobs_resolved_per_chunk(self, derenderer, tokenizer): + """Each streamed chunk carries logprobs with token_id:N resolved.""" + token_ids = tokenizer.encode("hello world")[:6] + mid = len(token_ids) // 2 + + state = None + for part in (token_ids[:mid], token_ids[mid:]): + chunk, state = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + part, logprobs=_placeholder_logprobs(part) + ), + state=state, + ) + logprobs = chunk.choices[0].logprobs + assert logprobs is not None and logprobs.content is not None + assert len(logprobs.content) == len(part) + for entry in logprobs.content: + assert not entry.token.startswith("token_id:"), ( + f"placeholder not resolved: {entry.token!r}" + ) + + @pytest.mark.asyncio + async def test_chat_stream_logprobs_multibyte_across_chunks( + self, derenderer, tokenizer + ): + """Byte-fallback correction works when a multi-byte character's + tokens are split across chunks (context carried in stream_state).""" + token_ids = tokenizer.encode("👍", add_special_tokens=False) + if len(token_ids) < 2 or "�" not in tokenizer.decode([token_ids[-1]]): + pytest.skip("Tokenizer does not byte-split this character") + + state = None + last_chunk = None + # One token per chunk: every entry after the first needs cross-chunk + # context to resolve without U+FFFD. + for i, tid in enumerate(token_ids): + last_chunk, state = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [tid], logprobs=_placeholder_logprobs([tid]) + ), + state=state, + ) + + assert last_chunk is not None + final_entry = last_chunk.choices[0].logprobs.content[0] + assert not final_entry.token.endswith("�"), ( + "byte-fallback correction failed across chunk boundary: " + f"{final_entry.token!r}" + ) + + @pytest.mark.asyncio + async def test_completion_stream_logprobs_text_offset_absolute( + self, derenderer, tokenizer + ): + """text_offset continues across chunks instead of restarting at 0.""" + token_ids = tokenizer.encode("hello world")[:6] + mid = len(token_ids) // 2 + + chunk1, state = await derenderer.derender_completion_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + token_ids[:mid], logprobs=_placeholder_logprobs(token_ids[:mid]) + ), + ) + chunk2, _ = await derenderer.derender_completion_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + token_ids[mid:], logprobs=_placeholder_logprobs(token_ids[mid:]) + ), + state=state, + ) + + assert chunk1.choices[0].logprobs.text_offset[0] == 0 + assert chunk2.choices[0].logprobs.text_offset[0] == len(chunk1.choices[0].text) + + @pytest.mark.asyncio + async def test_stream_logprob_state_stays_bounded(self, derenderer, tokenizer): + """Carried logprob context never exceeds the 4-token window.""" + token_ids = tokenizer.encode("the quick brown fox jumps over the lazy dog") + state = None + for tid in token_ids: + _, state = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + [tid], logprobs=_placeholder_logprobs([tid]) + ), + state=state, + ) + assert state is not None + assert len(state.logprob_context_token_ids) <= 4 + + @pytest.mark.asyncio + async def test_stream_without_logprobs_unchanged(self, derenderer, tokenizer): + """Chunks without logprobs keep logprobs=None on the output choice.""" + token_ids = tokenizer.encode("hello")[:3] + chunk, _ = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk(token_ids), + ) + assert chunk.choices[0].logprobs is None + + class TestDerenderChatStream: """derender_chat_stream: plain detok branch (no parser).""" diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index ad607648bd..94604b013f 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -546,6 +546,17 @@ class DerenderStreamState(BaseModel): window is transiently empty (e.g. usage only final chunk). """ + logprob_context_token_ids: list[int] = Field(default_factory=list, max_length=4) + """Trailing sampled token IDs carried across chunks so byte-fallback + (U+FFFD) correction during logprob placeholder resolution has context + at chunk boundaries. Bounded to the 4-token window that + ``_correct_decoded_token`` reads.""" + + logprob_text_offset: int = Field(default=0, ge=0) + """Cumulative emitted text length. Seeds ``text_offset`` for completion + streaming logprobs so offsets stay absolute across chunks, mirroring + ``initial_text_offset`` in the generate streaming path.""" + output_token_ids: list[int] = Field(default_factory=list) """All output tokens seen so far. Parser path only. diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 6f7cd10758..e1f67397ce 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Sequence from typing import Any from vllm.config import ModelConfig @@ -368,9 +369,30 @@ class OnlineDerenderer: tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special ) + # NOTE: parser-configured servers dispatch to + # _derender_chat_stream_parsed above and never reach this plain + # path. That parsed path does not resolve logprobs yet; when it + # does, mirror the generate chat streaming path, which suppresses + # logprobs entirely when a parser is configured and reasoning is + # hidden, because decoded logprob token text would leak hidden + # reasoning. + resolved_logprobs = None + if choice.logprobs is not None: + resolved_logprobs = _resolve_logprobs( + choice.logprobs, + tokenizer, + initial_context_token_ids=state.logprob_context_token_ids, + ) + include_role = not updated_state.role_sent - if include_role: - updated_state = updated_state.model_copy(update={"role_sent": True}) + updated_state = updated_state.model_copy( + update={ + "role_sent": True, + "logprob_context_token_ids": _logprob_context_tail( + state.logprob_context_token_ids, delta_tids + ), + } + ) delta = DeltaMessage( role="assistant" if include_role else None, @@ -380,6 +402,7 @@ class OnlineDerenderer: ChatCompletionResponseStreamChoice( index=choice.index, delta=delta, + logprobs=resolved_logprobs, finish_reason=choice.finish_reason, ) ) @@ -721,10 +744,32 @@ class OnlineDerenderer: new_text, updated_state = await self._detokenize_delta_async( tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special ) + + completion_logprobs = None + if choice.logprobs is not None: + resolved = _resolve_logprobs( + choice.logprobs, + tokenizer, + initial_context_token_ids=state.logprob_context_token_ids, + ) + completion_logprobs = _convert_chat_logprobs_to_completion_logprobs( + resolved, initial_text_offset=state.logprob_text_offset + ) + + updated_state = updated_state.model_copy( + update={ + "logprob_context_token_ids": _logprob_context_tail( + state.logprob_context_token_ids, delta_tids + ), + "logprob_text_offset": state.logprob_text_offset + len(new_text), + } + ) + stream_choices.append( CompletionResponseStreamChoice( index=choice.index, text=new_text, + logprobs=completion_logprobs, finish_reason=choice.finish_reason, ) ) @@ -750,6 +795,18 @@ class OnlineDerenderer: return chunk, updated_state +# Number of preceding sampled token IDs `_correct_decoded_token` reads to +# repair U+FFFD from byte-fallback tokenization. +_LOGPROB_CONTEXT_WINDOW = 4 + + +def _logprob_context_tail( + context_token_ids: list[int], delta_token_ids: list[int] +) -> list[int]: + """Advance the carried logprob context by this chunk's sampled tokens.""" + return (list(context_token_ids) + list(delta_token_ids))[-_LOGPROB_CONTEXT_WINDOW:] + + def _parse_token_id_placeholder(token: str) -> int | None: """Extract token ID from a 'token_id:N' placeholder string.""" if not token.startswith("token_id:"): @@ -799,13 +856,20 @@ def _correct_decoded_token( def _resolve_logprobs( - logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike + logprobs: ChatCompletionLogProbs, + tokenizer: TokenizerLike, + initial_context_token_ids: Sequence[int] = (), ) -> ChatCompletionLogProbs: - """Resolve token_id:N placeholders in a ChatCompletionLogProbs object.""" + """Resolve token_id:N placeholders in a ChatCompletionLogProbs object. + + ``initial_context_token_ids`` seeds the byte-fallback correction context + with sampled IDs from preceding chunks (streaming), so multi-byte + characters split across chunk boundaries still resolve. + """ if logprobs.content is None: return logprobs - context_token_ids: list[int] = [] + context_token_ids: list[int] = list(initial_context_token_ids) resolved_content = [] for entry in logprobs.content: @@ -845,9 +909,14 @@ def _resolve_logprobs( def _convert_chat_logprobs_to_completion_logprobs( logprobs: ChatCompletionLogProbs, + initial_text_offset: int = 0, ) -> CompletionLogProbs: """Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs - (parallel flat lists) as required by the /v1/completions response schema.""" + (parallel flat lists) as required by the /v1/completions response schema. + + ``initial_text_offset`` keeps ``text_offset`` absolute across streaming + chunks, mirroring the generate streaming path. + """ if logprobs.content is None: return CompletionLogProbs() @@ -856,7 +925,7 @@ def _convert_chat_logprobs_to_completion_logprobs( top_logprobs_list: list[dict[str, float] | None] = [] text_offset: list[int] = [] - offset = 0 + offset = initial_text_offset for entry in logprobs.content: text_offset.append(offset) tokens.append(entry.token) From 35cd03ebff344cc933ef7f42e45cfb19c76d0db7 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Tue, 29 Sep 2026 14:50:12 -0400 Subject: [PATCH 046/248] [CI] Mint the CRCR report's OIDC token after the wait, not before (#59259) --- .buildkite/scripts/crcr-report.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/crcr-report.sh b/.buildkite/scripts/crcr-report.sh index 3900b80339..0c114f4653 100644 --- a/.buildkite/scripts/crcr-report.sh +++ b/.buildkite/scripts/crcr-report.sh @@ -71,9 +71,11 @@ if [[ -z "${BK_TOKEN}" ]]; then fi AUDIENCE="pytorch-cross-repo-ci-relay" +# Pre-flight only; the token minted here is discarded. Checking now means an +# unusable OIDC setup costs seconds rather than being discovered after hours of +# polling. The token that is actually sent is minted after the wait, below. # OIDC redaction also requires the unavailable Job API socket. -OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" -if [[ -z "${OIDC_TOKEN}" ]]; then +if [[ -z "$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" ]]; then echo "could not mint a Buildkite OIDC token -- skipping report" exit 0 fi @@ -142,7 +144,7 @@ POLL_INTERVAL_S="${CRCR_POLL_INTERVAL_S:-900}" # meaningless whenever the step was itself delayed. # # Resolved after the first fetch, from the build's own created_at. -MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-25200}" +MAX_BUILD_AGE_S="${CRCR_MAX_BUILD_AGE_S:-14400}" WAIT_DEADLINE="" # Used until created_at is known. Without it, an API outage on the very first # request leaves the deadline unresolved and the loop spins to the step timeout. @@ -220,6 +222,17 @@ done # Resolved from this script's location: the pipeline runs it from # /vllm-workspace/tests, so a repo-relative path would not resolve. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Minted here, not before the poll loop: a Buildkite OIDC token defaults to a +# five-minute lifetime and the loop above can run to the build-age deadline. +# Build 91827 waited 6h48m and then took a 401 on all 360 callbacks from a +# token that had expired hours earlier. +OIDC_TOKEN="$(buildkite-agent oidc request-token --skip-redaction --audience "${AUDIENCE}" 2>/dev/null)" +if [[ -z "${OIDC_TOKEN}" ]]; then + echo "could not mint a Buildkite OIDC token after the wait -- skipping report" + exit 0 +fi + python3 "${SCRIPT_DIR}/crcr_report.py" \ --build-json "${BUILD_JSON}" \ --callback-url "${CALLBACK_URL}" \ From 30ae8248b663ea56398d44d30c0d8276a598c4a0 Mon Sep 17 00:00:00 2001 From: YukioZzz Date: Wed, 30 Sep 2026 03:14:24 +0800 Subject: [PATCH 047/248] [KVConnector][MoRIIO] Support K3 DSpark hybrid READ (#57700) Signed-off-by: Yichao Zhu Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../unit/test_moriio_hma_scheduler.py | 200 +++++++++++++++++- .../unit/test_nixl_connector_hma.py | 9 + .../kv_transfer/kv_connector/utils.py | 12 ++ .../v1/moriio/moriio_connector.py | 103 ++++++--- .../kv_connector/v1/nixl/base_scheduler.py | 9 +- 5 files changed, 288 insertions(+), 45 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py index 13934f896a..87ea53225a 100644 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -21,6 +21,9 @@ import pytest import torch from vllm.platforms import current_platform +from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager +from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs mori_available = importlib.util.find_spec("mori") is not None @@ -44,6 +47,16 @@ MoRIIOMode = moriio_connector.MoRIIOMode moriio_common = importlib.import_module( "vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common" ) +get_role = moriio_common.get_role +set_role = moriio_common.set_role + + +@pytest.fixture +def restore_moriio_role(): + """Undo the process-global role a full scheduler construction leaves behind.""" + role = get_role() + yield + set_role(role) class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore[name-defined] @@ -53,6 +66,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore def __init__(self, **attrs): self._mamba_group_ids: list[int] = [] self._attn_group_ids: list[int] = [0] + self._num_ssm_scratch_blocks = 0 self._ssm_state_slots_are_positional = False self._is_hma_required = False self.kv_cache_config = SimpleNamespace( @@ -110,22 +124,56 @@ def _gdn_split_info(conv_rows=3, key_dim=4, value_dim=8, dtype_size=2): ) -def _mamba_spec(num_states: int = 2): +def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0): shapes = tuple((1, 1) for _ in range(num_states)) return moriio_connector.MambaSpec( block_size=16, shapes=shapes, dtypes=(torch.float32,) * num_states, mamba_cache_mode="all", + num_speculative_blocks=num_speculative_blocks, ) -def _gate_vllm_config(*, read_mode: bool = True, speculative_config=None): +def _spec_config(method: str): + return SimpleNamespace(method=method, use_dspark=lambda: method == "dspark") + + +def _gate_vllm_config( + *, + read_mode: bool = True, + speculative_config=None, + block_size: int = 16, + num_lookahead_tokens: int = 0, + num_speculative_tokens: int = 0, + mamba_cache_mode: str = "all", + dcp_size: int = 1, +): return SimpleNamespace( kv_transfer_config=SimpleNamespace( - kv_connector_extra_config={"read_mode": read_mode} + kv_connector_extra_config={ + "read_mode": read_mode, + "host_ip": "127.0.0.1", + "handshake_port": 6001, + "notify_port": 6002, + }, + kv_role="kv_consumer", ), speculative_config=speculative_config, + cache_config=SimpleNamespace( + block_size=block_size, + mamba_cache_mode=mamba_cache_mode, + ), + scheduler_config=SimpleNamespace(disable_hybrid_kv_cache_manager=False), + num_lookahead_tokens=num_lookahead_tokens, + num_speculative_tokens=num_speculative_tokens, + parallel_config=SimpleNamespace( + tensor_parallel_size=1, + decode_context_parallel_size=dcp_size, + data_parallel_rank=0, + data_parallel_size_local=1, + data_parallel_size=1, + ), ) @@ -196,6 +244,7 @@ def test_exchange_blocks_ignore_transfer_disabled_group(): def test_scheduler_rejects_multiple_attention_groups_with_mamba(): + """A drafter owning its own transfer group is refused, with a hint.""" config = SimpleNamespace( kv_cache_groups=[ SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), @@ -205,7 +254,7 @@ def test_scheduler_rejects_multiple_attention_groups_with_mamba(): ) config.transfer_groups = tuple(config.kv_cache_groups) - with pytest.raises(moriio_common.MoRIIOError, match="exactly one transferable"): + with pytest.raises(moriio_common.MoRIIOError, match="separate attention group"): moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) @@ -251,7 +300,7 @@ def test_scheduler_rejects_mamba_group_without_two_states(): moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config) -def test_scheduler_rejects_speculative_hybrid_read(): +def test_scheduler_rejects_non_dspark_speculative_hybrid_read(): config = SimpleNamespace( kv_cache_groups=[ SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()), @@ -262,7 +311,105 @@ def test_scheduler_rejects_speculative_hybrid_read(): with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"): moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config(speculative_config=object()), "engine", config + _gate_vllm_config(speculative_config=_spec_config("ngram")), + "engine", + config, + ) + + +def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True): + """Mamba first, so a wrapped full-attention group at index 1 has to be found. + + A bare ``isinstance(FullAttentionSpec)`` misses the wrapper and leaves + ``_full_attn_group_idx`` at its initial 0, which is the Mamba group. + """ + full = FullAttentionSpec( + block_size=4, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + dcp_sharded=dcp_sharded, + ) + config = SimpleNamespace( + kv_cache_groups=[ + SimpleNamespace( + enable_kv_transfer=True, + kv_cache_spec=_mamba_spec(num_speculative_blocks=2), + ), + SimpleNamespace( + enable_kv_transfer=True, + kv_cache_spec=UniformTypeKVCacheSpecs( + block_size=4, kv_cache_specs={"target": full, "draft": full} + ), + ), + ], + ) + config.transfer_groups = tuple(config.kv_cache_groups) + return config + + +@pytest.mark.usefixtures("restore_moriio_role") +@pytest.mark.parametrize( + "dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)] +) +def test_scheduler_accepts_dspark_with_wrapped_full_attention( + dcp_size, dcp_sharded, expected_tail +): + config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded) + vllm_config = _gate_vllm_config( + speculative_config=_spec_config("dspark"), + block_size=4, + num_lookahead_tokens=7, + num_speculative_tokens=7, + mamba_cache_mode="align", + dcp_size=dcp_size, + ) + scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config) + + assert scheduler._full_attn_group_idx == 1 + assert scheduler._num_ssm_scratch_blocks == 2 + assert scheduler._max_decode_tail_blocks == expected_tail + + # A first decode step reserves padded target rows AND drafter lookahead. + spec = config.transfer_groups[1].kv_cache_spec + manager = FullAttentionManager( + spec, + block_pool=BlockPool(32, enable_caching=False, hash_block_size=4), + enable_caching=False, + kv_cache_group_id=1, + scheduler_block_size=16, + dcp_world_size=dcp_size, + ) + imported_tokens = manager.block_size + target_tokens = 1 + vllm_config.num_speculative_tokens + manager.allocate_new_blocks( + "decode", + num_tokens=imported_tokens + target_tokens + vllm_config.num_lookahead_tokens, + num_tokens_main_model=imported_tokens + target_tokens, + ) + local = [b.block_id for b in manager.req_to_blocks["decode"]] + remote = [100] + assert scheduler._align_read_blocks( + local, remote, scheduler._max_decode_tail_blocks + ) == (local[:1], remote) + with pytest.raises(ValueError, match="tail"): + scheduler._align_read_blocks( + local + [99], remote, scheduler._max_decode_tail_blocks + ) + + +def test_scheduler_rejects_dspark_with_positional_mamba_cache(): + """Positional mode counts lookahead in the Mamba allocation, so the + one-block recurrent-state tail bound no longer holds.""" + with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"): + moriio_connector.MoRIIOConnectorScheduler( + _gate_vllm_config( + speculative_config=_spec_config("dspark"), + block_size=4, + mamba_cache_mode="all", + ), + "engine", + _dspark_wrapped_attention_config(), ) @@ -316,6 +463,30 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode(): assert sched.split_block_groups(([1], [40, 41])) == ([1], [[41]]) +@pytest.mark.parametrize( + "positional,blocks,expected", + [ + # Trailing scratch slots go in either cache mode. + (True, [40, 41, 42], [40]), + # Outside "all" the running state is picked only after they are gone. + (False, [40, 41, 42, 43], [41]), + # Never empty: a list shorter than the scratch count keeps one state. + (True, [40], [40]), + (False, [], []), + ], +) +def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected): + sched = _FakeScheduler( + _has_mamba=True, + _attn_group_ids=[0], + _mamba_group_ids=[1], + _num_ssm_scratch_blocks=2, + _ssm_state_slots_are_positional=positional, + ) + + assert sched.split_block_groups(([1], blocks)) == ([1], [expected]) + + # -------------------------------------------------------------------------- # request_finished_all_groups # -------------------------------------------------------------------------- @@ -434,6 +605,23 @@ def test_update_state_pairs_shorter_local_blocks_with_remote_suffix(): assert sched._req_kv_params["req"]["remote_block_ids"] == [[12], [92]] +def test_update_state_pairs_trimmed_mamba_state_with_remote_state(): + """The DSpark shape end to end: scratch slots are stripped, the running + state is then selected, and the attention list loses its lookahead tail.""" + sched = _make_read_scheduler() + sched._ssm_state_slots_are_positional = False + sched._num_ssm_scratch_blocks = 2 + sched._max_decode_tail_blocks = 2 + request = _make_read_request([[10, 11], [90]]) + # Mamba group: [null, running state, scratch, scratch]. + blocks = _FakeBlocks(all_groups=([100, 101, 102, 103], [200, 201, 202, 203])) + + sched.update_state_after_alloc(request, blocks, num_external_tokens=256) + + assert sched._reqs_need_recv["req"][1] == [[100, 101], [201]] + assert sched._req_kv_params["req"]["remote_block_ids"] == [[10, 11], [90]] + + def test_update_state_full_attention_hit_still_carries_mamba_state(): sched = _make_read_scheduler() request = _make_read_request([[10], [90]]) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 125dfa6143..629e8c7ecc 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -2055,6 +2055,15 @@ def test_exchange_clipped_blocks_ssm_single_state(): # Scratch slots not allocated: the state slot still survives. assert sched.get_exchange_clipped_blocks(([1], [5]))[1] == [5] + assert sched.get_exchange_clipped_blocks(([1], [])) == ([1], []) + # Per-step partial lists must not be interpreted as state + scratch slots. + assert sched.get_exchange_clipped_blocks(([1], [7, 8]), clip_ssm=False) == ( + [1], + [7, 8], + ) + sched.blocks_per_sw = [2, 0] + assert sched.get_exchange_clipped_blocks(([1, 2, 3], [7, 8, 9])) == ([2, 3], [7]) + # Non-mamba models pass through unchanged. fa_sched = make_nixl_scheduler(has_mamba=False) assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],) diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index b645e97c4d..30021cde50 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -33,6 +33,18 @@ EngineId = str BlockIds = tuple[list[int], ...] | list[list[int]] +def clip_ssm_state_blocks( + blocks: list[int], num_spec_blocks: int, positional: bool +) -> list[int]: + """Drop speculative scratch slots, retaining position-indexed or running state.""" + if not blocks: + return blocks + # Keep at least the running-state slot, even before scratch allocation. + if num_scratch := min(num_spec_blocks, len(blocks) - 1): + blocks = blocks[:-num_scratch] + return blocks if positional else blocks[-1:] + + def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None): # NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is # used for faster transfer. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 0a8965a670..b482e6a8c4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -17,7 +17,10 @@ import torch import zmq from vllm.config import CUDAGraphMode, VllmConfig -from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds +from vllm.distributed.kv_transfer.kv_connector.utils import ( + BlockIds, + clip_ssm_state_blocks, +) from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, @@ -88,10 +91,10 @@ from vllm.utils.network_utils import ( from vllm.v1.attention.selector import get_attn_backend from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( - FullAttentionSpec, KVCacheConfig, MambaSpec, SlidingWindowSpec, + is_full_attention_spec, ) from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import RequestStatus @@ -108,7 +111,7 @@ logger = init_logger(__name__) # wait only needs to be long enough for it to make progress. _SQ_FULL_BACKOFF_INITIAL_S = 0.001 _SQ_FULL_BACKOFF_MAX_S = 0.05 -_MAX_LOCAL_DECODE_TAIL_BLOCKS = 1 +_MAX_LOCAL_MAMBA_TAIL_BLOCKS = 1 try: @@ -436,6 +439,26 @@ def _split_kv_cache_group_kinds( return attn, mamba +def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None: + speculative_config = vllm_config.speculative_config + if speculative_config is None: + return + # DSpark is allowlisted because it is the only hybrid speculative method + # validated end to end with MoRIIO READ. + if not speculative_config.use_dspark(): + raise MoRIIOError( + "MoRIIO hybrid READ supports DSpark speculative decoding only, got " + f"method={speculative_config.method!r}" + ) + if vllm_config.cache_config.mamba_cache_mode == "all": + # Positional allocation counts lookahead, so its local tail can exceed + # the single running-state block supported by hybrid READ. + raise MoRIIOError( + "MoRIIO hybrid READ does not support DSpark with " + "mamba_cache_mode='all'; use 'align'" + ) + + def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None: """Validate the common packed-state contract and return its representative.""" specs = tuple(specs) @@ -477,22 +500,22 @@ class MoRIIOConnectorScheduler: kv_cache_config ) self._has_mamba = bool(self._mamba_group_ids) + mamba_spec = None if self._has_mamba: if len(self._attn_group_ids) != 1: raise MoRIIOError( "MoRIIO hybrid READ requires exactly one transferable " "attention group, got " - f"{len(self._attn_group_ids)} attention groups" + f"{len(self._attn_group_ids)}; a drafter that owns a " + "separate attention group is not supported, give it the " + "target's group or disable KV transfer for it" ) mamba_specs = [ cast(MambaSpec, kv_cache_config.transfer_groups[group_id].kv_cache_spec) for group_id in self._mamba_group_ids ] - _validate_mamba_specs(mamba_specs) - if vllm_config.speculative_config is not None: - raise MoRIIOError( - "MoRIIO hybrid READ does not support speculative decoding" - ) + mamba_spec = _validate_mamba_specs(mamba_specs) + _validate_hybrid_speculation(vllm_config) if self.mode != MoRIIOMode.READ: raise MoRIIOError( "MoRIIO hybrid (mamba/KDA) transfer is implemented for READ " @@ -503,18 +526,31 @@ class MoRIIOConnectorScheduler: self._ssm_state_slots_are_positional = ( vllm_config.cache_config.mamba_cache_mode == "all" ) - self.block_size = vllm_config.cache_config.block_size - self._max_decode_tail_blocks = ( - _MAX_LOCAL_DECODE_TAIL_BLOCKS - if self._has_mamba - else math.ceil((vllm_config.num_lookahead_tokens + 1) / self.block_size) + self._num_ssm_scratch_blocks = ( + mamba_spec.num_speculative_blocks if mamba_spec is not None else 0 ) + self.block_size = vllm_config.cache_config.block_size + attn_block_size = self.block_size + max_decode_tail_tokens = 1 + vllm_config.num_lookahead_tokens + if self._has_mamba: + attn_spec = kv_cache_config.transfer_groups[ + self._attn_group_ids[0] + ].kv_cache_spec + attn_block_size = attn_spec.block_size + if attn_spec.dcp_sharded: + attn_block_size *= ( + vllm_config.parallel_config.decode_context_parallel_size + ) + # Admission can pad the final prompt token to 1 + num_spec rows; + # allocate_slots reserves drafter lookahead in addition to those. + max_decode_tail_tokens += vllm_config.num_speculative_tokens + self._max_decode_tail_blocks = cdiv(max_decode_tail_tokens, attn_block_size) self.engine_id: EngineId = engine_id self._is_hma_required = ( not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) + not is_full_attention_spec(g.kv_cache_spec) for g in kv_cache_config.transfer_groups ) ) @@ -524,9 +560,9 @@ class MoRIIOConnectorScheduler: unsupported = [ type(g.kv_cache_spec).__name__ for g in kv_cache_config.transfer_groups - if not isinstance( - g.kv_cache_spec, - (FullAttentionSpec, SlidingWindowSpec, MambaSpec), + if not ( + is_full_attention_spec(g.kv_cache_spec) + or isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec)) ) ] if unsupported: @@ -565,7 +601,7 @@ class MoRIIOConnectorScheduler: self._full_attn_group_idx = 0 self._full_attn_block_size = self.block_size for gi, group in enumerate(kv_cache_config.kv_cache_groups): - if isinstance(group.kv_cache_spec, FullAttentionSpec): + if is_full_attention_spec(group.kv_cache_spec): self._full_attn_group_idx = gi self._full_attn_block_size = getattr( group.kv_cache_spec, "block_size", self.block_size @@ -953,6 +989,9 @@ class MoRIIOConnectorScheduler: ) aligned_local_mamba: list[list[int]] = [] aligned_remote_mamba: list[list[int]] = [] + # Scratch slots were removed by + # split_block_groups, so only one local running + # state tail can remain. for local_group, remote_group in zip( mamba_block_groups, remote_mamba_groups, @@ -960,7 +999,9 @@ class MoRIIOConnectorScheduler: ): aligned_local, aligned_remote = ( self._align_read_blocks( - local_group, remote_group, 1 + local_group, + remote_group, + _MAX_LOCAL_MAMBA_TAIL_BLOCKS, ) ) aligned_local_mamba.append(aligned_local) @@ -1257,21 +1298,15 @@ class MoRIIOConnectorScheduler: transfer_block_ids = self.kv_cache_config.select_transfer_block_ids(block_ids) attn = list(transfer_block_ids[self._attn_group_ids[0]]) mamba_groups = [ - self._clip_mamba_group(list(transfer_block_ids[group_id])) + clip_ssm_state_blocks( + list(transfer_block_ids[group_id]), + self._num_ssm_scratch_blocks, + self._ssm_state_slots_are_positional, + ) for group_id in self._mamba_group_ids ] return attn, mamba_groups - def _clip_mamba_group(self, blocks: list[int]) -> list[int]: - """Keep only the state-bearing slots of one mamba kv cache group.""" - if not blocks: - return blocks - if not self._ssm_state_slots_are_positional: - # Single running state: everything before it is a null placeholder - # or the previous step's superseded state. - blocks = blocks[-1:] - return blocks - @staticmethod def _align_read_blocks( local_block_ids: list[int], @@ -1282,9 +1317,9 @@ class MoRIIOConnectorScheduler: Prefix hits can make the decode's unhashed suffix shorter than the prefill's full list, so read from the matching remote suffix. The - decoder can instead have one extra tail block for the locally - recomputed final token; that block has no remote source and is not - transferred. + decoder can instead have extra tail blocks for the locally recomputed + final token and speculative slots; these have no remote source and are + not transferred. """ if len(local_block_ids) > len(remote_block_ids): extra_decode_blocks = len(local_block_ids) - len(remote_block_ids) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index e264a94aaf..0705f944f6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -13,6 +13,7 @@ from vllm import envs from vllm.distributed.kv_transfer.kv_connector.utils import ( BlockIds, EngineId, + clip_ssm_state_blocks, yield_req_data, ) from vllm.distributed.kv_transfer.kv_connector.v1.base import ( @@ -288,11 +289,9 @@ class NixlBaseConnectorScheduler: and blocks and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None ): - if n_spec := min(n_spec_blocks, len(blocks) - 1): - blocks = blocks[:-n_spec] - if not self._ssm_state_slots_are_positional: - # Never empty: downstream reads that as a full prefix hit. - blocks = blocks[-1:] + blocks = clip_ssm_state_blocks( + blocks, n_spec_blocks, self._ssm_state_slots_are_positional + ) clipped.append(blocks) return tuple(clipped) From 7aa8372ef76b333d179029b1ed6270cc76f3c7c5 Mon Sep 17 00:00:00 2001 From: Aarushi Jain <142941703+aarushjain29@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:25:01 -0500 Subject: [PATCH 048/248] [ROCm][CI] Run Basic Correctness Sleep Mode on MI300 for now (#59262) Signed-off-by: aarushjain29 --- .buildkite/test-amd.yaml | 6 +++--- .buildkite/test_areas/basic_correctness.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index a6e2001aef..2e02d1b404 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3110,11 +3110,11 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s basic_correctness/memory/cumem -- label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" # TBD +- label: ":amd: (MI300) Basic Correctness Sleep Mode" # TBD timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false - agent_pool: mi355_dpx + agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index a5904fe5ac..d1d595ad0a 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -50,9 +50,9 @@ steps: - pytest -v -s basic_correctness/memory/sleep_mode mirror: amd: - label: ":amd: (MI355 DPX) Basic Correctness Sleep Mode" + label: ":amd: (MI300) Basic Correctness Sleep Mode" dind: false - device: mi355_dpx + device: mi300_1 timeout_in_minutes: 60 depends_on: - image-build-amd From 19a4d66a0bdd01405ea14ab9a5ea12fb0ff87cf2 Mon Sep 17 00:00:00 2001 From: hubunt <150658615+hubunt@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:37:12 +0800 Subject: [PATCH 049/248] [Bugfix] Avoid JSON constraints for native tool parsers (#47512) Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../test_structural_tag_registry.py | 97 +++++++++++++++++++ vllm/tool_parsers/abstract_tool_parser.py | 7 ++ 2 files changed, 104 insertions(+) diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 01bc53968e..3f22da17cd 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -6,15 +6,18 @@ from types import SimpleNamespace from unittest.mock import MagicMock import pytest +from openai.types.responses import ToolChoiceFunction from xgrammar import Grammar, StructuralTag from xgrammar.testing import _is_grammar_accept_string +import vllm.envs as envs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedFunction, ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionToolsParam, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.parser.abstract_parser import DelegatingParser from vllm.parser.plamo3 import ( BEGIN_TOOL_ARGUMENTS, @@ -700,6 +703,100 @@ def test_non_structural_tag_parser_uses_schema_constraints( assert out.structured_outputs.structural_tag is None +_NAMED_CHAT_TOOL_CHOICE = {"type": "function", "function": {"name": "get_weather"}} + + +def _responses_request(tool_choice) -> ResponsesRequest: + return ResponsesRequest.model_validate( + { + "input": "hi", + "tools": [ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + } + ], + "tool_choice": tool_choice, + } + ) + + +@pytest.mark.parametrize("tool_choice", ["required", _NAMED_CHAT_TOOL_CHOICE]) +def test_native_format_parser_skips_json_schema_constraint( + sample_tools: list[ChatCompletionToolsParam], + tool_choice, +): + # Parsers that extract required/named calls from their native format must + # not be forced into JSON, which their extractor cannot parse. + class NativeFormatToolParser(ToolParser): + supports_required_and_named = False + + parser = NativeFormatToolParser(MagicMock()) + request = ChatCompletionRequest( + messages=[], + model="m", + # Named tool choice validation reads tools in their JSON form. + tools=[tool.model_dump(exclude_none=True) for tool in sample_tools], + tool_choice=tool_choice, + ) + + out = parser.adjust_request(request) + + assert out.structured_outputs is None + + +@pytest.mark.parametrize( + "tool_choice", + ["required", ToolChoiceFunction(type="function", name="get_weather")], +) +def test_native_format_parser_skips_json_schema_constraint_responses( + tool_choice, +): + class NativeFormatToolParser(ToolParser): + supports_required_and_named = False + + parser = NativeFormatToolParser(MagicMock()) + request = _responses_request(tool_choice) + + out = parser.adjust_request(request) + + assert out.structured_outputs is None + assert out.text is None + + +@pytest.mark.parametrize("tool_choice", ["required", _NAMED_CHAT_TOOL_CHOICE]) +def test_glm47_without_strict_tool_calling_skips_json_schema_constraint( + monkeypatch: pytest.MonkeyPatch, + sample_tools: list[ChatCompletionToolsParam], + tool_choice, +): + # With strict tool calling off, no structural tag is attached, so GLM-4.7 + # must decode its native XML instead of a JSON tool-call list. + monkeypatch.setattr(envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", False) + + class TestParser(DelegatingParser): + tool_parser_cls = Glm47MoeModelToolParser + + request = ChatCompletionRequest( + messages=[], + model="m", + # Named tool choice validation reads tools in their JSON form. + tools=[tool.model_dump(exclude_none=True) for tool in sample_tools], + tool_choice=tool_choice, + ) + parser = TestParser(MagicMock(), tools=sample_tools) + parser._reasoning_parser = MagicMock(adjust_request=lambda request: request) + + out = parser.adjust_request(request) + + assert out.structured_outputs is None + assert out.skip_special_tokens is False + + def test_get_structural_tag_disables_reasoning( monkeypatch: pytest.MonkeyPatch, sample_tools_strict: list[ChatCompletionToolsParam], diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 953782daa9..bcbb5757c1 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -133,6 +133,13 @@ class ToolParser: ): return request + # Parsers that extract required/named tool calls from their native + # format (supports_required_and_named=False) must not be forced into + # the JSON tool-call format below: they would never find a call in + # the JSON output and would return it as content instead. + if not self.supports_required_and_named: + return request + json_schema_from_tool = get_json_schema_from_tools( tool_choice=request.tool_choice, tools=request.tools ) From a96ee5935d769ab72ca5c4dea5ee9b9964c9a45e Mon Sep 17 00:00:00 2001 From: sashko-zakharchuk Date: Tue, 29 Sep 2026 21:46:00 +0200 Subject: [PATCH 050/248] [MRV2] Add DRY as a custom logits processor example (#50584) Signed-off-by: Oleksandr Zakharchuk --- examples/features/logits_processor/README.md | 8 + examples/features/logits_processor/dry.py | 827 ++++++++++++++ tests/v1/sample/test_dry.py | 1031 ++++++++++++++++++ 3 files changed, 1866 insertions(+) create mode 100644 examples/features/logits_processor/dry.py create mode 100644 tests/v1/sample/test_dry.py diff --git a/examples/features/logits_processor/README.md b/examples/features/logits_processor/README.md index 07ca07dc71..bddf77b503 100644 --- a/examples/features/logits_processor/README.md +++ b/examples/features/logits_processor/README.md @@ -28,6 +28,14 @@ A special case of wrapping a request-level logits processor where the processor python examples/features/logits_processor/custom_req_init.py ``` +### `dry.py` — DRY repetition penalty (Model Runner V2) + +Implements the DRY (Don't Repeat Yourself) repetition penalty, ported from llama.cpp, as `DryState`, a custom logits processor for the Model Runner V2 interface. Requests turn it on through `SamplingParams.extra_args`, for example `{"dry_multiplier": 0.8}`. The example sends one prompt twice in one batch, once without DRY and once with it, and prints both outputs. `tests/v1/sample/test_dry.py` imports the processor from this file. + +```bash +python examples/features/logits_processor/dry.py +``` + ## Key Concepts - **Batch-level vs. request-level**: vLLM processes logits at the batch level for efficiency. If you have a per-request processor, you need to wrap it using the patterns shown in `custom_req.py` and `custom_req_init.py`. diff --git a/examples/features/logits_processor/dry.py b/examples/features/logits_processor/dry.py new file mode 100644 index 0000000000..30585215c6 --- /dev/null +++ b/examples/features/logits_processor/dry.py @@ -0,0 +1,827 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""This example implements the DRY (Don't Repeat Yourself) repetition penalty as +a Model Runner V2 custom logits processor, ``DryState``. + +A token that would extend a repeat of at least ``allowed_length`` tokens loses +``multiplier * base ** (repeat_length - allowed_length)`` from its logit, with the +exponent clamped as in llama.cpp. +Parameter names and matching semantics follow llama.cpp's +``llama_sampler_dry_apply``. llama.cpp's version is itself ported from pi6am's +koboldcpp implementation of p-e-w's scheme for text-generation-webui. + +Pass the class to ``LLM`` and configure it per request through ``extra_args``:: + + LLM(..., logits_processors=[DryState]) + SamplingParams(extra_args={"dry_multiplier": 0.8}) + +The recognized keys are ``dry_multiplier`` (0.0, off), ``dry_base`` (1.75), +``dry_allowed_length`` (2), ``dry_penalty_last_n`` (-1, the whole context) +and ``dry_sequence_breakers``. This processor claims the ``dry_*`` namespace, +so any other ``dry_*`` key fails the request. Speculative decoding is not +supported and is refused at construction. + +Run the example with:: + + python examples/features/logits_processor/dry.py + +It sends one prompt twice in one batch, once without DRY and once with it, and +prints both outputs, yielding an output similar to that shown below: + +Generated Outputs: +------------------------------------------------------------ +Prompt: 'The future of AI is', without DRY +Output: ' in the hands of the people.\n\nThe future of AI is in the hands of + the people.\n\nThe future of AI is in the hands of the + people.\n\nThe future of AI is in the hands of the people.\n\nThe + future of AI is in the hands of the people.\n' +------------------------------------------------------------ +Prompt: 'The future of AI is', with DRY +Output: ' in the hands of the people.\n\nThe future of AI is the future of + the human race.\n\nThe future of AI is a future of the human race, + and the future of humanity.\n\nThe future of AI is an AI that is + capable of making decisions that are not based on human + intelligence.' +------------------------------------------------------------ +""" + +import math +import weakref +from typing import TYPE_CHECKING, Any, NamedTuple + +import numpy as np +import torch + +from vllm.logger import init_logger +from vllm.sampling_params import SamplingParams +from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.utils.torch_utils import async_tensor_h2d +from vllm.v1.worker.gpu.sample.logits_processor import ( + LogitsContext, + LogitsProcessor, + LogitsProcRequestState, +) + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +logger = init_logger(__name__) + +_J_BUDGET = 2048 +# Run lengths feed an int16 sum, so they must stay below 2**15. +assert _J_BUDGET < 2**15, "_J_BUDGET must fit the int16 run-length sum" + +# Byte budget for the per-chunk transients of the match scan. The gather +# W32[:, idx1] materializes [R, chunk, J] int32 (4 B/elem) before the +# comparison reduces it to bool; with the masks, the int8 cumprod and +# the int16 row sums, the marginal transient cost is ~8 B/elem. 24 keeps +# headroom for the fixed R*vocab penalty accumulator floor and allocator slack. +_CHUNK_BYTE_BUDGET = 256 * 1024 * 1024 +_CHUNK_PEAK_BYTES_PER_ELEM = 24 + +# llama.cpp's FLOAT_MAX_LOG (src/llama-sampler.cpp): ln(float32 max). +_FLOAT_MAX_LOG = 88.7228391 + +# Breakers are truncated to this many code points. llama.cpp cuts at this many +# bytes (llama_sampler_init_dry), which keeps fewer characters of a non-ASCII +# breaker. +_MAX_BREAKER_CHAR_LEN = 40 + +# Upper bound on distinct breaker sets cached per tokenizer. +_MAX_CACHED_BREAKER_SETS = 64 + +STR_SPEC_DEC_REJECTS_DRY = ( + "The DRY logits processor is not supported when speculative decoding is enabled." +) + +DEFAULT_DRY_SEQUENCE_BREAKERS = ("\n", ":", '"', "*") +"""llama.cpp's default DRY sequence breakers.""" + +MAX_DRY_SEQUENCE_BREAKERS = 64 +"""Upper bound on the dry_sequence_breakers list length. Each multi-character +breaker in an uncached set scans the vocabulary.""" + +_DRY_INT_MAX = 2**31 - 1 +"""Upper bound on the integral DRY parameters, matching llama-server's +INT32_MAX cap. The state below stores them in an int64 numpy array, where +anything at or above 2**63 raises OverflowError inside execute_model and +takes the engine down.""" + +_MAX_CACHED_BREAKER_MASKS = 64 +"""Upper bound on distinct breaker sets holding a device mask, mirroring +_MAX_CACHED_BREAKER_SETS on the host side.""" + +_WARMUP_WINDOW = 8 +"""Window length of the dry_core call in DryState.__init__, long enough to +hold a match at the default dry_allowed_length.""" + +_DRY_DEFAULTS: dict[str, Any] = { + "dry_multiplier": 0.0, + "dry_base": 1.75, + "dry_allowed_length": 2, + "dry_penalty_last_n": -1, # whole context; llama.cpp's default is 64 + "dry_sequence_breakers": DEFAULT_DRY_SEQUENCE_BREAKERS, +} +"""Recognized extra_args keys and their defaults.""" + + +# --------------------------------------------------------------------------- +# The match computation +# --------------------------------------------------------------------------- +# ``_dry_penalties`` is a sequential port of llama.cpp's Z-algorithm, used as the +# reference. ``dry_core`` is the vectorized form: the comparisons that determine +# the match length ending at position ``i`` lie along one diagonal, so a +# cumprod-sum along ``j`` of an ``[R, K, J]`` tensor gives every match length +# without a sequential scan. Capping ``J`` at ``allowed_length + max_exponent`` +# is exact, because the exponent clamp maps every longer match to the same +# penalty. Requests whose cap is unusable (``max_exponent == 0``, i.e. +# ``base <= 1.000001``, or a cap beyond ``_J_BUDGET``) use the sequential form. + + +def max_exponent(base: float) -> int: + """Exponent clamp, mirroring llama.cpp bit-for-bit. + + llama.cpp computes ``FLOAT_MAX_LOG / std::log(dry_base)`` entirely in + float32. Computing the quotient in float64 lands on the wrong side of + the integer truncation for some bases: at ``base=2.0`` float64 gives + 127.99999998 -> 127 while float32 gives exactly 128.0 -> 128. + """ + if base <= 1.000001: + return 0 + return int(np.float32(_FLOAT_MAX_LOG) / np.log(np.float32(base))) + + +def _dry_penalties( + window: list[int], + breakers: frozenset[int], + multiplier: float, + base: float, + allowed_length: int, + max_exp: int, +) -> dict[int, float]: + """Compute DRY penalties for one request. + + Port of the scan in ``llama_sampler_dry_apply``: a reverse-direction + Z-algorithm finds, for each position, the length of the match between + the window suffix and the sequence ending at that position; each + match's follower token is charged the penalty for the longest repeat + it would extend. + + Returns a dict mapping token id -> penalty to subtract from its logit. + """ + m = len(window) + + def rat(i: int) -> int: # reverse access: rat(0) == last token + return window[m - 1 - i] + + # Step 1: the nearest breaker from the end caps the match length. + rep_limit = m + for i in range(m): + if rat(i) in breakers: + rep_limit = i + break + if rep_limit < allowed_length: + return {} + + # Step 2: reverse-direction Z-algorithm -> per-position repeat counts + # (forward-indexed into ``window`` via cnt[last - k]). + cnt = [0] * m + last = m - 1 + lt = rt = 0 + for k in range(1, m): + if k > rt: + # Outside the current Z-box: extend naively. + z = 0 + while z + k < m and rat(z) == rat(z + k): + z += 1 + cnt[last - k] = min(z, rep_limit) + if z > 0: + lt, rt = k, k + z - 1 + else: + p = k - lt + right_part_len = rt - k + 1 + if cnt[last - p] < right_part_len: + # Fully inside the Z-box: copy. + cnt[last - k] = min(cnt[last - p], rep_limit) + else: + # Touches the right edge: extend past it. + j = rt + 1 + while j < m and rat(j) == rat(j - k): + j += 1 + cnt[last - k] = min(j - k, rep_limit) + lt, rt = k, j - 1 + + # Step 3: map each repeat's follower token to the longest repeat that + # it would extend. + max_token_repeat: dict[int, int] = {} + for i in range(m - 1): + repeat_len = cnt[i] + if repeat_len >= allowed_length: + tok = window[i + 1] + if max_token_repeat.get(tok, -1) < repeat_len: + max_token_repeat[tok] = repeat_len + + # Step 4: exponential penalty, exponent clamped for float32 safety. + # Breaker tokens are never penalized. A value overflowing float32 + # saturates the logit to -inf downstream, as in llama.cpp. + penalties: dict[int, float] = {} + for tok, repeat_len in max_token_repeat.items(): + if tok in breakers: + continue + exponent = repeat_len - allowed_length + if max_exp and exponent > max_exp: + exponent = max_exp + penalties[tok] = multiplier * (base**exponent) + return penalties + + +def dry_core( + logits: torch.Tensor, + row_idx: torch.Tensor, + W: torch.Tensor, + n_r: torch.Tensor, + allowed: torch.Tensor, + max_exp: torch.Tensor, + mult: torch.Tensor, + base: torch.Tensor, + breaker_masks: list[torch.Tensor | None], + j_budget: int, +) -> torch.Tensor: + """Apply DRY penalties in place given batched window tensors. + + The tensor bounds below are not checked, to avoid a sync. + + Args: + logits: [B, vocab] float tensor, modified in place. + row_idx: [R] int64, row of ``logits`` for each DRY request. + W: [R, N] int64 windows, right-aligned (window tokens occupy the + trailing ``n_r`` columns; leading columns are ignored via ``n_r`` + masks, whatever they hold). + n_r: [R] int64 window lengths. + allowed: [R] int64 per-request allowed length. + max_exp: [R] int64 per-request exponent ceiling, > 0. + mult: [R] float32 per-request multiplier, >= 0, already rounded + through float32, as llama.cpp stores it. + base: [R] float32 per-request base, >= 1, rounded the same way. + breaker_masks: per-request [vocab] bool masks (or None for no + breakers). ``vocab`` must equal ``logits.shape[-1]``. + j_budget: max(allowed + max_exp), <= _J_BUDGET, computed by the caller on + the host. + + """ + device = logits.device + vocab = logits.shape[-1] + R, N = W.shape + + # j_budget arrives as an unchecked host int; this check fails loudly rather + # than wrapping an int16 run length. The routing predicate in DryState.apply + # is what actually bounds it. + if j_budget > _J_BUDGET: + raise ValueError(f"j_budget {j_budget} over _J_BUDGET {_J_BUDGET}") + # base >= 1 and mult >= 0 are preconditions too (see the amax comment below). + # They live in device tensors and reading them back would sync; use_dry() + # and validate_params enforce them instead. + + # rep_limit: distance from the end of the nearest breaker (llama.cpp step 1). + # A breaker at column c is j = N-1-c tokens from the end. + rep_limit = n_r.clone() + # bool(bm.any()) would be a per-request host sync; the None check says the same. + any_breakers = any(bm is not None for bm in breaker_masks) + bmask = None + if any_breakers: + # Stacked once, used twice: nearest-breaker search here, penalty zeroing + # after the scan. The masks are cached and resident (DryState._breaker_masks), + # so the stack costs one byte per (row, vocab) entry. + bmask = torch.stack( + [ + bm + if bm is not None + else torch.zeros(vocab, dtype=torch.bool, device=device) + for bm in breaker_masks + ] + ) + Bwin = bmask.gather(1, W.clamp(min=0)) + valid_cols = torch.arange(N, device=device)[None, :] >= (N - n_r)[:, None] + Bwin &= valid_cols + has_breaker = Bwin.any(dim=1) + # max breaker column -> nearest to the end. + max_col = torch.where( + has_breaker, + (Bwin * torch.arange(1, N + 1, device=device)[None, :]).max(dim=1).values + - 1, + torch.zeros_like(n_r), + ) + rep_limit = torch.where(has_breaker, (N - 1) - max_col, n_r) + + # llama.cpp: if rep_limit < allowed_length, the request produces nothing. + active = rep_limit >= allowed + + # Token ids fit int32, and gathering int32 halves the dominant per-chunk transient. + W32 = W.to(torch.int32) + # J is a tensor shape, so it must be known host-side; the caller computes it + # from numpy (see DryState.apply) to avoid a per-step device readback. + J = max(1, min(j_budget, N)) + idx2 = torch.arange(N - 1, N - 1 - J, -1, device=device) # [J] + suffix = W32.gather(1, idx2.expand(R, J)) # [R, J] + K = N - 1 # offsets 1..N-1 + chunk = max(1, _CHUNK_BYTE_BUDGET // (_CHUNK_PEAK_BYTES_PER_ELEM * max(1, R * J))) + + # amax keeps one penalty per token, for its longest match, however the offsets + # are chunked, as long as the penalty does not fall as the match grows + # (base >= 1, mult >= 0). + pen = torch.zeros(R * vocab + 1, dtype=torch.float32, device=device) + + for k0 in range(1, K + 1, chunk): + k1 = min(k0 + chunk, K + 1) + ks = torch.arange(k0, k1, device=device) # [C] + C = ks.shape[0] + # idx1[c, j] = N-1-j-k ; invalid (out of window) entries masked. + idx1 = idx2[None, :] - ks[:, None] # [C, J] + invalid = idx1 < (N - n_r)[:, None, None] # [R, C, J] + eq = W32[:, idx1.clamp(min=0)] == suffix[:, None, :] # [R, C, J] + eq &= ~invalid + del invalid + # Run length of leading True along j = the match length. Explicit dtypes avoid + # int64 intermediate copies; values are 0/1 and runs are <= J <= _J_BUDGET, + # so int8/int16 are exact. + L = eq.cumprod(dim=2, dtype=torch.int8).sum(dim=2, dtype=torch.int16) + del eq + # Do not narrow rep_limit to L's int16; window lengths can exceed it. + L = torch.minimum(L, rep_limit[:, None]) + + # Follower token of offset k lives at column N-k; the offset counts only + # while position i = N-1-k is inside the window (k <= n_r - 1). + valid_k = ks[None, :] <= (n_r - 1)[:, None] # [R, C] + charge = (allowed[:, None] <= L) & valid_k & active[:, None] + # No `if charge.any()` and no boolean indexing: both sync the host. Uncharged + # entries scatter to a trash slot one past the accumulator end, never read. + followers = W.gather(1, (N - ks).clamp(max=N - 1).expand(R, C)) + rows = torch.arange(R, device=device)[:, None].expand(R, C) * vocab + flat = rows + followers.clamp(min=0) + # A Python int, not a device tensor: torch.tensor(x, device=...) here would be a + # pageable host-to-device copy, which is itself a synchronization. + flat = torch.where(charge, flat, R * vocab) + # L takes rep_limit's dtype from the minimum above. The cast keeps the + # exponent int64 whatever dtype n_r, allowed and max_exp arrive in. + exponent = torch.minimum(L.to(torch.int64) - allowed[:, None], max_exp[:, None]) + # float64 pow, as llama.cpp's std::pow(float, int); a float32 pow overflows + # early (0.8 * 2**128 is finite in float32). + p_chunk = mult[:, None].double() * torch.pow( + base[:, None].double(), exponent.to(torch.float64) + ) + # No where() on p_chunk: uncharged entries went to the trash slot and are + # never read. + pen.scatter_reduce_( + 0, + flat.reshape(-1), + p_chunk.to(torch.float32).reshape(-1), + reduce="amax", + include_self=True, + ) + + # Of the penalty terms, only the penalty itself is held at [R, vocab] (4 B/entry); + # the exponent, its float64 cast, the pow result and the product stay at [R, C] + # inside the loop. + # Breakers zero the penalty rather than the logit, so a breaker keeps its value. + # The narrowing to logits.dtype below is a no-op under vLLM's sampler, which + # always passes float32 logits. + pen2 = pen[:-1].view(R, vocab) + if bmask is not None: + pen2.masked_fill_(bmask, 0.0) + # index_add_ avoids the extra [R, vocab] gather+copy of `logits[row_idx] -= pen2`. + pen2.neg_() + logits.index_add_( + 0, row_idx, pen2 if logits.dtype == pen2.dtype else pen2.to(logits.dtype) + ) + return logits + + +# --------------------------------------------------------------------------- +# Breaker resolution +# --------------------------------------------------------------------------- + + +class _VocabIndex(NamedTuple): + """A tokenizer's decoded vocabulary, plus a character index into it. + + ``char_ids`` maps every character in ``texts`` to the ids of the tokens + whose text contains it. + """ + + texts: list[str] + char_ids: dict[str, list[int]] + + +# Per-tokenizer caches, weakly keyed so tokenizers can be collected: +# tokenizer -> its decoded vocabulary and character index, and +# tokenizer -> {breaker string tuple -> resolved breaker ids}. +_BreakerIdsPerTokenizer = dict[tuple[str, ...], list[int]] +_vocab_index_cache: "weakref.WeakKeyDictionary[Any, _VocabIndex]" = ( + weakref.WeakKeyDictionary() +) +_breaker_ids_cache: "weakref.WeakKeyDictionary[Any, _BreakerIdsPerTokenizer]" = ( + weakref.WeakKeyDictionary() +) + + +def _vocab_index(tokenizer: Any) -> _VocabIndex: + """Decode every token of ``tokenizer`` once and index its characters.""" + index = _vocab_index_cache.get(tokenizer) + if index is None: + # Include added tokens above vocab_size, which llama.cpp's scan also covers. + n_ids = getattr(tokenizer, "max_token_id", tokenizer.vocab_size - 1) + 1 + texts = tokenizer.batch_decode([[i] for i in range(n_ids)]) + char_ids: dict[str, list[int]] = {} + for i, text in enumerate(texts): + for char in set(text): + char_ids.setdefault(char, []).append(i) + index = _VocabIndex(texts, char_ids) + _vocab_index_cache[tokenizer] = index + return index + + +def resolve_dry_breakers(tokenizer: Any, breaker_strs: tuple[str, ...]) -> list[int]: + """Resolve breaker strings to the ids of every token whose text contains one. + + This is llama.cpp's ``get_overlapping_token_sequences`` rule, without its + multi-token restart sequences. Resolution runs in ``add_request``, inside + ``execute_model``, so a single-character breaker (llama.cpp's defaults and + most custom sets) resolves by lookup in the character index instead of a + vocabulary scan. Results are cached per (tokenizer, breaker set). + """ + breaker_strs = tuple(s[:_MAX_BREAKER_CHAR_LEN] for s in breaker_strs if s) + if not breaker_strs: + return [] + per_tok = _breaker_ids_cache.setdefault(tokenizer, {}) + cached = per_tok.get(breaker_strs) + if cached is not None: + return list(cached) + + index = _vocab_index(tokenizer) + ids: set[int] = set() + for s in breaker_strs: + if len(s) == 1: + ids.update(index.char_ids.get(s, ())) + else: + ids.update(i for i, text in enumerate(index.texts) if s in text) + result = sorted(ids) + if len(per_tok) >= _MAX_CACHED_BREAKER_SETS: + # Evict the oldest entry (dict preserves insertion order). + per_tok.pop(next(iter(per_tok))) + per_tok[breaker_strs] = result + return list(result) + + +# --------------------------------------------------------------------------- +# The logits processor +# --------------------------------------------------------------------------- + + +def _dry_args(sampling_params: SamplingParams) -> dict[str, Any]: + """Read the request's DRY arguments, filling in the defaults. + + A key present with value None counts as unset, matching how + ``SamplingParams`` coerces its own None-valued arguments. + """ + extra = sampling_params.extra_args or {} + return { + name: default if extra.get(name) is None else extra[name] + for name, default in _DRY_DEFAULTS.items() + } + + +def use_dry(multiplier: float, base: float, penalty_last_n: int) -> bool: + """Whether DRY applies, by llama.cpp's gate in llama_sampler_dry_apply.""" + return bool(multiplier) and base >= 1.0 and penalty_last_n != 0 + + +class DryState(LogitsProcessor): + def __init__(self, vllm_config: "VllmConfig", req_states: LogitsProcRequestState): + # Refuse speculative decoding here, since admission cannot see its config. + if vllm_config.speculative_config is not None: + raise ValueError(STR_SPEC_DEC_REJECTS_DRY) + self.req_states = req_states + max_num_reqs = req_states.max_num_reqs + self.vocab_size = req_states.vocab_size + self.device = req_states.device + + # float32, to round multiplier and base as llama.cpp's float members do. + self.multiplier = np.zeros(max_num_reqs, dtype=np.float32) + self.base = np.zeros(max_num_reqs, dtype=np.float32) + self.allowed_length = np.zeros(max_num_reqs, dtype=np.int64) + self.penalty_last_n = np.zeros(max_num_reqs, dtype=np.int64) + self.max_exponent = np.zeros(max_num_reqs, dtype=np.int64) + self.use_dry = np.zeros(max_num_reqs, dtype=bool) + + # req_idx -> its breaker set, and breaker set -> [vocab] bool device + # mask shared by every request that asked for the same breakers. + self.breaker_ids: dict[int, frozenset[int]] = {} + self._breaker_masks: dict[frozenset[int], torch.Tensor] = {} + + self._warned_unresolved = False + + # Deferred import: the tokenizer registry pulls in transformers, and + # the frontend imports this module only to validate params. + from vllm.tokenizers import cached_tokenizer_from_config + + # None under skip_tokenizer_init. + self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config) + # Resolve the default set here to keep the vocabulary decode out of + # execute_model, where add_request runs. + self._default_breaker_ids = self._resolve_breakers( + DEFAULT_DRY_SEQUENCE_BREAKERS + ) + self._warm_up_dry_core() + + def _warm_up_dry_core(self) -> None: + """Load dry_core's CUDA kernels before a request needs them. + + The operands match the dtypes and ranks apply passes, and include a breaker + mask so that the breaker path's kernels load too. + """ + vocab = self.vocab_size + # The breaker takes the last id, which must not be the window's token 0. + if vocab < 2: + return + device = self.device + allowed = _DRY_DEFAULTS["dry_allowed_length"] + base = _DRY_DEFAULTS["dry_base"] + max_exp = max_exponent(base) + + def col(value: float, dtype: torch.dtype) -> torch.Tensor: + return torch.full((1,), value, dtype=dtype, device=device) + + breakers = torch.zeros(vocab, dtype=torch.bool, device=device) + breakers.narrow(0, vocab - 1, 1).fill_(True) + dry_core( + torch.zeros(1, vocab, dtype=torch.float32, device=device), + row_idx=col(0, torch.int64), + W=torch.zeros(1, _WARMUP_WINDOW, dtype=torch.int64, device=device), + n_r=col(_WARMUP_WINDOW, torch.int64), + allowed=col(allowed, torch.int64), + max_exp=col(max_exp, torch.int64), + mult=col(1.0, torch.float32), + base=col(base, torch.float32), + breaker_masks=[breakers], + j_budget=allowed + max_exp, + ) + + @classmethod + def validate_params(cls, sampling_params: SamplingParams) -> None: + """Check the ``dry_*`` keys of ``extra_args`` at request admission. + + Raises: + ValueError: on an unknown or out-of-range DRY argument. + + """ + extra = sampling_params.extra_args or {} + unknown = sorted( + key for key in extra if key.startswith("dry_") and key not in _DRY_DEFAULTS + ) + if unknown: + # A misspelled key would otherwise be ignored silently. + raise ValueError( + f"Unknown dry_* extra_args: {', '.join(unknown)}. " + f"Supported keys: {', '.join(_DRY_DEFAULTS)}." + ) + + args = _dry_args(sampling_params) + for name in ( + "dry_multiplier", + "dry_base", + "dry_allowed_length", + "dry_penalty_last_n", + ): + value = args[name] + # JSON true/false are bools, which pass isinstance(value, int). + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError( + f"{name} must be a number, got {type(value).__name__}." + ) + + multiplier = args["dry_multiplier"] + base = args["dry_base"] + allowed_length = args["dry_allowed_length"] + penalty_last_n = args["dry_penalty_last_n"] + breakers = args["dry_sequence_breakers"] + + if not math.isfinite(multiplier) or multiplier < 0.0: + raise ValueError( + f"dry_multiplier must be non-negative and finite, got {multiplier}." + ) + if not math.isfinite(base) or base < 0.0: + raise ValueError(f"dry_base must be non-negative and finite, got {base}.") + if ( + not isinstance(allowed_length, int) + or allowed_length < 0 + or allowed_length > _DRY_INT_MAX + ): + raise ValueError( + f"dry_allowed_length must be an integer in [0, {_DRY_INT_MAX}], " + f"got {allowed_length}." + ) + if ( + not isinstance(penalty_last_n, int) + or penalty_last_n < -1 + or penalty_last_n > _DRY_INT_MAX + ): + raise ValueError( + "dry_penalty_last_n must be an integer: -1 (whole context), " + f"0 (disable), or in [1, {_DRY_INT_MAX}], got {penalty_last_n}." + ) + if not isinstance(breakers, (list, tuple)) or any( + not isinstance(s, str) for s in breakers + ): + raise ValueError( + f"dry_sequence_breakers must be a list of strings, got {breakers!r}." + ) + if len(breakers) > MAX_DRY_SEQUENCE_BREAKERS: + raise ValueError( + f"dry_sequence_breakers supports at most " + f"{MAX_DRY_SEQUENCE_BREAKERS} entries, got {len(breakers)}." + ) + if multiplier and 0.0 <= base < 1.0: + # libllama has the same gate. llama-server resets such a base to its + # default instead. + logger.warning( + "dry_base=%s is below 1.0, which disables DRY entirely " + "(llama.cpp semantics), even though dry_multiplier=%s was " + "set. No repetition penalty will be applied.", + base, + multiplier, + ) + + def _resolve_breakers(self, breakers: tuple[str, ...]) -> frozenset[int]: + if self._tokenizer is None or not breakers: + return frozenset() + return frozenset(resolve_dry_breakers(self._tokenizer, breakers)) + + def add_request(self, req_idx: int, sampling_params: SamplingParams) -> bool: + args = _dry_args(sampling_params) + multiplier = args["dry_multiplier"] + base = args["dry_base"] + penalty_last_n = args["dry_penalty_last_n"] + enabled = use_dry(multiplier, base, penalty_last_n) + self.use_dry[req_idx] = enabled + self.breaker_ids.pop(req_idx, None) + if not enabled: + return False + self.multiplier[req_idx] = multiplier + self.base[req_idx] = base + self.allowed_length[req_idx] = args["dry_allowed_length"] + self.penalty_last_n[req_idx] = penalty_last_n + self.max_exponent[req_idx] = max_exponent(float(self.base[req_idx])) + + breakers = tuple(args["dry_sequence_breakers"]) + ids = ( + self._default_breaker_ids + if breakers == DEFAULT_DRY_SEQUENCE_BREAKERS + else self._resolve_breakers(breakers) + ) + if ids: + self.breaker_ids[req_idx] = ids + elif breakers and self._tokenizer is None and not self._warned_unresolved: + logger.warning( + "DRY sequence breakers were not resolved to token ids: this " + "engine has no tokenizer. Proceeding without breakers." + ) + self._warned_unresolved = True + return True + + def _breaker_mask(self, req_idx: int) -> torch.Tensor | None: + ids = self.breaker_ids.get(req_idx) + if not ids: + return None + mask = self._breaker_masks.get(ids) + if mask is None: + # Built on the host to avoid a sync; once per breaker set. + ids_np = np.fromiter(ids, dtype=np.int64, count=len(ids)) + ids_np = ids_np[ids_np < self.vocab_size] + m_np = np.zeros(self.vocab_size, dtype=bool) + m_np[ids_np] = True + mask = async_tensor_h2d(m_np, self.device) + if len(self._breaker_masks) >= _MAX_CACHED_BREAKER_MASKS: + # Evict the oldest set. A client can send a new set with every request. + self._breaker_masks.pop(next(iter(self._breaker_masks))) + self._breaker_masks[ids] = mask + return mask + + def apply(self, logits: torch.Tensor, ctx: LogitsContext) -> torch.Tensor: + req_indices = ctx.idx_mapping_np + active_rows = np.flatnonzero(self.use_dry[req_indices]) + if active_rows.size == 0: + return logits + if logits.shape[0] != req_indices.shape[0]: + raise RuntimeError("DRY received draft-expanded logits") + + # host-side bound; reading positions off GPU would sync per step + cur_len = ctx.seq_lens_upper_bound_np[active_rows].astype(np.int64) + + reqs = req_indices[active_rows] + last_n = self.penalty_last_n[reqs] + window_len = np.where(last_n == -1, cur_len, np.minimum(cur_len, last_n)) + allowed = self.allowed_length[reqs] + keep = window_len > allowed + if not np.any(keep): + return logits + active_rows = active_rows[keep] + reqs = reqs[keep] + cur_len = cur_len[keep] + window_len = window_len[keep] + allowed = allowed[keep] + max_exp = self.max_exponent[reqs] + + # Route degenerate-clamp requests (base <= 1.000001 or oversized + # cap) through the sequential reference implementation. + fast = (max_exp > 0) & (allowed + max_exp <= _J_BUDGET) + all_tokens = self.req_states.all_token_ids.gpu + + if np.any(fast): + f_rows = active_rows[fast] + f_reqs = reqs[fast] + f_len = window_len[fast] + N = int(f_len.max()) + reqs_t = async_tensor_h2d(f_reqs, self.device) + cur_t = async_tensor_h2d(cur_len[fast], self.device) + j = torch.arange(N, device=self.device) + # Right-aligned gather: column j holds token (cur_len - N + j); + # out-of-window columns are masked inside dry_core via n_r. + gather_idx = (cur_t[:, None] - N + j[None, :]).clamp(min=0) + W = all_tokens[reqs_t[:, None], gather_idx].long() + dry_core( + logits, + row_idx=async_tensor_h2d(f_rows, self.device), + W=W, + n_r=async_tensor_h2d(f_len, self.device), + allowed=async_tensor_h2d(allowed[fast], self.device), + max_exp=async_tensor_h2d(max_exp[fast], self.device), + mult=async_tensor_h2d(self.multiplier[f_reqs], self.device), + base=async_tensor_h2d(self.base[f_reqs], self.device), + breaker_masks=[self._breaker_mask(r) for r in f_reqs], + j_budget=int((allowed[fast] + max_exp[fast]).max()), + ) + + # The sequential fallback copies each window to the host, an expected sync. + slow = ~fast + if np.any(slow): + with gpu_sync_allowed(): + rows_list = [] + cols_list = [] + vals_list = [] + for row, req, w_len, cur in zip( + active_rows[slow], reqs[slow], window_len[slow], cur_len[slow] + ): + window = ( + all_tokens[int(req), int(cur) - int(w_len) : int(cur)] + .cpu() + .tolist() + ) + penalties = _dry_penalties( + window, + self.breaker_ids.get(int(req), frozenset()), + float(self.multiplier[req]), + float(self.base[req]), + int(self.allowed_length[req]), + int(self.max_exponent[req]), + ) + for tok, val in penalties.items(): + rows_list.append(int(row)) + cols_list.append(tok) + vals_list.append(val) + if rows_list: + logits[ + torch.tensor(rows_list, dtype=torch.int64, device=self.device), + torch.tensor(cols_list, dtype=torch.int64, device=self.device), + ] -= torch.tensor( + vals_list, dtype=torch.float32, device=self.device + ) + return logits + + +# --------------------------------------------------------------------------- +# The demo +# --------------------------------------------------------------------------- + + +def main(): + # Imported here so that loading DryState from this module does not import LLM. + from vllm import LLM + + prompts = ["The future of AI is"] * 2 + sampling_params_list = [ + SamplingParams(temperature=0.0, max_tokens=64), + SamplingParams( + temperature=0.0, max_tokens=64, extra_args={"dry_multiplier": 0.8} + ), + ] + llm = LLM(model="facebook/opt-125m", logits_processors=[DryState]) + outputs = llm.generate(prompts, sampling_params_list) + print("\nGenerated Outputs:\n" + "-" * 60) + for label, output in zip(("without DRY", "with DRY"), outputs): + print(f"Prompt: {output.prompt!r}, {label}") + print(f"Output: {output.outputs[0].text!r}") + print("-" * 60) + + +if __name__ == "__main__": + main() diff --git a/tests/v1/sample/test_dry.py b/tests/v1/sample/test_dry.py new file mode 100644 index 0000000000..c724dc681b --- /dev/null +++ b/tests/v1/sample/test_dry.py @@ -0,0 +1,1031 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the DRY (Don't Repeat Yourself) repetition penalty.""" + +import math +import random +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import numpy as np +import pytest +import torch + +import vllm.v1.worker.gpu.sample.logits_processor.loader as loader +from examples.features.logits_processor import dry +from examples.features.logits_processor.dry import ( + _MAX_CACHED_BREAKER_MASKS, + DEFAULT_DRY_SEQUENCE_BREAKERS, + DryState, + _dry_penalties, + use_dry, +) +from examples.features.logits_processor.dry import max_exponent as _max_exponent +from vllm.exceptions import VLLMValidationError +from vllm.sampling_params import SamplingParams + +DEVICE = torch.device("cpu") +DRY_FQCN = "examples.features.logits_processor.dry:DryState" + + +def _params(**dry_args) -> SamplingParams: + """SamplingParams carrying DRY arguments the way a client sends them.""" + return SamplingParams(extra_args=dry_args or None) + + +def _penalties(window, multiplier=1.0, base=2.0, allowed_length=1, breakers=()): + return _dry_penalties( + window, + frozenset(breakers), + multiplier, + base, + allowed_length, + _max_exponent(base), + ) + + +def test_worked_example(): + # llama.cpp's commented example (src/llama-sampler.cpp): tokens + # "a b c c b c y a b c" -> per-position repeat counts + # [0,0,3,1,0,2,0,0,0,0]; extenders: c (len 3), b (len 1), y (len 2). + # With multiplier=1, base=2, allowed_length=1: c -> 4, b -> 1, y -> 2. + got = _penalties([1, 2, 3, 3, 2, 3, 9, 1, 2, 3]) + assert got == {3: 4.0, 2: 1.0, 9: 2.0} + + +def test_z_box_equality_boundary(): + # The Z-algorithm's copy-vs-extend equality boundary: when + # cnt[last - p] == right_part_len the match may extend past the Z-box + # and must be re-scanned, not copied. Copying here drops + # token 0's penalty on this input. + got = _penalties([0, 0, 0, 1, 0, 0], allowed_length=2) + assert got == {0: 1.0, 1: 1.0} + + +def test_exponent_clamp_float32_semantics(): + # llama.cpp computes FLOAT_MAX_LOG / log(base) in float32. At base=2.0 + # that is exactly 128.0; float64 gives 127.99999998 -> 127. The clamp + # must land on 128 or long-repeat penalties come out half of + # llama.cpp's and never saturate to -inf. + assert _max_exponent(2.0) == 128 + assert _max_exponent(1.75) == 158 + assert _max_exponent(1.0) == 0 # disabled guard + assert _max_exponent(1.0000005) == 0 # llama.cpp's 1.000001 guard + + +def test_long_repeat_saturates_to_neg_inf(): + # 200 identical tokens at base=2.0: clamped exponent 128 -> + # 2**128 overflows float32 -> logit must land at exactly -inf, + # matching llama.cpp. Only the tensor path saturates, on the cast into + # dry_core's float32 accumulator. + state, all_tokens = _make_v2_state(max_num_reqs=1, vocab=16) + state.add_request( + 0, + _params( + dry_multiplier=1.0, + dry_base=2.0, + dry_allowed_length=0, + dry_sequence_breakers=[], + ), + ) + all_tokens[0, :200] = 7 + logits = torch.zeros(1, 16, device=DEVICE) + _apply(state, logits, np.array([0]), np.array([200])) + assert logits[0, 7].item() == -float("inf") + + +def test_allowed_length_threshold(): + # Match of length 2 is penalized at allowed_length=2, not at 3. + window = [1, 2, 3, 1, 2] + assert _penalties(window, allowed_length=3) == {} + assert _penalties(window, allowed_length=2) == {3: 1.0} + + +def test_breakers_cap_and_exclude(): + # An unbroken run is penalized; making that token a breaker + # collapses rep_limit to 0 and produces nothing. + assert _penalties([7] * 6, allowed_length=2) == {7: 8.0} + assert _penalties([7] * 6, allowed_length=2, breakers=[7]) == {} + + +def test_short_windows_are_safe(): + assert _penalties([]) == {} + assert _penalties([5]) == {} + + +class _StubTokenizer: + """Minimal tokenizer for breaker-resolution tests.""" + + _TEXTS = ["a", "\n", "b:\n", "hello", ":", "x:y", "*", "**"] + + vocab_size = len(_TEXTS) + + def batch_decode(self, ids_lists): + return [self._TEXTS[ids[0]] for ids in ids_lists] + + +def test_breaker_containment_resolution(): + # Resolution must match every token whose text contains the breaker + # string (llama.cpp get_overlapping_token_sequences), not only exact + # encodings. + from examples.features.logits_processor.dry import resolve_dry_breakers + + tok = _StubTokenizer() + assert resolve_dry_breakers(tok, ("\n",)) == [1, 2] + assert resolve_dry_breakers(tok, (":",)) == [2, 4, 5] + assert resolve_dry_breakers(tok, ("*",)) == [6, 7] + assert resolve_dry_breakers(tok, ()) == [] + # Repeat resolution is consistent, and callers get their own copy + # (mutating a result must not poison the cache). + first = resolve_dry_breakers(tok, ("\n",)) + first.append(999) + assert resolve_dry_breakers(tok, ("\n",)) == [1, 2] + + +def test_validate_params_rejects_json_booleans(): + for name in ( + "dry_multiplier", + "dry_base", + "dry_allowed_length", + "dry_penalty_last_n", + ): + with pytest.raises(ValueError, match=name): + DryState.validate_params(_params(**{name: True})) + + +def test_validate_params(): + for bad in ( + {"dry_multiplier": -1.0}, + {"dry_multiplier": float("nan")}, + {"dry_penalty_last_n": -7}, + {"dry_penalty_last_n": 2.5}, + {"dry_allowed_length": -1}, + {"dry_allowed_length": 1.5}, + {"dry_sequence_breakers": [1, 2]}, + {"dry_sequence_breakers": [f"b{i}" for i in range(65)]}, + ): + with pytest.raises(ValueError): + DryState.validate_params(_params(**bad)) + # Valid corner values pass, as does a request that says nothing at all. + DryState.validate_params(SamplingParams()) + DryState.validate_params(_params()) + DryState.validate_params(_params(dry_multiplier=0.0)) + DryState.validate_params(_params(dry_multiplier=1.0, dry_penalty_last_n=-1)) + DryState.validate_params(_params(dry_multiplier=1.0, dry_sequence_breakers=[])) + # An explicit null means "unset", as it does for every SamplingParams + # field; it must not be read as a non-numeric value and rejected. + DryState.validate_params(_params(dry_multiplier=None, dry_base=None)) + + +def test_validate_params_rejects_unknown_dry_keys(): + """An unknown dry_* key fails the request instead of being ignored.""" + with pytest.raises(ValueError, match="dry_multipliers"): + DryState.validate_params(_params(dry_multiplier=0.8, dry_multipliers=0.8)) + # Keys outside the namespace belong to somebody else; leave them alone. + DryState.validate_params(_params(dry_multiplier=0.8, target_token=3)) + + +def test_validate_params_raises_plain_valueerror(): + """validate_params raises ``ValueError``, as the V2 interface specifies.""" + with pytest.raises(ValueError) as excinfo: + DryState.validate_params(_params(dry_multiplier=-1.0)) + assert type(excinfo.value) is ValueError + + +def _oracle(window, breakers, multiplier, base, allowed_length, max_exponent): + """llama_sampler_dry_apply's steps, with a brute-force match scan in place of + the Z-algorithm that ``_dry_penalties`` ports.""" + m = len(window) + rep_limit = m + for i in range(m): + if window[m - 1 - i] in breakers: + rep_limit = i + break + if rep_limit < allowed_length: + return {} + cnt = [0] * m + for i in range(m - 1): + length = 0 + while ( + length < m + and i - length >= 0 + and window[i - length] == window[m - 1 - length] + ): + length += 1 + cnt[i] = min(length, rep_limit) + max_tok: dict[int, int] = {} + for i in range(m - 1): + if cnt[i] >= allowed_length: + tok = window[i + 1] + if max_tok.get(tok, -1) < cnt[i]: + max_tok[tok] = cnt[i] + penalties = {} + for tok, repeat_len in max_tok.items(): + if tok in breakers: + continue + exponent = repeat_len - allowed_length + if max_exponent and exponent > max_exponent: + exponent = max_exponent + penalties[tok] = multiplier * (base**exponent) + return penalties + + +def test_differential_against_oracle(): + rng = random.Random(0) + for case in range(400): + if case % 10 == 0: + # Long, highly repetitive: the only way to reach the clamp. + alphabet, n = rng.randint(1, 2), rng.randint(200, 320) + else: + alphabet, n = rng.randint(2, 6), rng.randint(0, 48) + window = [rng.randrange(alphabet) for _ in range(n)] + multiplier = rng.choice([0.5, 0.8, 1.0, 2.0]) + base = rng.choice([1.1, 1.75, 2.0, 3.0]) + allowed = rng.randint(0, 4) + breakers = frozenset( + rng.sample(range(alphabet), rng.randint(0, min(2, alphabet))) + ) + max_exp = _max_exponent(base) + want = _oracle(window, breakers, multiplier, base, allowed, max_exp) + got = _dry_penalties(window, breakers, multiplier, base, allowed, max_exp) + assert set(want) == set(got), f"case {case}: keys {want} vs {got}" + for tok in want: + a, b = want[tok], got[tok] + assert math.isclose(a, b, rel_tol=1e-9), ( + f"case {case} token {tok}: {a} vs {b}" + ) + + +def test_double_pow_saturation(): + # llama.cpp's std::pow(float, int) computes in double: 0.8 * 2**128 + # is finite in float32 (-2.72e38) while 1.0 * 2**128 saturates to + # -inf. A float32 pow gets the first case wrong. + for mult, expect_inf in ((0.8, False), (1.0, True)): + state, all_tokens = _make_v2_state(max_num_reqs=1, vocab=16) + state.add_request( + 0, + _params( + dry_multiplier=mult, + dry_base=2.0, + dry_allowed_length=0, + dry_sequence_breakers=[], + ), + ) + all_tokens[0, :200] = 7 + logits = torch.zeros(1, 16, device=DEVICE) + _apply(state, logits, np.array([0]), np.array([200])) + val = logits[0, 7].item() + if expect_inf: + assert val == -float("inf") + else: + assert math.isfinite(val) and math.isclose( + val, -0.8 * 2.0**128, rel_tol=1e-6 + ) + + +def test_dry_core_rejects_j_budget_over_the_budget(): + """dry_core raises on a j_budget over _J_BUDGET and accepts _J_BUDGET itself.""" + + def run(j_budget): + def col(value, dtype=torch.int64): + return torch.full((1,), value, dtype=dtype, device=DEVICE) + + dry.dry_core( + torch.zeros(1, 8, device=DEVICE), + row_idx=col(0), + W=torch.zeros(1, 4, dtype=torch.int64, device=DEVICE), + n_r=col(4), + allowed=col(2), + max_exp=col(1), + mult=col(1.0, torch.float32), + base=col(2.0, torch.float32), + breaker_masks=[None], + j_budget=j_budget, + ) + + run(dry._J_BUDGET) + with pytest.raises(ValueError, match="j_budget"): + run(dry._J_BUDGET + 1) + + +# --------------------------------------------------------------------------- +# The DryState custom logits processor +# --------------------------------------------------------------------------- + + +def _make_vllm_config(speculative_config=None, skip_tokenizer_init=True): + """Stub VllmConfig with only the fields DryState reads.""" + return SimpleNamespace( + speculative_config=speculative_config, + model_config=SimpleNamespace(skip_tokenizer_init=skip_tokenizer_init), + ) + + +def _make_v2_state( + max_num_reqs=8, + vocab=32, + max_model_len=256, + device=DEVICE, + tokenizer=None, +): + all_tokens = torch.zeros( + max_num_reqs, max_model_len, dtype=torch.int32, device=device + ) + req_states = SimpleNamespace( + max_num_reqs=max_num_reqs, + vocab_size=vocab, + device=device, + all_token_ids=SimpleNamespace(gpu=all_tokens), + ) + state = DryState(_make_vllm_config(), req_states) + if tokenizer is not None: + # Re-resolve the default set now that a tokenizer is installed. + state._tokenizer = tokenizer + state._default_breaker_ids = state._resolve_breakers( + DEFAULT_DRY_SEQUENCE_BREAKERS + ) + return state, all_tokens + + +def _apply(state, logits, idx_mapping_np, seq_lens_np): + """Call ``DryState.apply`` with a LogitsContext built as the sampler builds it. + + Args: + state: the ``DryState`` under test. + logits: [num_reqs, vocab] tensor, one row per request, modified in place. + idx_mapping_np: [num_reqs] batch position -> request slot. + seq_lens_np: [num_reqs] host-side context lengths. + + """ + from vllm.v1.worker.gpu.sample.logits_processor import LogitsContext + + n_reqs = idx_mapping_np.shape[0] + assert logits.shape[0] == n_reqs + idx_mapping = torch.from_numpy(idx_mapping_np).to(logits.device) + ctx = LogitsContext( + expanded_idx_mapping=idx_mapping, + idx_mapping=idx_mapping, + idx_mapping_np=idx_mapping_np, + expanded_local_pos=torch.zeros(n_reqs, dtype=torch.int64, device=logits.device), + input_ids=torch.zeros(n_reqs, dtype=torch.int32, device=logits.device), + pos=torch.from_numpy(seq_lens_np.astype("int64") - 1).to(logits.device), + seq_lens_upper_bound_np=seq_lens_np, + ) + return state.apply(logits, ctx) + + +def test_v2_worked_example_and_window_bounds(): + # The window is [0, seq_len), including the last input token. Without it every + # penalty lands on the continuation of the previous suffix instead of the one + # being sampled. + state, all_tokens = _make_v2_state() + params = _params( + dry_multiplier=1.0, + dry_base=2.0, + dry_allowed_length=1, + dry_sequence_breakers=[], + ) + state.add_request(3, params) + seq = [1, 2, 3, 3, 2, 3, 9, 1, 2, 3] + all_tokens[3, : len(seq)] = torch.tensor(seq, dtype=torch.int32) + + logits = torch.zeros(1, 32, device=DEVICE) + idx_mapping = np.array([3]) + # last input token sits at position len(seq)-1 + seq_lens = np.array([len(seq)]) + _apply(state, logits, idx_mapping, seq_lens) + assert logits[0, 3].item() == -4.0 + assert logits[0, 2].item() == -1.0 + assert logits[0, 9].item() == -2.0 + + +def test_v2_disabled_and_gating(): + state, _ = _make_v2_state() + assert not state.add_request(0, SamplingParams()) # dry off by default + assert not state.add_request(1, _params(dry_multiplier=1.0, dry_base=0.5)) + assert not state.add_request(2, _params(dry_multiplier=1.0, dry_penalty_last_n=0)) + assert not state.use_dry[:3].any() + # Re-adding a slot with DRY off after one with DRY on must clear it. + on = _params(dry_multiplier=0.8, dry_sequence_breakers=[]) + assert state.add_request(4, on) + assert state.use_dry[4] + assert not state.add_request(4, SamplingParams()) + assert not state.use_dry[4] + + +def test_v2_penalty_last_n_window(): + # A cap smaller than the history must limit the visible window. + state, all_tokens = _make_v2_state() + params = _params( + dry_multiplier=1.0, + dry_base=2.0, + dry_allowed_length=1, + dry_penalty_last_n=4, + dry_sequence_breakers=[], + ) + state.add_request(0, params) + seq = [7] * 20 + all_tokens[0, : len(seq)] = torch.tensor(seq, dtype=torch.int32) + logits = torch.zeros(1, 32, device=DEVICE) + _apply(state, logits, np.array([0]), np.array([20])) + # window = last 4 tokens of an identical run: longest match ending + # before the suffix is 3 (rep-limited by the window), penalty 2**(3-1). + assert logits[0, 7].item() == -4.0 + + +def test_spec_decode_is_refused_at_construction(): + """A speculative config makes DRY refuse to load.""" + req_states = SimpleNamespace( + max_num_reqs=4, + vocab_size=32, + device=DEVICE, + all_token_ids=SimpleNamespace(gpu=torch.zeros(4, 8, dtype=torch.int32)), + ) + config = _make_vllm_config(speculative_config=SimpleNamespace()) + with pytest.raises(ValueError, match="speculative decoding"): + DryState(config, req_states) + + +def test_v2_draft_expanded_logits_raise(): + """``DryState.apply`` raises on more logits rows than requests. + + Only speculative decoding lays logits out that way, and construction refuses it. + """ + from vllm.v1.worker.gpu.sample.logits_processor import LogitsContext + + state, _ = _make_v2_state() + state.add_request(0, _params(dry_multiplier=0.8, dry_sequence_breakers=[])) + # One request with one draft token has two logits rows. + rows = torch.zeros(2, dtype=torch.int64, device=DEVICE) + ctx = LogitsContext( + expanded_idx_mapping=rows, + idx_mapping=rows[:1], + idx_mapping_np=np.array([0]), + expanded_local_pos=torch.arange(2, device=DEVICE), + input_ids=torch.zeros(2, dtype=torch.int32, device=DEVICE), + pos=torch.tensor([7, 8], device=DEVICE), + seq_lens_upper_bound_np=np.array([9]), + ) + logits = torch.zeros(2, 32, device=DEVICE) + with pytest.raises(RuntimeError, match="draft-expanded"): + state.apply(logits, ctx) + assert not logits.any() + + +def test_constructs_without_speculative_config(): + state, _ = _make_v2_state() + assert isinstance(state, DryState) + assert state._tokenizer is None # skip_tokenizer_init in the stub config + + +@pytest.mark.parametrize("chunk_budget", [4096, dry._CHUNK_BYTE_BUDGET]) +def test_v2_matches_reference_fuzz(chunk_budget, monkeypatch): + # The V2 window-gather and routing path must agree with the sequential + # reference on randomized histories, including degenerate bases that + # route through the slow path. At these sizes the default budget is always one + # chunk. The 4096 B case splits most calls into several, to cover how per-chunk + # results combine. + monkeypatch.setattr(dry, "_CHUNK_BYTE_BUDGET", chunk_budget) + state, all_tokens = _make_v2_state(max_num_reqs=8, vocab=16, max_model_len=200) + rng = random.Random(7) + for trial in range(60): + n_reqs = rng.randint(1, 5) + histories = {} + seq_lens = [] + idx_list = [] + for r in range(n_reqs): + args: dict[str, Any] = dict( + dry_multiplier=rng.choice([0.8, 1.0, 2.0]), + dry_base=rng.choice([1.0000005, 1.1, 1.75, 2.0]), + dry_allowed_length=rng.randint(0, 3), + dry_penalty_last_n=rng.choice([-1, 5, 50]), + dry_sequence_breakers=[], + ) + state.add_request(r, _params(**args)) + # Breaker ids go straight into the state and must follow add_request, + # which pops the slot's ids. String resolution is tested above. + breakers = frozenset(rng.sample(range(4), rng.randint(0, 2))) + if breakers: + state.breaker_ids[r] = breakers + n = rng.randint(2, 180) + hist = [rng.randrange(rng.randint(1, 4) + 1) for _ in range(n)] + all_tokens[r, :n] = torch.tensor(hist, dtype=torch.int32) + histories[r] = (args, hist, breakers) + idx_list.append(r) + seq_lens.append(n) + + logits = torch.zeros(n_reqs, 16, device=DEVICE) + _apply(state, logits, np.array(idx_list), np.array(seq_lens)) + for r in range(n_reqs): + args, hist, breakers = histories[r] + base32 = float(np.float32(args["dry_base"])) + if base32 < 1.0 or not args["dry_multiplier"]: + want: dict[int, float] = {} + else: + last_n = args["dry_penalty_last_n"] + w = len(hist) if last_n == -1 else min(len(hist), last_n) + allowed = args["dry_allowed_length"] + window = hist[-w:] if w > allowed else None + want = ( + _dry_penalties( + window, + breakers, + float(np.float32(args["dry_multiplier"])), + base32, + allowed, + _max_exponent(base32), + ) + if window is not None + else {} + ) + for tok in range(16): + expected = -want.get(tok, 0.0) + got = logits[r, tok].item() + assert math.isclose(expected, got, rel_tol=1e-5, abs_tol=1e-6), ( + f"trial {trial} req {r} tok {tok}: {expected} vs {got}" + ) + + +def test_v2_batch_sharded_sampling_matches_unsharded(monkeypatch): + """Each rank's DRY rows under batch-sharded sampling match the unsharded rows. + + With ``--enable-batch-sharded-sampling``, ``BatchSharder`` gives every TP rank + a local batch of the requests it owns and localizes the two fields DRY reads + from ``LogitsContext``: ``idx_mapping_np``, and ``seq_lens_cpu_upper_bound``, + which the sampler passes as ``seq_lens_upper_bound_np``. Its Triton plan kernel + writes only device fields DRY does not read, so it is replaced here and the + sharding runs on the host. + """ + from dataclasses import replace + from unittest import mock + + from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers + from vllm.v1.worker.gpu.sample import batch_shard + + vocab = 16 + state, all_tokens = _make_v2_state( + max_num_reqs=8, vocab=vocab, tokenizer=_StubTokenizer() + ) + # Slots out of batch order, so a rank's rows are not its slots, and only rank 2 at + # tp_size 3 owns a prefix of the batch. Slot 3 has DRY off, slot 6 takes the + # sequential path and slot 0 caps its window. Slot 5 breaks on "\n" (tokens 1 + # and 2), slot 6 on "a" (token 0), and slots 7 and 1 on "b" (token 2). + slots = np.array([5, 2, 7, 0, 3, 6, 1]) + seq_lens = np.array([40, 23, 57, 31, 12, 48, 36]) + breakers = {5: ["\n"], 6: ["a"], 7: ["b"], 1: ["b"]} + rng = random.Random(5) + for slot, n in zip(slots.tolist(), seq_lens.tolist()): + args: dict[str, Any] = dict( + dry_multiplier=0.8, dry_sequence_breakers=breakers.get(slot, []) + ) + if slot == 6: + args["dry_base"] = 1.0000005 + if slot == 0: + args["dry_penalty_last_n"] = 16 + state.add_request(slot, SamplingParams() if slot == 3 else _params(**args)) + hist = [rng.randrange(3) for _ in range(n)] + all_tokens[slot, :n] = torch.tensor(hist, dtype=torch.int32) + assert state.breaker_ids == {5: {1, 2}, 6: {0}, 7: {2}, 1: {2}} + full = torch.zeros(len(slots), vocab, device=DEVICE) + _apply(state, full, slots, seq_lens) + assert (full < 0).any(dim=1).tolist() == [s != 3 for s in slots.tolist()] + + buffers = InputBuffers(max_num_reqs=8, max_num_tokens=len(slots), device=DEVICE) + batch = replace( + InputBatch.make_dummy(len(slots), len(slots), buffers), + req_ids=[f"req-{i}" for i in range(len(slots))], + idx_mapping=torch.from_numpy(slots).to(DEVICE), + idx_mapping_np=slots, + seq_lens_cpu_upper_bound=torch.from_numpy(seq_lens), + ) + monkeypatch.setattr(batch_shard, "_build_shard_plan_kernel", mock.MagicMock()) + for tp_size in (2, 3): + covered: list[int] = [] + for rank in range(tp_size): + group = SimpleNamespace(rank_in_group=rank, world_size=tp_size) + with mock.patch.object(batch_shard, "get_tp_group", return_value=group): + sharder = batch_shard.BatchSharder( + max_num_reqs=8, max_num_logits_per_req=1, device=DEVICE + ) + local, _, _, _ = sharder.shard_sampler_inputs(batch, None) + owned = [int(req_id.split("-")[1]) for req_id in local.req_ids] + covered += owned + logits = torch.zeros(len(owned), vocab, device=DEVICE) + seq_lens_np = local.seq_lens_cpu_upper_bound.numpy() + _apply(state, logits, local.idx_mapping_np, seq_lens_np) + assert torch.equal(logits, full[owned]), f"tp_size {tp_size} rank {rank}" + assert sorted(covered) == list(range(len(slots))) + + +def test_v2_breaker_strings_resolve_and_reach_dry_core(): + # Breaker strings arrive in extra_args and are resolved by the processor's + # own tokenizer; the ids must reach the penalty computation, and the slot + # must not inherit them when it is reused. + state, all_tokens = _make_v2_state(tokenizer=_StubTokenizer()) + breaking = _params( + dry_multiplier=1.0, + dry_base=2.0, + dry_allowed_length=1, + dry_sequence_breakers=["*"], + ) + state.add_request(0, breaking) + # "*" is token 6 and is contained in token 7 ("**"), so both break. + assert state.breaker_ids[0] == frozenset({6, 7}) + all_tokens[0, :8] = 7 + logits = torch.zeros(1, 32, device=DEVICE) + _apply(state, logits, np.array([0]), np.array([8])) + assert logits[0, 7].item() == 0.0 + + fresh = _params( + dry_multiplier=1.0, + dry_base=2.0, + dry_allowed_length=1, + dry_sequence_breakers=[], + ) + state.add_request(0, fresh) + assert 0 not in state.breaker_ids + logits2 = torch.zeros(1, 32, device=DEVICE) + _apply(state, logits2, np.array([0]), np.array([8])) + assert logits2[0, 7].item() < 0.0 + + +def test_breaker_mask_cache_is_bounded(): + """The device-mask cache must not grow with the number of distinct sets.""" + state, _ = _make_v2_state(vocab=128) + sets = [frozenset({i}) for i in range(_MAX_CACHED_BREAKER_MASKS + 8)] + for breakers in sets: + state.breaker_ids[0] = breakers + assert state._breaker_mask(0) is not None + assert len(state._breaker_masks) <= _MAX_CACHED_BREAKER_MASKS + # The first set was evicted and the newest kept. + assert sets[-1] in state._breaker_masks + assert sets[0] not in state._breaker_masks + + +def test_default_breaker_set_is_resolved_eagerly(): + """A request that names no breakers must not pay to resolve them.""" + state, _ = _make_v2_state(tokenizer=_StubTokenizer()) + # "\n" -> {1, 2}, ":" -> {2, 4, 5}, '"' -> {}, "*" -> {6, 7}. + assert state._default_breaker_ids == frozenset({1, 2, 4, 5, 6, 7}) + state.add_request(0, _params(dry_multiplier=0.8)) + assert state.breaker_ids[0] is state._default_breaker_ids + # Naming the default set explicitly must land on the same object. + state.add_request( + 1, + _params( + dry_multiplier=0.8, + dry_sequence_breakers=list(DEFAULT_DRY_SEQUENCE_BREAKERS), + ), + ) + assert state.breaker_ids[1] is state._default_breaker_ids + + +def test_init_warms_up_dry_core_once(monkeypatch): + """__init__ calls dry_core once, the way apply calls it.""" + calls = [] + # Taken before the patch below replaces it in the same module. + original = dry.dry_core + + def spy(logits, **kwargs): + calls.append((logits, kwargs)) + return original(logits, **kwargs) + + def kernel_args(logits, kwargs): + def sig(t): + return None if t is None else (t.dtype, t.dim(), t.device) + + masks = {sig(m) for m in kwargs["breaker_masks"]} + other = { + name: sig(arg) if isinstance(arg, torch.Tensor) else type(arg) + for name, arg in kwargs.items() + if name != "breaker_masks" + } + return sig(logits), masks, other + + monkeypatch.setattr("examples.features.logits_processor.dry.dry_core", spy) + _make_v2_state(vocab=1) + assert not calls, "a vocabulary of one leaves no id for the breaker" + state, all_tokens = _make_v2_state(vocab=64, tokenizer=_StubTokenizer()) + assert len(calls) == 1 + assert (calls[0][0] < 0).any(), "the warmup matched nothing" + assert not state._breaker_masks + assert not state.breaker_ids + assert not state.use_dry.any() + + state.add_request(0, _params(dry_multiplier=0.8)) + all_tokens[0, :8] = 3 + _apply(state, torch.zeros(1, 64, device=DEVICE), np.array([0]), np.array([8])) + assert len(calls) == 2 + assert kernel_args(*calls[0]) == kernel_args(*calls[1]) + + +def test_no_tokenizer_warns_once_and_still_penalizes(): + # Without a tokenizer, breaker strings are dropped with one warning. + state, all_tokens = _make_v2_state() + assert state._tokenizer is None + assert state._default_breaker_ids == frozenset() + state.add_request(0, _params(dry_multiplier=0.8)) + assert state._warned_unresolved + assert 0 not in state.breaker_ids + all_tokens[0, :8] = 7 + logits = torch.zeros(1, 32, device=DEVICE) + _apply(state, logits, np.array([0]), np.array([8])) + assert logits[0, 7].item() < 0.0 + + +def test_v2_degenerate_base_routes_to_unclamped_slow_path(): + # base <= 1.000001 gives max_exponent == 0, which in the sequential + # reference means "no clamp"; the vectorized path would clamp the + # exponent to 0, so such requests must route to the reference scan. + state, all_tokens = _make_v2_state() + params = _params( + dry_multiplier=1.0, + dry_base=1.0000005, + dry_allowed_length=0, + dry_sequence_breakers=[], + ) + state.add_request(0, params) + all_tokens[0, :150] = 7 + logits = torch.zeros(1, 32, device=DEVICE) + _apply(state, logits, np.array([0]), np.array([150])) + base32 = float(np.float32(1.0000005)) + want = -1.0 * base32**149 + got = logits[0, 7].item() + assert math.isclose(got, want, rel_tol=1e-6) + assert got != -1.0 + + +def test_breaker_resolution_covers_added_tokens(): + class AddedTokenTokenizer: + _TEXTS = ["a", "\n", "b", "<|im_start|>"] + vocab_size = 3 # the added token sits above vocab_size + max_token_id = 3 + + def batch_decode(self, ids_lists): + return [self._TEXTS[ids[0]] for ids in ids_lists] + + from examples.features.logits_processor.dry import resolve_dry_breakers + + tok = AddedTokenTokenizer() + assert resolve_dry_breakers(tok, ("<|im_start|>",)) == [3] + assert resolve_dry_breakers(tok, ("\n",)) == [1] + + +class _TextsTokenizer: + """Tokenizer stub over an explicit list of decoded token texts.""" + + def __init__(self, texts: list[str], vocab_size: int): + self._texts = texts + self.vocab_size = vocab_size + self.max_token_id = len(texts) - 1 + + def batch_decode(self, ids_lists): + return [self._texts[ids[0]] for ids in ids_lists] + + +def _containment_scan(texts: list[str], breaker_strs: tuple[str, ...]) -> list[int]: + """Resolution as a plain scan: the semantics the character index must keep.""" + ids: set[int] = set() + for s in breaker_strs: + ids.update(i for i, text in enumerate(texts) if s in text) + return sorted(ids) + + +def test_breaker_index_matches_containment_scan(): + """Breaker resolution must match a plain containment scan. + + Every character of a synthetic vocabulary is checked, including multi-byte + and combining ones, added tokens and an empty token, then multi-character + and mixed breaker sets. + """ + from examples.features.logits_processor.dry import ( + _MAX_BREAKER_CHAR_LEN, + _vocab_index, + resolve_dry_breakers, + ) + + rng = random.Random(11) + # U+4E2D and U+1F600 are multi-byte in UTF-8. "e\u0301" is two code points, so + # U+0301 alone resolves by lookup and the pair by scan. + alphabet = list('abcde \n:"*.,') + ["\u4e2d", "\u00e9", "e\u0301", "\U0001f600"] + texts = [""] # a token that decodes to nothing + for _ in range(8000): + texts.append("".join(rng.choices(alphabet, k=rng.randint(1, 12)))) + vocab_size = len(texts) + # Added tokens, above vocab_size; one holds the vocabulary's only U+2028. + texts += ["<|im_start|>", "<|im_end|>", "\u2028sep"] + tok = _TextsTokenizer(texts, vocab_size) + + def check(breakers: tuple[str, ...]) -> list[int]: + kept = tuple(s[:_MAX_BREAKER_CHAR_LEN] for s in breakers if s) + want = _containment_scan(texts, kept) + assert resolve_dry_breakers(tok, breakers) == want, breakers + return want + + # Every character present is indexed, so a lookup miss means "in no token". + present = sorted(set("".join(texts))) + assert set(_vocab_index(tok).char_ids) == set(present) + for char in present: + check((char,)) + assert len(check(("a",))) > 2000 # a character in thousands of tokens + assert check(("\u2028",)) == [len(texts) - 1] # only above vocab_size + for absent in ("q", "Z", "\u2603", "\U0001f601", "\u0300"): + assert check((absent,)) == [] + + substrings = [] + for _ in range(300): + text = rng.choice(texts[1:vocab_size]) + start = rng.randrange(len(text)) + substrings.append(text[start : start + rng.randint(2, 5)]) + for s in substrings + ["<|im_start|>", "e\u0301", "\u2028sep", "zz", "a" * 41]: + check((s,)) + for _ in range(200): + check(tuple(rng.sample(present, rng.randint(1, 4)) + rng.sample(substrings, 2))) + assert check(()) == [] + assert check(("",)) == [] + assert check(("", "\n")) == check(("\n",)) + + +# The accumulator scales with R and the chunk transients do not, hence two batch +# sizes. The breaker path's masks get their own case. +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize( + ("n_reqs", "limit_mib", "n_breakers"), + [(32, 128, 0), (256, 256, 0), (256, 288, 3900)], +) +def test_peak_memory_bounded(n_reqs, limit_mib, n_breakers): + # base=1.1 gives max_exponent=930, a large J for the vectorized path + # (the cap is allowed_length + max_exponent <= _J_BUDGET = 2048). + device = torch.device("cuda") + rng = random.Random(3) + window = 2048 + state, all_tokens = _make_v2_state( + max_num_reqs=n_reqs, vocab=128256, max_model_len=window, device=device + ) + params = _params( + dry_multiplier=0.8, + dry_base=1.1, + dry_allowed_length=2, + dry_sequence_breakers=[], + ) + breaker_ids: frozenset[int] = frozenset() + if n_breakers: + # About the size of llama.cpp's default set on a Llama-3 tokenizer. Id 0 is + # in the history, so breakers cut some matches but not all. + breaker_ids = frozenset({0} | set(rng.sample(range(8, 128256), n_breakers - 1))) + # Eight symbols, so 2-token repeats are common. + hist = torch.tensor( + [[rng.randrange(8) for _ in range(window)] for _ in range(n_reqs)], + dtype=torch.int32, + ) + all_tokens.copy_(hist.to(device)) + for r in range(n_reqs): + state.add_request(r, params) + if breaker_ids: + state.breaker_ids[r] = breaker_ids + logits = torch.zeros(n_reqs, 128256, device=device) + torch.accelerator.synchronize() + torch.accelerator.reset_peak_memory_stats() + base_alloc = torch.accelerator.memory_allocated() + _apply(state, logits, np.arange(n_reqs), np.full(n_reqs, window)) + torch.accelerator.synchronize() + peak = torch.accelerator.max_memory_allocated() - base_alloc + charged = int((logits < 0).sum().item()) + assert charged > 0, "no token was penalized" + # The limits catch the exponent and pow being held at full [R, vocab] width. + limit = limit_mib * 1024 * 1024 + assert peak < limit, f"peak {peak / 2**20:.1f} MiB over {limit / 2**20:.0f} MiB" + + +# --------------------------------------------------------------------------- +# Plugin wiring: loading by FQCN, and validation at request admission. +# --------------------------------------------------------------------------- + + +def test_loads_through_the_loader_by_fqcn(): + """The example's FQCN must resolve and construct.""" + req_states = SimpleNamespace( + device=DEVICE, + max_num_reqs=4, + vocab_size=32, + all_token_ids=SimpleNamespace(gpu=torch.zeros(4, 8, dtype=torch.int32)), + prompt_len=None, + prefill_len=None, + total_len=None, + ) + procs = loader.build_custom_logits_processors( + _make_vllm_config(), req_states, False, [DRY_FQCN] + ) + assert [type(p) for p in procs] == [DryState] + + +def test_loader_import_does_not_initialize_cuda(): + """Importing dry.py must not initialize CUDA. + + The frontend imports it under ``guard_cuda_initialization()`` to validate + params, and a device touched at import time fails the engine at startup. + """ + code = ( + "import torch\n" + "from vllm.utils.torch_utils import guard_cuda_initialization\n" + "with guard_cuda_initialization():\n" + " import examples.features.logits_processor.dry # noqa: F401\n" + "assert not torch.cuda.is_initialized()\n" + ) + # From the repository root, where `examples` is importable. + subprocess.run( + [sys.executable, "-c", code], check=True, cwd=Path(__file__).parents[3] + ) + + +def test_admission_validator_rejects_bad_extra_args(): + """The real frontend validator must turn a bad dry_* value into a 400. + + ``build_custom_logits_processors_params_validator`` is what + ``InputProcessor`` calls per request; it wraps a processor's ``ValueError`` + as ``VLLMValidationError``, which the entrypoints map to a client error. + """ + loader._cached_load_v2_logitsprocs.cache_clear() + validate = loader.build_custom_logits_processors_params_validator([DRY_FQCN]) + validate(SamplingParams()) + validate(_params(dry_multiplier=0.8)) + with pytest.raises(VLLMValidationError, match="dry_multiplier"): + validate(_params(dry_multiplier=-1.0)) + with pytest.raises(VLLMValidationError, match="dry_multipliers"): + validate(_params(dry_multiplier=0.8, dry_multipliers=0.8)) + + +@pytest.mark.parametrize("field", ["dry_penalty_last_n", "dry_allowed_length"]) +def test_dry_int_params_upper_bound(field): + """Both integer fields are capped at INT32_MAX; see ``_DRY_INT_MAX``.""" + for bad in (2**63, 2**31): + with pytest.raises(ValueError, match=field): + DryState.validate_params(_params(dry_multiplier=0.8, **{field: bad})) + # The boundary itself is legal, and so are the ordinary values. + DryState.validate_params(_params(dry_multiplier=0.8, **{field: 2**31 - 1})) + DryState.validate_params(_params(dry_multiplier=0.8, **{field: 2})) + + +def test_dry_base_below_one_warns_that_it_disabled_dry(): + """`dry_base < 1.0` disables DRY, as in llama.cpp; validate_params warns. + + Patches the module logger because vLLM's logger does not propagate to caplog. + """ + from unittest.mock import patch + + with patch("examples.features.logits_processor.dry.logger") as log: + DryState.validate_params(_params(dry_multiplier=0.8, dry_base=0.8)) + assert not use_dry(0.8, 0.8, -1), "base < 1.0 must disable DRY" + assert log.warning.called, "expected a warning" + assert "disables DRY" in log.warning.call_args[0][0] + + # A normal configuration says nothing. + with patch("examples.features.logits_processor.dry.logger") as log: + DryState.validate_params(_params(dry_multiplier=0.8, dry_base=1.75)) + assert use_dry(0.8, 1.75, -1) + assert not log.warning.called + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_result_does_not_depend_on_the_chunk_budget(): + """Logits must not depend on how the offsets are chunked.""" + from examples.features.logits_processor import dry as dc + + device = torch.device("cuda") + vocab, window, n_reqs = 64, 600, 1 + # A short alphabet repeated, so one follower token recurs at offsets far + # enough apart to land in different chunks. + W = torch.tensor([([5, 6, 7] * 200)[:window]], dtype=torch.int64, device=device) + col = lambda v, dt=torch.int64: torch.full((n_reqs,), v, dtype=dt, device=device) # noqa: E731 + + def run(budget): + saved = dc._CHUNK_BYTE_BUDGET + dc._CHUNK_BYTE_BUDGET = budget + try: + logits = torch.zeros(n_reqs, vocab, device=device) + dc.dry_core( + logits, + torch.arange(n_reqs, device=device), + W, + col(window), + col(2), + col(60), + col(0.8, torch.float32), + col(1.75, torch.float32), + [None] * n_reqs, + j_budget=62, + ) + return logits + finally: + dc._CHUNK_BYTE_BUDGET = saved + + many = run(4096) + one = run(1 << 30) + assert torch.equal(many, one), ( + f"chunking changed the result: {many[0][many[0] < 0][:3].tolist()} vs " + f"{one[0][one[0] < 0][:3].tolist()}" + ) + assert (many < 0).any(), "nothing was penalized" From 5dd693122c51198345847f384648fc5ed6314157 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Tue, 29 Sep 2026 21:27:45 +0100 Subject: [PATCH 051/248] [MyPy][2/N] Fix mypy errors in small tests/ dirs (#51043) Signed-off-by: Martin Hickey Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../passes/distributed/test_async_tp.py | 2 +- .../distributed/test_fusion_all_reduce.py | 4 +- .../distributed/test_sequence_parallelism.py | 2 +- tests/compile/passes/test_fusion_attn.py | 4 +- .../test_mla_rope_kvcache_cat_fusion.py | 3 +- ..._rocm_aiter_qk_norm_rope_kvcache_fusion.py | 7 +- .../passes/test_rope_kvcache_fusion.py | 5 +- .../passes/test_silu_mul_quant_fusion.py | 9 ++- .../test_silu_mul_quant_manual_fusion.py | 2 +- .../test_api_server_process_manager.py | 7 +- .../serve/lora/test_serving_models.py | 4 +- .../test_authentication_middleware.py | 5 +- .../tokenize/test_serving_tokenization.py | 9 +-- .../serve/tokenize/test_tokenization.py | 8 +-- .../serve/utils/test_fingerprint.py | 9 ++- .../entrypoints/unit_tests/test_chat_utils.py | 34 +++++----- tests/kernels/quantization/test_block_fp8.py | 5 +- .../quantization/test_cutlass_scaled_mm.py | 5 +- .../kernels/quantization/test_cutlass_w4a8.py | 5 +- .../quantization/test_cutlass_w4a8_moe.py | 3 +- tests/kernels/quantization/test_fp8_quant.py | 8 ++- tests/kernels/quantization/test_machete_mm.py | 7 +- .../kernels/quantization/test_marlin_gemm.py | 2 +- .../quantization/test_marlin_tile_padding.py | 7 +- .../quantization/test_nvfp4_emulation.py | 2 +- .../test_scaled_mm_kernel_selection.py | 2 +- .../layers/test_fused_shared_expert.py | 22 ++++--- .../layers/test_mla_short_prefill_indexer.py | 8 +-- .../layers/test_pooler_activations.py | 4 +- .../layers/test_pooler_heads.py | 10 ++- .../layers/test_pooler_methods.py | 5 +- ...test_bge_m3_sparse_io_processor_plugins.py | 3 +- ...test_colbert_query_io_processor_plugins.py | 2 +- tests/plugins_tests/test_endpoint_plugins.py | 4 +- .../test_oot_registration_offline.py | 2 +- .../test_stats_logger_plugins.py | 8 ++- .../test_terratorch_io_processor_plugins.py | 3 +- tests/samplers/test_beam_search.py | 10 +-- tests/samplers/test_beam_search_online.py | 15 +++-- tests/samplers/test_no_bad_words.py | 13 ++-- tests/tokenizers_/test_detokenize.py | 28 ++++++-- tests/tokenizers_/test_hf.py | 4 +- tests/tokenizers_/test_mistral.py | 38 +++++++---- tests/tokenizers_/test_registry.py | 2 +- .../test_gemma4_responses_adjust_request.py | 2 +- tests/tool_use/test_kimi_k3_tool_parser.py | 49 ++++++++------ tests/tool_use/test_muse_glimmer.py | 56 ++++++++++------ .../test_responses_request_validations.py | 1 + tests/tool_use/test_tool_choice_required.py | 11 ++-- tests/utils_/test_argparse_utils.py | 18 +++--- tests/utils_/test_cache.py | 2 +- tests/utils_/test_network_utils.py | 4 +- tests/v1/cudagraph/test_cudagraph_dispatch.py | 6 +- tests/v1/cudagraph/test_cudagraph_manager.py | 9 +-- tests/v1/cudagraph/test_encoder_cudagraph.py | 64 +++++++++++++++++-- tests/v1/determinism/test_batch_invariance.py | 17 ++--- .../determinism/test_batch_invariance_vlm.py | 9 +-- .../test_cutlass_batch_invariance.py | 14 ++-- .../determinism/test_nvfp4_batch_invariant.py | 2 +- .../test_rms_norm_batch_invariant.py | 2 + tests/v1/executor/test_executor.py | 10 +-- .../v1/simple_kv_offload/test_integration.py | 1 + tests/v1/simple_kv_offload/test_kv_events.py | 6 +- tests/v1/simple_kv_offload/test_scheduler.py | 9 +-- tests/v1/simple_kv_offload/test_worker.py | 43 +++++++------ .../test_backend_guidance.py | 21 +++--- .../test_backend_xgrammar_lark.py | 9 ++- .../test_backend_xgrammar_plamo3.py | 6 +- .../test_structured_output_manager.py | 27 ++++---- tests/v1/structured_output/test_validation.py | 21 +++--- tools/pre_commit/mypy.py | 32 +++++----- 71 files changed, 491 insertions(+), 301 deletions(-) diff --git a/tests/compile/passes/distributed/test_async_tp.py b/tests/compile/passes/distributed/test_async_tp.py index c0fd4c00a8..6601f7a2f0 100644 --- a/tests/compile/passes/distributed/test_async_tp.py +++ b/tests/compile/passes/distributed/test_async_tp.py @@ -285,7 +285,7 @@ def test_async_tp_pass_requires_full_graph_compilation(): ] async_tp_pass = object.__new__(AsyncTPPass) - async_tp_pass.compilation_config = vllm_config.compilation_config + async_tp_pass.compilation_config = vllm_config.compilation_config # type: ignore[assignment] with pytest.raises( AssertionError, match="AsyncTPPass requires full-graph compilation" diff --git a/tests/compile/passes/distributed/test_fusion_all_reduce.py b/tests/compile/passes/distributed/test_fusion_all_reduce.py index 1c99cc2533..a7b4cf0ed0 100644 --- a/tests/compile/passes/distributed/test_fusion_all_reduce.py +++ b/tests/compile/passes/distributed/test_fusion_all_reduce.py @@ -111,7 +111,7 @@ def test_fused_ar_workspace_hidden_dim( target_hidden: int, draft_hidden: int | None, expected: int ): config = _make_workspace_hidden_config(target_hidden, draft_hidden) - assert _fused_ar_workspace_hidden_dim(config) == expected + assert _fused_ar_workspace_hidden_dim(config) == expected # type: ignore[arg-type] def test_fused_ar_workspace_hidden_dim_spec_without_draft(): @@ -120,7 +120,7 @@ def test_fused_ar_workspace_hidden_dim_spec_without_draft(): model_config=SimpleNamespace(get_hidden_size=lambda: 2048), speculative_config=SimpleNamespace(draft_model_config=None), ) - assert _fused_ar_workspace_hidden_dim(config) == 2048 + assert _fused_ar_workspace_hidden_dim(config) == 2048 # type: ignore[arg-type] class TestAllReduceRMSNormModel(torch.nn.Module): diff --git a/tests/compile/passes/distributed/test_sequence_parallelism.py b/tests/compile/passes/distributed/test_sequence_parallelism.py index c40d75f675..b5c390e24c 100644 --- a/tests/compile/passes/distributed/test_sequence_parallelism.py +++ b/tests/compile/passes/distributed/test_sequence_parallelism.py @@ -222,7 +222,7 @@ def test_sequence_parallelism_pass_requires_full_graph_compilation(): ] sequence_parallelism_pass = object.__new__(SequenceParallelismPass) - sequence_parallelism_pass.compilation_config = vllm_config.compilation_config + sequence_parallelism_pass.compilation_config = vllm_config.compilation_config # type: ignore[assignment] sequence_parallelism_pass.min_token_num = 1 with pytest.raises( diff --git a/tests/compile/passes/test_fusion_attn.py b/tests/compile/passes/test_fusion_attn.py index ccc7e624db..759970f14d 100644 --- a/tests/compile/passes/test_fusion_attn.py +++ b/tests/compile/passes/test_fusion_attn.py @@ -33,6 +33,7 @@ from vllm.config import ( VllmConfig, set_current_vllm_config, ) +from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.quantization.utils.quant_utils import ( @@ -198,6 +199,7 @@ class TestAttentionAiterFp8StaticQuantPatternModel( def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fp8_linear.kernel.quant_fp8.use_aiter = True + assert self.fp8_linear.input_scale is not None self.fp8_linear.input_scale.fill_(0.125) @@ -307,7 +309,7 @@ def test_attention_quant_pattern( head_size: int, batch_size: int, dtype: torch.dtype, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, custom_ops: str, model_name: str, model_class: type[AttentionQuantPatternModel], diff --git a/tests/compile/passes/test_mla_rope_kvcache_cat_fusion.py b/tests/compile/passes/test_mla_rope_kvcache_cat_fusion.py index 3c894ee141..6031ecda0c 100644 --- a/tests/compile/passes/test_mla_rope_kvcache_cat_fusion.py +++ b/tests/compile/passes/test_mla_rope_kvcache_cat_fusion.py @@ -24,6 +24,7 @@ from vllm.config import ( PassConfig, VllmConfig, ) +from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import MLAAttention from vllm.model_executor.layers.linear import ColumnParallelLinear @@ -254,7 +255,7 @@ def test_mla_rope_kvcache_cat_fusion( block_size: int, is_neox: bool, dtype: torch.dtype, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch, ): torch.set_default_device("cuda") diff --git a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py index ee557235fd..c32469bfe2 100644 --- a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py +++ b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py @@ -39,6 +39,7 @@ from vllm.config import ( PassConfig, VllmConfig, ) +from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.layernorm import RMSNorm @@ -342,7 +343,7 @@ def _run_qk_norm_rope_kvcache_fusion_test( use_shuffle_kv_layout: str, kv_layout: KVCacheLayout, dtype: torch.dtype, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, rms_norm_eps: float, custom_op: str, monkeypatch: pytest.MonkeyPatch, @@ -611,7 +612,7 @@ def test_qk_norm_rope_kvcache_fusion( kv_layout: KVCacheLayout, block_size: int, dtype: torch.dtype, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, rms_norm_eps: float, custom_op: str, monkeypatch: pytest.MonkeyPatch, @@ -663,7 +664,7 @@ def test_qk_norm_mrope_kvcache_fusion( mrope_interleaved: bool, is_neox: bool, kv_layout: KVCacheLayout, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch, ): _run_qk_norm_rope_kvcache_fusion_test( diff --git a/tests/compile/passes/test_rope_kvcache_fusion.py b/tests/compile/passes/test_rope_kvcache_fusion.py index ea3454227b..8327bfbd45 100644 --- a/tests/compile/passes/test_rope_kvcache_fusion.py +++ b/tests/compile/passes/test_rope_kvcache_fusion.py @@ -30,6 +30,7 @@ from vllm.config import ( PassConfig, VllmConfig, ) +from vllm.config.cache import CacheDType from vllm.config.utils import Range from vllm.forward_context import get_forward_context, set_forward_context from vllm.model_executor.layers.attention import Attention @@ -324,7 +325,7 @@ def test_rope_kvcache_fusion( block_size: int, is_neox: bool, dtype: torch.dtype, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch, ): torch.set_default_device("cuda") @@ -468,7 +469,7 @@ def test_rope_static_qquant_kvcache_fusion( block_size: int, is_neox: bool, dtype: torch.dtype, - kv_cache_dtype: str, + kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch, ): torch.set_default_device("cuda") diff --git a/tests/compile/passes/test_silu_mul_quant_fusion.py b/tests/compile/passes/test_silu_mul_quant_fusion.py index 7d291cc504..a397dd9406 100644 --- a/tests/compile/passes/test_silu_mul_quant_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_fusion.py @@ -60,7 +60,7 @@ class TestSiluMulFp8QuantModel(torch.nn.Module): def __init__( self, hidden_size: int, - force_kernel: FP8ScaledMMLinearKernel, + force_kernel: type[FP8ScaledMMLinearKernel], dtype: torch.dtype, **kwargs, ): @@ -282,7 +282,7 @@ def test_fusion_silu_and_mul_quant( ], enable_silu_mul_custom_op: bool, enable_quant_fp8_custom_op: bool, - force_kernel: FP8ScaledMMLinearKernel | None, + force_kernel: type[FP8ScaledMMLinearKernel] | None, monkeypatch: pytest.MonkeyPatch, ): if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported(): @@ -330,7 +330,10 @@ def test_fusion_silu_and_mul_quant( passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)] backend = TestBackend(*passes) model = model_class( - hidden_size=hidden_size, force_kernel=force_kernel, x=x, dtype=dtype + hidden_size=hidden_size, + force_kernel=force_kernel, # type: ignore[arg-type] + x=x, + dtype=dtype, ) # First dimension dynamic diff --git a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py index 10dc401a26..8bb6a7e1f1 100644 --- a/tests/compile/passes/test_silu_mul_quant_manual_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_manual_fusion.py @@ -84,7 +84,7 @@ def test_manual_fusion_fp8_static_with_linear( num_tokens: int, hidden_size: int, dtype: torch.dtype, - force_kernel: FP8ScaledMMLinearKernel, + force_kernel: type[FP8ScaledMMLinearKernel], ): """Test manual fusion with real FP8 linear layer (kFp8StaticTensorSym). diff --git a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py index 80a2577dba..b4390f2dc4 100644 --- a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py +++ b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py @@ -307,7 +307,8 @@ def test_external_process_monitoring(api_server_args): def run_with_exception_capture(): try: wait_for_completion_or_failure( - api_server_manager=manager, coordinator=mock_coordinator + api_server_manager=manager, + coordinator=mock_coordinator, # type: ignore[arg-type] ) except Exception as e: result["exception"] = e @@ -376,7 +377,7 @@ def test_gather_actual_addresses_end_to_end(): manager = APIServerProcessManager( listen_address=f"tcp://{host}:0", sock=sock, - args="test_args", + args="test_args", # type: ignore[arg-type] num_servers=num_servers, input_addresses=placeholder_inputs, output_addresses=placeholder_outputs, @@ -423,7 +424,7 @@ def test_gather_actual_addresses_child_crash_before_report(): manager = APIServerProcessManager( listen_address=f"tcp://{host}:0", sock=sock, - args="test_args", + args="test_args", # type: ignore[arg-type] num_servers=num_servers, input_addresses=placeholder_inputs, output_addresses=placeholder_outputs, diff --git a/tests/entrypoints/serve/lora/test_serving_models.py b/tests/entrypoints/serve/lora/test_serving_models.py index 755f875333..3cbb0a480d 100644 --- a/tests/entrypoints/serve/lora/test_serving_models.py +++ b/tests/entrypoints/serve/lora/test_serving_models.py @@ -108,8 +108,8 @@ async def test_unload_lora_adapter_success(): response = await serving_models.load_lora_adapter(request) assert len(serving_models.lora_requests) == 1 - request = UnloadLoRAAdapterRequest(lora_name="adapter1") - response = await serving_models.unload_lora_adapter(request) + unload_request = UnloadLoRAAdapterRequest(lora_name="adapter1") + response = await serving_models.unload_lora_adapter(unload_request) assert response == LORA_UNLOADING_SUCCESS_MESSAGE.format(lora_name="adapter1") assert len(serving_models.lora_requests) == 0 diff --git a/tests/entrypoints/serve/middleware/test_authentication_middleware.py b/tests/entrypoints/serve/middleware/test_authentication_middleware.py index 528261e644..a65db9a0be 100644 --- a/tests/entrypoints/serve/middleware/test_authentication_middleware.py +++ b/tests/entrypoints/serve/middleware/test_authentication_middleware.py @@ -84,7 +84,10 @@ def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]] # Register routers for this specific task (development mode already enabled). register_api_routers( - args, app, supported_tasks=(task,), model_config=MockModelConfig() + args, + app, + supported_tasks=(task,), + model_config=MockModelConfig(), # type: ignore[arg-type] ) routes = get_all_http_routes(app) diff --git a/tests/entrypoints/serve/tokenize/test_serving_tokenization.py b/tests/entrypoints/serve/tokenize/test_serving_tokenization.py index 057d7d1e8a..5e68ec382c 100644 --- a/tests/entrypoints/serve/tokenize/test_serving_tokenization.py +++ b/tests/entrypoints/serve/tokenize/test_serving_tokenization.py @@ -19,6 +19,7 @@ from vllm.entrypoints.serve.tokenize.api_router import attach_router from vllm.entrypoints.serve.tokenize.protocol import ( TokenizeChatRequest, TokenizeCompletionRequest, + TokenizeResponse, ) from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.exceptions import VLLMNotFoundError, VLLMValidationError @@ -95,7 +96,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path(): mock_engine.renderer = MagicMock() serving = _build_serving_tokenization(mock_engine) - serving.online_renderer.preprocess_chat = AsyncMock( + serving.online_renderer.preprocess_chat = AsyncMock( # type: ignore[method-assign] return_value=( [{"role": "user", "content": "Test"}], [{"prompt_token_ids": [1, 2, 3]}], @@ -109,7 +110,7 @@ async def test_tokenize_chat_skips_mm_cache_for_renderer_only_path(): response = await serving.create_tokenize(request, MagicMock(headers={})) - assert response.tokens == [1, 2, 3] + assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3] assert ( serving.online_renderer.preprocess_chat.call_args.kwargs["skip_mm_cache"] is True @@ -125,7 +126,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path(): mock_engine.renderer = MagicMock() serving = _build_serving_tokenization(mock_engine) - serving.online_renderer.preprocess_completion = AsyncMock( + serving.online_renderer.preprocess_completion = AsyncMock( # type: ignore[method-assign] return_value=[{"prompt_token_ids": [1, 2, 3]}] ) @@ -136,7 +137,7 @@ async def test_tokenize_completion_skips_mm_cache_for_renderer_only_path(): response = await serving.create_tokenize(request, MagicMock(headers={})) - assert response.tokens == [1, 2, 3] + assert isinstance(response, TokenizeResponse) and response.tokens == [1, 2, 3] assert ( serving.online_renderer.preprocess_completion.call_args.kwargs["skip_mm_cache"] is True diff --git a/tests/entrypoints/serve/tokenize/test_tokenization.py b/tests/entrypoints/serve/tokenize/test_tokenization.py index e0a70cfd67..bf51d8dee7 100644 --- a/tests/entrypoints/serve/tokenize/test_tokenization.py +++ b/tests/entrypoints/serve/tokenize/test_tokenization.py @@ -101,13 +101,13 @@ async def test_tokenize_chat( if continue_final: conversation.append({"role": "assistant", "content": "Sure,"}) - prompt = tokenizer.apply_chat_template( + prompt = tokenizer.apply_chat_template( # type: ignore[call-arg] add_generation_prompt=add_generation, continue_final_message=continue_final, conversation=conversation, tokenize=False, ) - tokens = tokenizer.encode(prompt, add_special_tokens=add_special) + tokens = tokenizer.encode(prompt, add_special_tokens=add_special) # type: ignore[arg-type] response = requests.post( server.url_for("tokenize"), @@ -169,14 +169,14 @@ async def test_tokenize_chat_with_tools( if continue_final: conversation.append({"role": "assistant", "content": "Sure,"}) - prompt = tokenizer.apply_chat_template( + prompt = tokenizer.apply_chat_template( # type: ignore[call-arg] add_generation_prompt=add_generation, continue_final_message=continue_final, conversation=conversation, tools=tools, tokenize=False, ) - tokens = tokenizer.encode(prompt, add_special_tokens=add_special) + tokens = tokenizer.encode(prompt, add_special_tokens=add_special) # type: ignore[arg-type] response = requests.post( server.url_for("tokenize"), diff --git a/tests/entrypoints/serve/utils/test_fingerprint.py b/tests/entrypoints/serve/utils/test_fingerprint.py index 46ec6255f4..c5b5e08fc2 100644 --- a/tests/entrypoints/serve/utils/test_fingerprint.py +++ b/tests/entrypoints/serve/utils/test_fingerprint.py @@ -18,7 +18,7 @@ def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"): enable_expert_parallel=ep, ) ) - c.compute_hash = lambda: digest # type: ignore[attr-defined] + c.compute_hash = lambda: digest return c @@ -59,6 +59,7 @@ def test_get_respects_set_default(): fp.set_default_fingerprint_mode("hash") hashed = fp.get_system_fingerprint(cfg) + assert hashed is not None assert hashed != full assert "tp8" not in hashed @@ -72,5 +73,7 @@ def test_get_respects_set_default(): def test_compute_hash_failure_does_not_raise(): cfg = _cfg() cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom")) - assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash") - assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash") + full_fp = fp.build_system_fingerprint(cfg, "full") + assert full_fp is not None and full_fp.endswith("-nohash") + hash_fp = fp.build_system_fingerprint(cfg, "hash") + assert hash_fp is not None and hash_fp.endswith("-nohash") diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index ab57140c2b..0493803a54 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -17,6 +17,7 @@ from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( MEDIA_CONNECTOR_REGISTRY, AsyncMultiModalItemTracker, + ChatCompletionMessageParam, ConversationMessage, _load_embeds_dict, _parse_metadata_array, @@ -215,7 +216,7 @@ def video_url(): @pytest.fixture(scope="module") def audio_url(): audio = AudioAsset("mary_had_lamb") - return encode_audio_url(*audio.audio_and_sample_rate) + return encode_audio_url(*audio.audio_and_sample_rate) # type: ignore[arg-type] def _assert_mm_data_is_image_input( @@ -769,7 +770,9 @@ async def test_text_only_chat_does_not_initialize_media_connector( ): load_connector = MagicMock() monkeypatch.setattr(MEDIA_CONNECTOR_REGISTRY, "load", load_connector) - messages = [{"role": "user", "content": "Who are you?"}] + messages: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "Who are you?"} + ] parse_chat_messages( messages, @@ -2457,7 +2460,7 @@ def test_parse_chat_messages_image_vision_chunk( kimi_k2_5_model_config, image_url, ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2493,7 +2496,7 @@ def test_parse_chat_messages_video_vision_chunk( kimi_k2_5_model_config, video_url, ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2530,7 +2533,7 @@ def test_parse_chat_messages_image_vision_chunk_with_uuid( image_url, ): image_uuid = "image_123" - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2568,7 +2571,7 @@ def test_parse_chat_messages_video_vision_chunk_with_uuid( video_url, ): video_uuid = "video_456" - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2606,7 +2609,7 @@ def test_parse_chat_messages_mixed_vision_chunk( image_url, video_url, ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2655,7 +2658,7 @@ def test_parse_chat_messages_mixed_vision_chunk_with_uuid( ): image_uuid = "image_123" video_uuid = "video_456" - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2707,7 +2710,7 @@ async def test_parse_chat_messages_mixed_vision_chunk_async( image_url, video_url, ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2757,7 +2760,7 @@ async def test_parse_chat_messages_mixed_vision_chunk_with_uuid_async( ): image_uuid = "image_123" video_uuid = "video_456" - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2808,7 +2811,7 @@ async def test_parse_chat_messages_image_vision_chunk_async( kimi_k2_5_model_config, image_url, ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2845,7 +2848,7 @@ async def test_parse_chat_messages_video_vision_chunk_async( kimi_k2_5_model_config, video_url, ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2883,7 +2886,7 @@ async def test_parse_chat_messages_image_vision_chunk_with_uuid_async( image_url, ): image_uuid = "image_123" - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2922,7 +2925,7 @@ async def test_parse_chat_messages_video_vision_chunk_with_uuid_async( video_url, ): video_uuid = "video_456" - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": [ @@ -2997,8 +3000,7 @@ async def test_resolve_items_runs_modalities_concurrently_and_preserves_order(): finally: active_fetches -= 1 - tracker = AsyncMultiModalItemTracker(MagicMock()) - tracker._model_config.is_multimodal_model = True + tracker = AsyncMultiModalItemTracker(MagicMock(is_multimodal_model=True)) tracker.__dict__["mm_processor"] = MagicMock() tracker._items_by_modality["video"] = [ lambda: _fetch("video-0", 0.02), diff --git a/tests/kernels/quantization/test_block_fp8.py b/tests/kernels/quantization/test_block_fp8.py index 038b94bd2d..7d139644e7 100644 --- a/tests/kernels/quantization/test_block_fp8.py +++ b/tests/kernels/quantization/test_block_fp8.py @@ -37,7 +37,8 @@ from vllm.utils.flashinfer import ( ) from vllm.utils.import_utils import has_deep_gemm -if current_platform.get_device_capability() < (9, 0): +capability = current_platform.get_device_capability() +if capability is None or capability < (9, 0): pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True) vllm_config = VllmConfig() @@ -258,7 +259,7 @@ def test_w8a8_block_fp8_torch_scaled_mm_matmul(): stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__( BlockWiseTorchFP8ScaledMMLinearKernel ) - stub.config = types.SimpleNamespace(out_dtype=out_dtype) + stub.config = types.SimpleNamespace(out_dtype=out_dtype) # type: ignore[assignment] out = stub.apply_block_scaled_mm( A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda() ) diff --git a/tests/kernels/quantization/test_cutlass_scaled_mm.py b/tests/kernels/quantization/test_cutlass_scaled_mm.py index 9e1d6d2cc7..26b39008aa 100644 --- a/tests/kernels/quantization/test_cutlass_scaled_mm.py +++ b/tests/kernels/quantization/test_cutlass_scaled_mm.py @@ -62,7 +62,8 @@ PER_TOKEN_GROUP_SHAPE = (1, -1) PER_OUT_CH_GROUP_SHAPE = (-1, 1) capability = current_platform.get_device_capability() -capability = capability[0] * 10 + capability[1] +# Fall back to 0 so import doesn't fail on unknown capability +capability_int = (capability[0] * 10 + capability[1]) if capability is not None else 0 def rand_int8(shape: tuple, device: str = "cuda"): @@ -639,7 +640,7 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool): def test_cutlass_support_opcheck(): - opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability,)) + opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability_int,)) @pytest.mark.parametrize("num_experts", [8, 64]) diff --git a/tests/kernels/quantization/test_cutlass_w4a8.py b/tests/kernels/quantization/test_cutlass_w4a8.py index 33bb7bccc7..1bb63ae8be 100644 --- a/tests/kernels/quantization/test_cutlass_w4a8.py +++ b/tests/kernels/quantization/test_cutlass_w4a8.py @@ -29,7 +29,8 @@ if not current_platform.is_cuda(): # `is_quant_method_supported` conflates kernels with quantization methods # an assumption which is breaking down as quantizations methods can have # have kernels and some kernels support multiple quantization methods. -IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9 +capability = current_platform.get_device_capability() +IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9 MNK_SHAPES = [ (1, 128, 128), @@ -207,7 +208,7 @@ def mm_test_helper( a=tensors.a, b_q=tensors.w_q, b_group_scales=tensors.w_g_s, - b_group_size=group_size, + b_group_size=group_size, # type: ignore[arg-type] b_channel_scales=tensors.w_ch_s, a_token_scales=tensors.w_tok_s, ) diff --git a/tests/kernels/quantization/test_cutlass_w4a8_moe.py b/tests/kernels/quantization/test_cutlass_w4a8_moe.py index 10a76e553e..b8c10d0510 100644 --- a/tests/kernels/quantization/test_cutlass_w4a8_moe.py +++ b/tests/kernels/quantization/test_cutlass_w4a8_moe.py @@ -17,8 +17,9 @@ from vllm.platforms import current_platform from vllm.scalar_type import ScalarType, scalar_types from vllm.utils.torch_utils import set_random_seed +capability = current_platform.get_device_capability() IS_SUPPORTED_BY_GPU = ( - current_platform.is_cuda() and current_platform.get_device_capability()[0] >= 9 + current_platform.is_cuda() and capability is not None and capability[0] >= 9 ) diff --git a/tests/kernels/quantization/test_fp8_quant.py b/tests/kernels/quantization/test_fp8_quant.py index 29bbb23af6..b936d13fd4 100644 --- a/tests/kernels/quantization/test_fp8_quant.py +++ b/tests/kernels/quantization/test_fp8_quant.py @@ -12,6 +12,7 @@ from tests.kernels.quant_utils import ( ) from tests.kernels.utils import opcheck from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, scaled_quantize, ) from vllm.platforms import current_platform @@ -208,7 +209,10 @@ def test_static_fp8_quant_group_2d( x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") ref_out, scale = scaled_quantize( - x, group_shape, current_platform.fp8_dtype(), compute_dtype=torch.float32 + x, + GroupShape(*group_shape), + current_platform.fp8_dtype(), + compute_dtype=torch.float32, ) ops_out, ops_scale = ops.scaled_fp8_quant(x, scale=scale, group_shape=group_shape) @@ -236,7 +240,7 @@ def test_static_fp8_quant_1d_scale( x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda") ref_out, scale_2d = scaled_quantize( - x, group_shape, FP8_DTYPE, compute_dtype=torch.float32 + x, GroupShape(*group_shape), FP8_DTYPE, compute_dtype=torch.float32 ) # Flatten scale to 1D for testing 1D scale path diff --git a/tests/kernels/quantization/test_machete_mm.py b/tests/kernels/quantization/test_machete_mm.py index 62d0ba4f14..0e69a09a01 100644 --- a/tests/kernels/quantization/test_machete_mm.py +++ b/tests/kernels/quantization/test_machete_mm.py @@ -38,7 +38,8 @@ CUDA_DEVICES = [ # `is_quant_method_supported` conflates kernels with quantization methods # an assumption which is breaking down as quantizations methods can have # have kernels and some kernels support multiple quantization methods. -IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9 +capability = current_platform.get_device_capability() +IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9 MNK_SHAPES = [ (1, 128, 128), @@ -305,7 +306,7 @@ def test_machete_all_schedules(shape, types: TypeConfig): if types.group_scale_type is None: group_sizes = [None] else: - group_sizes = query_machete_supported_group_sizes(types.act_type) + group_sizes = list(query_machete_supported_group_sizes(types.act_type)) for group_size in group_sizes: if not group_size_valid(shape, group_size): @@ -334,7 +335,7 @@ def test_machete_heuristic(shape, types: TypeConfig): if types.group_scale_type is None: group_sizes = [None] else: - group_sizes = query_machete_supported_group_sizes(types.act_type) + group_sizes = list(query_machete_supported_group_sizes(types.act_type)) for group_size in group_sizes: if not group_size_valid(shape, group_size): diff --git a/tests/kernels/quantization/test_marlin_gemm.py b/tests/kernels/quantization/test_marlin_gemm.py index 29910390bb..1b674b3c98 100644 --- a/tests/kernels/quantization/test_marlin_gemm.py +++ b/tests/kernels/quantization/test_marlin_gemm.py @@ -342,7 +342,7 @@ def marlin_generate_valid_test_cases(): [quant_test_config["b_type"]], quant_test_config.get("c_type", f16_types), quant_test_config["group_blocks"], - ) + ) # type: ignore[call-overload] for sub_case in inner_combinations: if ( diff --git a/tests/kernels/quantization/test_marlin_tile_padding.py b/tests/kernels/quantization/test_marlin_tile_padding.py index 8b55910a78..ef5911decd 100644 --- a/tests/kernels/quantization/test_marlin_tile_padding.py +++ b/tests/kernels/quantization/test_marlin_tile_padding.py @@ -11,6 +11,7 @@ import pytest import torch from vllm import _custom_ops as ops +from vllm.model_executor.layers.linear import LinearBase from vllm.model_executor.layers.quantization.utils.marlin_utils import ( GPTQ_MARLIN_TILE, apply_gptq_marlin_linear, @@ -524,13 +525,15 @@ def test_check_marlin_supports_layer_allow_tile_padding(): ) # Tile-misaligned but group-aligned: rejected strictly, allowed w/ padding - layer = _FakeLinear(4640, 512, input_size=2048) + layer: LinearBase = _FakeLinear( # type: ignore[assignment] + 4640, 512, input_size=2048 + ) assert not check_marlin_supports_layer(layer, 128) assert check_marlin_supports_layer(layer, 128, allow_tile_padding=True) assert check_marlin_supports_layer(layer, -1, allow_tile_padding=True) # A group straddling the TP shard cannot be fixed by padding - layer = _FakeLinear(4608, 4672, input_size=18688) + layer = _FakeLinear(4608, 4672, input_size=18688) # type: ignore[assignment] assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True) diff --git a/tests/kernels/quantization/test_nvfp4_emulation.py b/tests/kernels/quantization/test_nvfp4_emulation.py index 3a136484ac..0e005760b0 100644 --- a/tests/kernels/quantization/test_nvfp4_emulation.py +++ b/tests/kernels/quantization/test_nvfp4_emulation.py @@ -206,7 +206,7 @@ def test_nvfp4_emulation_support_check_rejects_bias_and_lora( in_dtype=torch.bfloat16, device="cuda", routing_method=RoutingMethodType.TopK, - **config_kwargs, + **config_kwargs, # type: ignore[arg-type] ) supported, reason = Nvfp4QuantizationEmulationTritonExperts.is_supported_config( diff --git a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py index d682ed05b0..f8380dcecf 100644 --- a/tests/kernels/quantization/test_scaled_mm_kernel_selection.py +++ b/tests/kernels/quantization/test_scaled_mm_kernel_selection.py @@ -35,7 +35,7 @@ pytestmark = pytest.mark.cpu_test def test_auto_linear_backend_is_unchanged_without_overrides(): - kernels = [object] + kernels: list[type] = [object] quantizations = ( "fp8_block_w8a8", "fp8_w8a8", diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 42f134e6de..89502fbf9d 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -17,6 +17,7 @@ import vllm.config as vllm_config_module from vllm.config import VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe import utils as fused_moe_utils from vllm.model_executor.layers.fused_moe.layer import determine_expert_counts +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig from vllm.model_executor.layers.quantization.utils.config_utils import ( @@ -140,10 +141,15 @@ def get_deepseek_v4_quark_config(exclude: list[str]) -> dict[str, Any]: return quantization_config +def _stub_quant_config() -> QuantizationConfig: + return cast(QuantizationConfig, object()) + + def get_fse_test_model_config( model_type: str, quantization_config: dict[str, Any], ) -> tuple[object, type[nn.Module]]: + config: object if model_type == "minimax_m3": config = MiniMaxM3TextConfig( hidden_size=128, @@ -301,7 +307,7 @@ def test_resolve_layer_fused_shared_expert_skips_compatibility_when_disabled( ) assert not fused_moe_utils.resolve_layer_fused_shared_expert( - object(), "model.layers.0.mlp" + _stub_quant_config(), "model.layers.0.mlp" ) @@ -316,7 +322,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter( assert ( fused_moe_utils.resolve_layer_fused_shared_expert( - object(), "model.layers.0.mlp" + _stub_quant_config(), "model.layers.0.mlp" ) is False ) @@ -325,7 +331,7 @@ def test_resolve_layer_fused_shared_expert_normalizes_unavailable_aiter( def test_resolve_layer_fused_shared_expert_passes_module_prefixes( monkeypatch: pytest.MonkeyPatch, ) -> None: - quant_config = object() + quant_config = _stub_quant_config() monkeypatch.setattr( fused_moe_utils.rocm_aiter_ops, "is_fusion_moe_shared_experts_enabled", @@ -374,7 +380,7 @@ def test_resolve_layer_fused_shared_expert_rejects_incompatible_quantization( ) assert not fused_moe_utils.resolve_layer_fused_shared_expert( - object(), "model.layers.0.mlp" + _stub_quant_config(), "model.layers.0.mlp" ) assert "shared experts are excluded" in caplog.text @@ -407,7 +413,7 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix( ) compatible, reason = is_shared_expert_quant_fse_compatible( - DeepseekV4Config(), + DeepseekV4Config(), # type: ignore[arg-type] "model.layers.2.ffn.experts", "model.layers.2.ffn.shared_experts", ) @@ -818,7 +824,7 @@ def test_models_fse_init( model_type, quantization_config ) vllm_config = VllmConfig() - vllm_config.model_config = SimpleNamespace( + vllm_config.model_config = SimpleNamespace( # type: ignore[assignment] hf_config=config, hf_text_config=config, dtype=torch.bfloat16, @@ -887,7 +893,7 @@ def test_models_fse_init( DeepSeekV4MTP, ) - vllm_config.speculative_config = SimpleNamespace( + vllm_config.speculative_config = SimpleNamespace( # type: ignore[assignment] draft_model_config=SimpleNamespace(hf_config=config), method="mtp", parallel_drafting=False, @@ -1214,7 +1220,7 @@ def test_quark_packed_layer_config_must_match_global_config() -> None: def test_non_quark_shared_expert_fse_is_incompatible() -> None: compatible, reason = is_shared_expert_quant_fse_compatible( - object(), + _stub_quant_config(), "model.layers.0.mlp.experts", "model.layers.0.mlp.shared_experts", ) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 77c52fb291..70243e5557 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -46,7 +46,7 @@ def test_sparse_attention_refreshes_batch_state_inside_eager_segment( with pytest.raises(BatchStateRefreshed): DeepseekV32Attention._sparse_indexer_and_attn( - layer, + layer, # type: ignore[arg-type] torch.empty(1, dtype=torch.long), torch.empty(1, 1), torch.empty(1, 1, 1), @@ -80,7 +80,7 @@ def make_indexer_metadata( num_decode_tokens=num_decode_tokens, num_prefills=num_prefills, num_prefill_tokens=num_prefill_tokens, - prefill=SimpleNamespace(chunks=[]) if num_prefills else None, + prefill=SimpleNamespace(chunks=[]) if num_prefills else None, # type: ignore[arg-type] ) @@ -129,7 +129,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision( slot_mapping=slot_mapping, ) if indexer_metadata.num_decodes: - indexer_metadata.decode = object() + indexer_metadata.decode = object() # type: ignore[assignment] mla_metadata = make_mla_metadata( use_dense_mha=batch_kind != "force_mqa", num_decode_tokens=mla_num_decode_tokens, @@ -307,7 +307,7 @@ def test_deepseek_v32_dispatches_selected_mha( output = torch.empty(2, 2) DeepseekV32Attention._sparse_indexer_and_attn( - layer, + layer, # type: ignore[arg-type] torch.arange(2), torch.empty(2, 2), q_nope, diff --git a/tests/model_executor/layers/test_pooler_activations.py b/tests/model_executor/layers/test_pooler_activations.py index 697207a68b..381d267583 100644 --- a/tests/model_executor/layers/test_pooler_activations.py +++ b/tests/model_executor/layers/test_pooler_activations.py @@ -224,11 +224,11 @@ class TestResolveClassifierActFn: model_config = SimpleNamespace( hf_config=SimpleNamespace(num_labels=3, problem_type="") ) - result = resolve_classifier_act_fn(model_config, act_fn=None) + result = resolve_classifier_act_fn(model_config, act_fn=None) # type: ignore[arg-type] assert isinstance(result, PoolerClassify) assert result.num_labels == 3 def test_passes_through_provided_act_fn(self): custom = PoolerIdentity() - result = resolve_classifier_act_fn(None, act_fn=custom) + result = resolve_classifier_act_fn(None, act_fn=custom) # type: ignore[arg-type] assert result is custom diff --git a/tests/model_executor/layers/test_pooler_heads.py b/tests/model_executor/layers/test_pooler_heads.py index 99097636f9..a80c5b8344 100644 --- a/tests/model_executor/layers/test_pooler_heads.py +++ b/tests/model_executor/layers/test_pooler_heads.py @@ -15,6 +15,7 @@ from vllm.model_executor.layers.pooler.tokwise.heads import ( TokenEmbeddingPoolerHead, ) from vllm.pooling_params import PoolingParams +from vllm.tasks import PoolingTask from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates _HIDDEN = 16 @@ -24,7 +25,7 @@ _BATCH = 3 def _make_params( n: int, *, - task: str = "embed", + task: PoolingTask = "embed", dimensions: int | None = None, use_activation: bool | None = None, ) -> list[PoolingParams]: @@ -320,6 +321,7 @@ class TestTokenEmbeddingPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_embed") out = head.forward_chunk(x, param) + assert out is not None assert out.dtype == torch.float16 def test_projector(self): @@ -328,6 +330,7 @@ class TestTokenEmbeddingPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_embed") out = head.forward_chunk(x, param) + assert out is not None assert out.shape == (5, 8) assert torch.allclose(out, proj(x)) @@ -336,6 +339,7 @@ class TestTokenEmbeddingPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_embed", dimensions=4) out = head.forward_chunk(x, param) + assert out is not None assert out.shape == (5, 4) assert torch.equal(out, x[..., :4]) @@ -360,6 +364,7 @@ class TestTokenEmbeddingPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_embed", dimensions=4) out = head.forward_chunk(x, param) + assert out is not None assert out.shape == (5, 4) assert torch.equal(out, proj(x)[..., :4]) @@ -368,6 +373,7 @@ class TestTokenEmbeddingPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_embed", dimensions=4, use_activation=True) out = head.forward_chunk(x, param) + assert out is not None assert out.shape == (5, 4) norms = torch.linalg.norm(out, dim=-1) assert torch.allclose(norms, torch.ones(5), atol=1e-5) @@ -416,6 +422,7 @@ class TestTokenClassifierPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_classify") out = head.forward_chunk(x, param) + assert out is not None assert out.dtype == torch.float16 def test_classifier(self): @@ -424,6 +431,7 @@ class TestTokenClassifierPoolerHead: x = torch.randn(5, _HIDDEN) param = PoolingParams(task="token_classify") out = head.forward_chunk(x, param) + assert out is not None assert out.shape == (5, 3) assert torch.allclose(out, clf(x)) diff --git a/tests/model_executor/layers/test_pooler_methods.py b/tests/model_executor/layers/test_pooler_methods.py index 2abf3d7280..bfe2cf0aed 100644 --- a/tests/model_executor/layers/test_pooler_methods.py +++ b/tests/model_executor/layers/test_pooler_methods.py @@ -26,6 +26,7 @@ from vllm.model_executor.layers.pooler.tokwise.methods import ( get_tok_pooling_method, ) from vllm.pooling_params import PoolingParams +from vllm.tasks import PoolingTask from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates _CPU = torch.device("cpu") @@ -66,7 +67,7 @@ def _make_pooling_cursor( def _make_metadata( prompt_lens: list[int], *, - tasks: list[str] | None = None, + tasks: list[PoolingTask] | None = None, token_ids: list[list[int]] | None = None, pooling_params: list[PoolingParams] | None = None, num_scheduled_tokens: list[int] | None = None, @@ -319,7 +320,7 @@ def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type): ), ) classifier = torch.nn.Linear(4, 3) - with set_current_vllm_config(config): + with set_current_vllm_config(config): # type: ignore[arg-type] pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier) token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]] diff --git a/tests/plugins_tests/test_bge_m3_sparse_io_processor_plugins.py b/tests/plugins_tests/test_bge_m3_sparse_io_processor_plugins.py index 5178c83d3d..8f3fc6b389 100644 --- a/tests/plugins_tests/test_bge_m3_sparse_io_processor_plugins.py +++ b/tests/plugins_tests/test_bge_m3_sparse_io_processor_plugins.py @@ -114,7 +114,8 @@ async def test_bge_m3_sparse_plugin_online( response = ret.json() # Verify the request response is in the correct format - assert (parsed_response := IOProcessorResponse(**response).data) + parsed_response: object = IOProcessorResponse(**response).data + assert parsed_response # Verify the output is formatted as expected for this plugin assert _get_attr_or_val(parsed_response, "data") diff --git a/tests/plugins_tests/test_colbert_query_io_processor_plugins.py b/tests/plugins_tests/test_colbert_query_io_processor_plugins.py index 930c493fdd..749adcf63f 100644 --- a/tests/plugins_tests/test_colbert_query_io_processor_plugins.py +++ b/tests/plugins_tests/test_colbert_query_io_processor_plugins.py @@ -79,7 +79,7 @@ def _post_pooling(server: RemoteOpenAIServer, data: dict): ret = requests.post(server.url_for("pooling"), json=request_payload) ret.raise_for_status() response = ret.json() - parsed_response = IOProcessorResponse(**response).data + parsed_response: object = IOProcessorResponse(**response).data assert parsed_response return parsed_response diff --git a/tests/plugins_tests/test_endpoint_plugins.py b/tests/plugins_tests/test_endpoint_plugins.py index 2d92153900..51d96643d9 100644 --- a/tests/plugins_tests/test_endpoint_plugins.py +++ b/tests/plugins_tests/test_endpoint_plugins.py @@ -172,7 +172,7 @@ async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatc monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin") state = State() - await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args()) + await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args()) # type: ignore[arg-type] assert not hasattr(state, "dummy_engine_client") @@ -223,7 +223,7 @@ async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch): ) fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"]) - await init_endpoint_plugins_state(fake_engine_client, app.state, args) + await init_endpoint_plugins_state(fake_engine_client, app.state, args) # type: ignore[arg-type] assert app.state.dummy_engine_client is fake_engine_client diff --git a/tests/plugins_tests/test_oot_registration_offline.py b/tests/plugins_tests/test_oot_registration_offline.py index ac5361c66f..4fac55f82d 100644 --- a/tests/plugins_tests/test_oot_registration_offline.py +++ b/tests/plugins_tests/test_oot_registration_offline.py @@ -92,7 +92,7 @@ def test_oot_registration_multimodal( ) first_token = llm.get_tokenizer().decode(0) - outputs = llm.generate(prompts, sampling_params) + outputs = llm.generate(prompts, sampling_params) # type: ignore[arg-type] for output in outputs: generated_text = output.outputs[0].text diff --git a/tests/plugins_tests/test_stats_logger_plugins.py b/tests/plugins_tests/test_stats_logger_plugins.py index eb03b1fde4..518015f76a 100644 --- a/tests/plugins_tests/test_stats_logger_plugins.py +++ b/tests/plugins_tests/test_stats_logger_plugins.py @@ -23,7 +23,8 @@ def test_stat_logger_plugin_is_discovered(monkeypatch: pytest.MonkeyPatch): # instantiate and confirm the right type vllm_config = VllmConfig() instance = factories[0](vllm_config) - assert isinstance(instance, DummyStatLogger) + # The `is` check above narrows DummyStatLogger to a Callable + assert isinstance(instance, DummyStatLogger) # type: ignore[arg-type] def test_no_plugins_loaded_if_env_empty(monkeypatch: pytest.MonkeyPatch): @@ -66,10 +67,11 @@ async def test_stat_logger_plugin_integration_with_engine( engine = AsyncLLM.from_engine_args(engine_args=engine_args) + assert engine.logger_manager is not None assert len(engine.logger_manager.stat_loggers) == 2 - assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 + assert len(engine.logger_manager.stat_loggers[0].per_engine_stat_loggers) == 1 # type: ignore[attr-defined] assert isinstance( - engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], + engine.logger_manager.stat_loggers[0].per_engine_stat_loggers[0], # type: ignore[attr-defined] DummyStatLogger, ) diff --git a/tests/plugins_tests/test_terratorch_io_processor_plugins.py b/tests/plugins_tests/test_terratorch_io_processor_plugins.py index b4c84b30d2..03e808e4cd 100644 --- a/tests/plugins_tests/test_terratorch_io_processor_plugins.py +++ b/tests/plugins_tests/test_terratorch_io_processor_plugins.py @@ -95,7 +95,8 @@ async def test_prithvi_mae_plugin_online( response = ret.json() # verify the request response is in the correct format - assert (parsed_response := IOProcessorResponse(**response)) + parsed_response: IOProcessorResponse = IOProcessorResponse(**response) + assert parsed_response # verify the output is formatted as expected for this plugin plugin_data = parsed_response.data diff --git a/tests/samplers/test_beam_search.py b/tests/samplers/test_beam_search.py index 8bcf4c48ab..daf6f4386c 100644 --- a/tests/samplers/test_beam_search.py +++ b/tests/samplers/test_beam_search.py @@ -16,6 +16,7 @@ from transformers import AutoModelForSeq2SeqLM from vllm import CompletionOutput, RequestOutput from vllm.assets.audio import AudioAsset from vllm.entrypoints.llm import LLM +from vllm.inputs import TokensInput from vllm.logprobs import Logprob, SampleLogprobs from vllm.platforms import current_platform from vllm.sampling_params import BeamSearchParams, StructuredOutputsParams @@ -107,11 +108,12 @@ def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts( llm.renderer = Mock(get_tokenizer=Mock(return_value=tokenizer)) monkeypatch.setattr(llm, "_preprocess_cmpl", lambda prompts: prompts) monkeypatch.setattr(llm, "_render_and_run_requests", run_requests) - prompts = [ + # Engine inputs, since _preprocess_cmpl is patched to the identity + prompts: list[TokensInput] = [ {"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2] ] params = BeamSearchParams(beam_width=2, max_tokens=3) - outputs = llm.beam_search(prompts, params) + outputs = llm.beam_search(prompts, params) # type: ignore[arg-type] aborted, normal = outputs expected_tokens = ( @@ -324,7 +326,7 @@ def test_beam_search_structured_output( llm = LLM( model=model, - dtype=dtype, + dtype=dtype, # type: ignore[arg-type] max_model_len=512, structured_outputs_config=dict( backend="xgrammar", @@ -343,7 +345,7 @@ def test_beam_search_structured_output( "Generate a JSON object for a person with name and age:", ] - outputs = llm.beam_search(prompts, params) + outputs = llm.beam_search(prompts, params) # type: ignore[arg-type] assert len(outputs) == len(prompts) for output in outputs: diff --git a/tests/samplers/test_beam_search_online.py b/tests/samplers/test_beam_search_online.py index ccb40cc659..4c48c6189d 100644 --- a/tests/samplers/test_beam_search_online.py +++ b/tests/samplers/test_beam_search_online.py @@ -5,6 +5,7 @@ import pytest from vllm import CompletionOutput, RequestOutput from vllm.entrypoints.generate.beam_search.online import BeamSearchOnlineMixin +from vllm.inputs import TokensInput from vllm.logprobs import Logprob, SampleLogprobs from vllm.sampling_params import BeamSearchParams @@ -56,13 +57,13 @@ class _EngineClient: class _Serving(BeamSearchOnlineMixin): - renderer = _Renderer() - engine_client = _EngineClient() + renderer = _Renderer() # type: ignore[assignment] + engine_client = _EngineClient() # type: ignore[assignment] @pytest.mark.asyncio async def test_beam_search_handles_extra_logprob_candidates() -> None: - prompt = { + prompt: TokensInput = { "type": "token", "prompt": "prompt", "prompt_token_ids": [1], @@ -90,7 +91,7 @@ async def test_beam_search_handles_extra_logprob_candidates() -> None: async def test_beam_search_respects_skip_special_tokens( skip_special_tokens: bool, expected_text: str ) -> None: - prompt = { + prompt: TokensInput = { "type": "token", "prompt": "prompt", "prompt_token_ids": [1], @@ -140,7 +141,11 @@ async def test_beam_search_abort_returns_partial_outputs( serving = _Serving() monkeypatch.setattr(serving.engine_client, "generate", generate) - prompt = {"type": "token", "prompt": "prompt", "prompt_token_ids": [prompt_token]} + prompt: TokensInput = { + "type": "token", + "prompt": "prompt", + "prompt_token_ids": [prompt_token], + } outputs = [ output async for output in serving.beam_search( diff --git a/tests/samplers/test_no_bad_words.py b/tests/samplers/test_no_bad_words.py index 74047d2f03..e060f41e89 100644 --- a/tests/samplers/test_no_bad_words.py +++ b/tests/samplers/test_no_bad_words.py @@ -8,11 +8,12 @@ Run `pytest tests/samplers/test_no_bad_words.py`. from transformers import AutoTokenizer -from vllm import LLM, SamplingParams +from tests.conftest import VllmRunner +from vllm import SamplingParams def _generate( - llm: LLM, + llm: VllmRunner, prompt: str, num_prompt_tokens: int, temperature: float = 0, @@ -56,7 +57,9 @@ class TestOneTokenBadWord: output_token_ids = self._generate(llm, bad_words=[self.TARGET_TOKEN]) assert self.target_token_id not in output_token_ids - def _generate(self, llm: LLM, bad_words: list[str] | None = None) -> list[int]: + def _generate( + self, llm: VllmRunner, bad_words: list[str] | None = None + ) -> list[int]: return _generate( llm=llm, prompt=self.PROMPT, @@ -151,7 +154,9 @@ class TestTwoTokenBadWord: self.neighbour_token_id2 in output_token_ids ) - def _generate(self, llm: LLM, bad_words: list[str] | None = None) -> list[int]: + def _generate( + self, llm: VllmRunner, bad_words: list[str] | None = None + ) -> list[int]: return _generate( llm=llm, prompt=self.PROMPT, diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py index 9db049090f..d0117e4711 100644 --- a/tests/tokenizers_/test_detokenize.py +++ b/tests/tokenizers_/test_detokenize.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Generator -from typing import Any +from typing import Any, cast import pytest from transformers import AutoTokenizer, PythonBackend, TokenizersBackend @@ -10,6 +10,7 @@ from transformers import AutoTokenizer, PythonBackend, TokenizersBackend from vllm.sampling_params import SamplingParams from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens from vllm.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers.protocol import TokenizerLike from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.detokenizer import ( FastIncrementalDetokenizer, @@ -281,9 +282,26 @@ class _MockTokenizer: return "".join(self._decoded[tid] for tid in ids) +def _mock_tokenizer( + raw_tokens: dict[int, str], + decoded_tokens: dict[int, str], + pre_tokenizer_type: str = "Metaspace", + replacement: str | None = "▁", +) -> TokenizerLike: + return cast( + TokenizerLike, + _MockTokenizer( + raw_tokens=raw_tokens, + decoded_tokens=decoded_tokens, + pre_tokenizer_type=pre_tokenizer_type, + replacement=replacement, + ), + ) + + def test_sentencepiece_leading_space_preserved(): """▁true and true must produce distinct strings.""" - tok = _MockTokenizer( + tok = _mock_tokenizer( raw_tokens={0: "▁true", 1: "true", 2: "▁false", 3: "false"}, decoded_tokens={0: "true", 1: "true", 2: "false", 3: "false"}, ) @@ -297,7 +315,7 @@ def test_sentencepiece_leading_space_preserved(): def test_whitespace_run_tokens_stay_distinct(): """▁, ▁▁, ▁▁▁ must produce different-length space strings.""" - tok = _MockTokenizer( + tok = _mock_tokenizer( raw_tokens={0: "▁", 1: "▁▁", 2: "▁▁▁"}, decoded_tokens={0: "", 1: " ", 2: " "}, ) @@ -307,7 +325,7 @@ def test_whitespace_run_tokens_stay_distinct(): def test_bpe_leading_space_already_preserved(): """GPT-2 BPE: Ġtrue already decodes to ' true', no fix needed.""" - tok = _MockTokenizer( + tok = _mock_tokenizer( raw_tokens={0: "Ġtrue", 1: "true"}, decoded_tokens={0: " true", 1: "true"}, pre_tokenizer_type="ByteLevel", @@ -319,7 +337,7 @@ def test_bpe_leading_space_already_preserved(): def test_logprobs_count_stable_across_k(): """logprobs=4 and logprobs=10 must return 4 and 10 entries.""" - tok = _MockTokenizer( + tok = _mock_tokenizer( raw_tokens={ 0: "▁true", 1: "a", diff --git a/tests/tokenizers_/test_hf.py b/tests/tokenizers_/test_hf.py index 61c81302f0..693cda4ab0 100644 --- a/tests/tokenizers_/test_hf.py +++ b/tests/tokenizers_/test_hf.py @@ -61,7 +61,7 @@ def test_thread_pool_tokenizer_pickle(model_id: str): unpickled_tokenizer = pickle.loads(pickle.dumps(pooled_tokenizer)) assert unpickled_tokenizer is not None assert isinstance(unpickled_tokenizer, ThreadSafeHFTokenizerMixin) - assert unpickled_tokenizer.encode("prompt") == reference_tokenizer.encode("prompt") + assert unpickled_tokenizer.encode("prompt") == reference_tokenizer.encode("prompt") # type: ignore[attr-defined] # Idempotence: wrapping an already-pooled tokenizer returns it unchanged. - assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer + assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer # type: ignore[type-var] diff --git a/tests/tokenizers_/test_mistral.py b/tests/tokenizers_/test_mistral.py index 5dfee37012..5229eaaf9c 100644 --- a/tests/tokenizers_/test_mistral.py +++ b/tests/tokenizers_/test_mistral.py @@ -10,6 +10,7 @@ from mistral_common.exceptions import InvalidMessageStructureException from mistral_common.guidance.grammar_factory import GrammarFactory from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy, SpecialTokens +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.exceptions import VLLMValidationError from vllm.tokenizers.mistral import ( @@ -21,7 +22,9 @@ from vllm.tokenizers.mistral import ( def test_validate_apply_chat_template_args(): # add_generation_prompt with assistant last message → error - messages = [{"role": "assistant", "content": "Hello"}] + messages: list[ChatCompletionMessageParam] = [ + {"role": "assistant", "content": "Hello"} + ] with pytest.raises(ValueError): _validate_apply_chat_template_args(messages, add_generation_prompt=True) @@ -32,7 +35,9 @@ def test_validate_apply_chat_template_args(): # both add_generation_prompt and continue_final_message → error with pytest.raises(ValueError): _validate_apply_chat_template_args( - messages, add_generation_prompt=True, continue_final_message=True + messages, + add_generation_prompt=True, + continue_final_message=True, ) # continue_final_message with assistant last message → ok @@ -835,7 +840,9 @@ class TestMistralTokenizer: ) def test_apply_chat_template_error(self, mistral_tokenizer: MistralTokenizer): - messages = [{"role": "user", "content": "Hello world !"}] + messages: list[ChatCompletionMessageParam] = [ + {"role": "user", "content": "Hello world !"} + ] with pytest.raises(ValueError): mistral_tokenizer.apply_chat_template( @@ -968,7 +975,7 @@ class TestMistralTokenizer: mistral_tokenizer: MistralTokenizer, ): assert mistral_tokenizer.batch_decode( - [[]], + [[]], # type: ignore[arg-type] ) == [""] def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer): @@ -2167,13 +2174,15 @@ class TestMistralTokenizer: def test_apply_chat_template_tool_optional_fields( self, mistral_tokenizer: MistralTokenizer, - messages: list[dict[str, Any]], + messages: list[ChatCompletionMessageParam], tools: list[dict[str, Any]], tekken_expected_substrings: list[str], spm_expected_substrings: list[str], ) -> None: output = mistral_tokenizer.apply_chat_template( - messages, tools=tools, add_generation_prompt=True + messages, + tools=tools, + add_generation_prompt=True, ) decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP) @@ -2188,7 +2197,7 @@ class TestMistralTokenizer: def test_apply_chat_template_tools_not_mutated( self, mistral_tokenizer: MistralTokenizer ) -> None: - messages: list[dict[str, Any]] = [ + messages: list[ChatCompletionMessageParam] = [ {"role": "user", "content": "Hello"}, ] tools: list[dict[str, Any]] = [ @@ -2209,7 +2218,9 @@ class TestMistralTokenizer: original_tools = copy.deepcopy(tools) mistral_tokenizer.apply_chat_template( - messages, tools=tools, add_generation_prompt=True + messages, + tools=tools, + add_generation_prompt=True, ) assert tools == original_tools @@ -2235,7 +2246,8 @@ class TestMistralTokenizer: ] output = mistral_tokenizer.apply_chat_template( - messages, add_generation_prompt=True + messages, # type: ignore[arg-type] + add_generation_prompt=True, ) decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP) @@ -2310,6 +2322,7 @@ def v15_mistral_tokenizer() -> MistralTokenizer: return MistralTokenizer.from_pretrained("mistralai/Mistral-Small-4-119B-2603") except Exception: pytest.skip("v15 tokenizer unavailable") + raise AssertionError("unreachable") @pytest.fixture(scope="module") @@ -2319,6 +2332,7 @@ def v13_mistral_tokenizer() -> MistralTokenizer: return MistralTokenizer.from_pretrained("mistralai/Magistral-Small-2509") except Exception: pytest.skip("v13 tokenizer unavailable") + raise AssertionError("unreachable") def test_v15_apply_chat_template_passes_reasoning_effort_high( @@ -2338,7 +2352,7 @@ def test_v15_apply_chat_template_passes_reasoning_effort_high( ) v15_mistral_tokenizer.apply_chat_template( - messages=_PASSTHROUGH_MESSAGES, + messages=_PASSTHROUGH_MESSAGES, # type: ignore[arg-type] reasoning_effort="high", ) @@ -2361,7 +2375,7 @@ def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default( fake_apply_chat_template, ) - v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES) + v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES) # type: ignore[arg-type] assert "reasoning_effort" in captured_kwargs[-1] assert captured_kwargs[-1]["reasoning_effort"] is None @@ -2384,7 +2398,7 @@ def test_pre_v15_apply_chat_template_omits_reasoning_effort( ) v13_mistral_tokenizer.apply_chat_template( - messages=_PASSTHROUGH_MESSAGES, + messages=_PASSTHROUGH_MESSAGES, # type: ignore[arg-type] reasoning_effort="high", ) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index a4ee4a755e..e1da601b64 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -205,7 +205,7 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path): side_effect=lambda tokenizer: tokenizer, ), ): - tokenizer = cached_tokenizer_from_config(model_config) + tokenizer = cached_tokenizer_from_config(model_config) # type: ignore[arg-type] assert tokenizer.is_fast is True finally: diff --git a/tests/tool_use/test_gemma4_responses_adjust_request.py b/tests/tool_use/test_gemma4_responses_adjust_request.py index fabfa74766..403c707924 100644 --- a/tests/tool_use/test_gemma4_responses_adjust_request.py +++ b/tests/tool_use/test_gemma4_responses_adjust_request.py @@ -148,7 +148,7 @@ def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None: description string that the previous two-step construction injected. """ parser = ToolParser.__new__(ToolParser) - parser.model_tokenizer = None + parser.model_tokenizer = None # type: ignore[assignment] request = _build_responses_request(tool_choice="required") ToolParser.adjust_request(parser, request) diff --git a/tests/tool_use/test_kimi_k3_tool_parser.py b/tests/tool_use/test_kimi_k3_tool_parser.py index b300b848e3..ac7dcfb8ac 100644 --- a/tests/tool_use/test_kimi_k3_tool_parser.py +++ b/tests/tool_use/test_kimi_k3_tool_parser.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +from typing import cast import pytest @@ -15,6 +16,7 @@ from vllm.exceptions import VLLMValidationError from vllm.parser.kimi_k3 import KimiK3Parser from vllm.parser.parser_manager import ParserManager from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser +from vllm.tokenizers.protocol import TokenizerLike from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser OPEN = "<|open|>" @@ -37,6 +39,10 @@ class DummyTokenizer: return [ord(ch) for ch in text] +def _dummy_tokenizer() -> TokenizerLike: + return cast(TokenizerLike, DummyTokenizer()) + + class KimiK3DelegatingParser(KimiK3Parser): reasoning_parser_cls = KimiK3ReasoningParser tool_parser_cls = KimiK3ToolParser @@ -124,7 +130,7 @@ def _tools(*calls: str) -> str: def test_extract_tool_calls_with_response_and_typed_arguments(): - parser = KimiK3ToolParser(DummyTokenizer()) + parser = KimiK3ToolParser(_dummy_tokenizer()) output = _response("answer") + _tools( _call( @@ -150,7 +156,7 @@ def test_extract_tool_calls_with_response_and_typed_arguments(): def test_delegating_parser_preserves_tool_calls_after_reasoning(): - parser = KimiK3DelegatingParser(DummyTokenizer()) + parser = KimiK3DelegatingParser(_dummy_tokenizer()) output = ( f"{THINK_OPEN}step{THINK_CLOSE}" + _response("answer") @@ -172,7 +178,7 @@ def test_delegating_parser_preserves_tool_calls_after_reasoning(): def test_delegating_parser_required_tool_choice_uses_xtml_parser(): - parser = KimiK3DelegatingParser(DummyTokenizer()) + parser = KimiK3DelegatingParser(_dummy_tokenizer()) request = _request().model_copy(update={"tool_choice": "required"}) output = ( f"{THINK_OPEN}step{THINK_CLOSE}" @@ -195,7 +201,7 @@ def test_delegating_parser_required_tool_choice_uses_xtml_parser(): def test_delegating_parser_named_tool_choice_uses_xtml_parser(): - parser = KimiK3DelegatingParser(DummyTokenizer()) + parser = KimiK3DelegatingParser(_dummy_tokenizer()) output = ( f"{THINK_OPEN}step{THINK_CLOSE}" + _response("") @@ -218,7 +224,7 @@ def test_delegating_parser_named_tool_choice_uses_xtml_parser(): def test_delegating_parser_auto_no_call_strips_consumed_response_prefix(): parser = KimiK3DelegatingParser( - DummyTokenizer(), chat_template_kwargs={"thinking": False} + _dummy_tokenizer(), chat_template_kwargs={"thinking": False} ) request = _request().model_copy( update={"chat_template_kwargs": {"thinking": False}} @@ -237,7 +243,7 @@ def test_delegating_parser_auto_no_call_strips_consumed_response_prefix(): def test_delegating_parser_required_call_strips_consumed_response_prefix(): parser = KimiK3DelegatingParser( - DummyTokenizer(), chat_template_kwargs={"thinking": False} + _dummy_tokenizer(), chat_template_kwargs={"thinking": False} ) request = _request().model_copy( update={ @@ -263,7 +269,7 @@ def test_delegating_parser_required_call_strips_consumed_response_prefix(): def test_delegating_parser_truncated_tools_do_not_leak_xtml(): parser = KimiK3DelegatingParser( - DummyTokenizer(), chat_template_kwargs={"thinking": False} + _dummy_tokenizer(), chat_template_kwargs={"thinking": False} ) request = _request().model_copy( update={ @@ -284,7 +290,7 @@ def test_delegating_parser_truncated_tools_do_not_leak_xtml(): def test_extract_tool_calls_unescapes_attributes(): - parser = KimiK3ToolParser(DummyTokenizer()) + parser = KimiK3ToolParser(_dummy_tokenizer()) output = _tools(_call("a&b"c", 1, _arg("k&q", "string", "v"))) extracted = parser.extract_tool_calls(output, _request()) @@ -295,7 +301,7 @@ def test_extract_tool_calls_unescapes_attributes(): def test_extract_tool_calls_allows_less_than_in_attributes(): - parser = KimiK3ToolParser(DummyTokenizer()) + parser = KimiK3ToolParser(_dummy_tokenizer()) output = _tools(_call("calc ChatCompletionRequest: + return cast(ChatCompletionRequest, _FakeReq()) + + +# The parsers accept a missing request at runtime. +_NO_REQ = cast(ChatCompletionRequest, None) + + +def _req(*names: str) -> ChatCompletionRequest: """A request with ``names`` registered as tools.""" - return SimpleNamespace( - tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names] + return cast( + ChatCompletionRequest, + SimpleNamespace( + tools=[SimpleNamespace(function=SimpleNamespace(name=n)) for n in names] + ), ) @@ -90,7 +103,7 @@ def test_single_tool_call_after_reasoning(): 'celsius\n' "\n<|eot|>" ) - out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None) + out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) assert out.tools_called and len(out.tool_calls) == 1 assert out.tool_calls[0].function.name == "weather.get" assert json.loads(out.tool_calls[0].function.arguments) == { @@ -112,7 +125,7 @@ def test_parallel_calls_across_eom_boundaries(): '4\n' "\n<|eot|>" ) - out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None) + out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) assert out.tools_called and len(out.tool_calls) == 2, len(out.tool_calls) assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"] # JSON-typed values decode to ints @@ -127,14 +140,14 @@ def test_echoed_invoke_in_reasoning_is_not_parsed(): "but I will not.<|eom|>" "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>" ) - out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None) + out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) assert not out.tools_called, "channel scoping failed -- echoed invoke parsed!" assert out.content == "The answer is 42.", repr(out.content) def test_plain_answer_yields_no_tool_calls(): out = MuseGlimmerToolParser.extract_tool_calls( - T, "to=user<|message|>Just a plain answer.<|eot|>", None + T, "to=user<|message|>Just a plain answer.<|eot|>", _NO_REQ ) assert not out.tools_called @@ -147,7 +160,7 @@ def test_json_object_array_and_bool_params_decode(): 'true\n' "\n<|eot|>" ) - out = MuseGlimmerToolParser.extract_tool_calls(T, raw, None) + out = MuseGlimmerToolParser.extract_tool_calls(T, raw, _NO_REQ) assert json.loads(out.tool_calls[0].function.arguments) == { "payload": {"nested": [1, 2, 3]}, "flag": True, @@ -166,10 +179,10 @@ def test_reasoning_to_toolcall_handoff(): 'Paris\n' "\n" ) - reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None) + reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ) assert reasoning == "Let me call the tool.", repr(reasoning) assert content is not None and "thinking<|eom|>" "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>" ) - reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None) + reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ) assert reasoning == "thinking", repr(reasoning) assert content == "The answer is 42.", repr(content) - assert not MuseGlimmerToolParser.extract_tool_calls(T, content, None).tools_called + assert not MuseGlimmerToolParser.extract_tool_calls( + T, content, _NO_REQ + ).tools_called def test_plain_content_without_framing_passes_through(): reasoning, content = MuseGlimmerReasoningParser.extract_reasoning( - R, "Just a direct answer.", None + R, "Just a direct answer.", _NO_REQ ) assert reasoning is None and content == "Just a direct answer.", ( reasoning, @@ -207,9 +222,10 @@ def test_reasoning_then_parallel_calls(): '3\n\n' "<|eot|>" ) - reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, None) + reasoning, content = MuseGlimmerReasoningParser.extract_reasoning(R, raw, _NO_REQ) assert reasoning == "need two calls", repr(reasoning) - out = MuseGlimmerToolParser.extract_tool_calls(T, content, None) + assert content is not None + out = MuseGlimmerToolParser.extract_tool_calls(T, content, _NO_REQ) assert [t.function.name for t in out.tool_calls] == ["math.add", "math.mul"], ( out.tool_calls ) @@ -231,7 +247,7 @@ def _stream(raw: str, chunk: int): R, prev, cur, delta, [], [], [] ) if dm is not None: - if getattr(dm, "reasoning", None): + if dm.reasoning: reasoning.append(dm.reasoning) content_delta = getattr(dm, "content", None) # Tool-channel content is an internal handoff to the tool parser, @@ -239,7 +255,7 @@ def _stream(raw: str, chunk: int): if content_delta and "" not in content_delta: content.append(content_delta) dt = MuseGlimmerToolParser.extract_tool_calls_streaming( - T, prev, cur, delta, [], [], [], _FakeReq() + T, prev, cur, delta, [], [], [], _fake_req() ) if dt is not None and dt.tool_calls: toolcalls.extend(dt.tool_calls) @@ -319,11 +335,11 @@ def test_streaming_reasoning_then_content(): def test_truncated_cot_no_toolcall_nonstreaming(): - out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _FakeReq()) + out = MuseGlimmerToolParser.extract_tool_calls(T, RAW_TRUNCATED, _fake_req()) assert not out.tools_called and out.tool_calls == [] # partial reasoning must still be recovered by the reasoning parser reasoning, _ = MuseGlimmerReasoningParser.extract_reasoning( - R, RAW_TRUNCATED, _FakeReq() + R, RAW_TRUNCATED, _fake_req() ) assert reasoning and "Maybe I should call" in reasoning, repr(reasoning) @@ -374,7 +390,7 @@ def test_trailing_segment_ambiguous_left_alone(): def test_no_registered_tools_passthrough(): out = MuseGlimmerToolParser.extract_tool_calls( - T, _call("get_weather.get_weather"), None + T, _call("get_weather.get_weather"), _NO_REQ ) assert out.tool_calls[0].function.name == "get_weather.get_weather" diff --git a/tests/tool_use/test_responses_request_validations.py b/tests/tool_use/test_responses_request_validations.py index caeb7042d6..4952ea964e 100644 --- a/tests/tool_use/test_responses_request_validations.py +++ b/tests/tool_use/test_responses_request_validations.py @@ -203,6 +203,7 @@ def test_input_image_detail_defaults_or_preserves(detail: str | None) -> None: request = ResponsesRequest.model_validate({"input": [item]}) + assert isinstance(request.input, list) assert request.input[0]["content"][0]["detail"] == ( "auto" if detail is None else detail ) diff --git a/tests/tool_use/test_tool_choice_required.py b/tests/tool_use/test_tool_choice_required.py index f37a3c9681..f2b98eb17b 100644 --- a/tests/tool_use/test_tool_choice_required.py +++ b/tests/tool_use/test_tool_choice_required.py @@ -306,18 +306,17 @@ def _collect_required_tool_streaming_json(output_json: str, delta_len: int) -> s combined_messages = "[" for message in messages: - if message.tool_calls[0].function.name: + fn = message.tool_calls[0].function + assert fn is not None + if fn.name: if len(combined_messages) > 1: combined_messages += "}," combined_messages += ( - '{"name": "' - + message.tool_calls[0].function.name - + '", "parameters": ' - + message.tool_calls[0].function.arguments + '{"name": "' + fn.name + '", "parameters": ' + (fn.arguments or "") ) else: - combined_messages += message.tool_calls[0].function.arguments + combined_messages += fn.arguments or "" combined_messages += "}]" return combined_messages diff --git a/tests/utils_/test_argparse_utils.py b/tests/utils_/test_argparse_utils.py index 89e0f9977d..815ac3fa19 100644 --- a/tests/utils_/test_argparse_utils.py +++ b/tests/utils_/test_argparse_utils.py @@ -476,32 +476,32 @@ def test_compilation_config_mode_validator(): """Test that CompilationConfig.mode field validator converts strings to integers.""" from vllm.config.compilation import CompilationConfig, CompilationMode - config = CompilationConfig(mode=0) + config = CompilationConfig(mode=0) # type: ignore[arg-type] assert config.mode == CompilationMode.NONE - config = CompilationConfig(mode=3) + config = CompilationConfig(mode=3) # type: ignore[arg-type] assert config.mode == CompilationMode.VLLM_COMPILE - config = CompilationConfig(mode="NONE") + config = CompilationConfig(mode="NONE") # type: ignore[arg-type] assert config.mode == CompilationMode.NONE - config = CompilationConfig(mode="STOCK_TORCH_COMPILE") + config = CompilationConfig(mode="STOCK_TORCH_COMPILE") # type: ignore[arg-type] assert config.mode == CompilationMode.STOCK_TORCH_COMPILE - config = CompilationConfig(mode="DYNAMO_TRACE_ONCE") + config = CompilationConfig(mode="DYNAMO_TRACE_ONCE") # type: ignore[arg-type] assert config.mode == CompilationMode.DYNAMO_TRACE_ONCE - config = CompilationConfig(mode="VLLM_COMPILE") + config = CompilationConfig(mode="VLLM_COMPILE") # type: ignore[arg-type] assert config.mode == CompilationMode.VLLM_COMPILE - config = CompilationConfig(mode="none") + config = CompilationConfig(mode="none") # type: ignore[arg-type] assert config.mode == CompilationMode.NONE - config = CompilationConfig(mode="vllm_compile") + config = CompilationConfig(mode="vllm_compile") # type: ignore[arg-type] assert config.mode == CompilationMode.VLLM_COMPILE with pytest.raises(ValidationError, match="Invalid compilation mode"): - CompilationConfig(mode="INVALID_MODE") + CompilationConfig(mode="INVALID_MODE") # type: ignore[arg-type] def test_flat_product(): diff --git a/tests/utils_/test_cache.py b/tests/utils_/test_cache.py index 8d09b22415..3fce04d268 100644 --- a/tests/utils_/test_cache.py +++ b/tests/utils_/test_cache.py @@ -128,7 +128,7 @@ def test_lru_cache(): def test_lru_cache_put_if_fits(): - cache = LRUCache(10, getsizeof=lambda x: x) + cache = LRUCache[str, int](10, getsizeof=lambda x: x) assert cache.put_if_fits("ok", 4) is True assert cache["ok"] == 4 diff --git a/tests/utils_/test_network_utils.py b/tests/utils_/test_network_utils.py index 02889c20f6..a6c359e269 100644 --- a/tests/utils_/test_network_utils.py +++ b/tests/utils_/test_network_utils.py @@ -206,8 +206,8 @@ def test_make_zmq_socket_ipv6(): def test_make_zmq_path(): - assert make_zmq_path("tcp", "127.0.0.1", "5555") == "tcp://127.0.0.1:5555" - assert make_zmq_path("tcp", "::1", "5555") == "tcp://[::1]:5555" + assert make_zmq_path("tcp", "127.0.0.1", 5555) == "tcp://127.0.0.1:5555" + assert make_zmq_path("tcp", "::1", 5555) == "tcp://[::1]:5555" def test_get_tcp_uri(): diff --git a/tests/v1/cudagraph/test_cudagraph_dispatch.py b/tests/v1/cudagraph/test_cudagraph_dispatch.py index ddf72c3b11..40874be5bd 100644 --- a/tests/v1/cudagraph/test_cudagraph_dispatch.py +++ b/tests/v1/cudagraph/test_cudagraph_dispatch.py @@ -257,7 +257,7 @@ class TestCudagraphDispatcher: def test_get_capture_descs_empty_when_not_initialized(self): """Test that get_capture_descs returns empty list when keys not initialized.""" comp_config = CompilationConfig( - cudagraph_mode="FULL", + cudagraph_mode=CUDAGraphMode.FULL, mode=CompilationMode.NONE, cudagraph_capture_sizes=[1, 8], ) @@ -419,7 +419,7 @@ def _run_and_monitor_call( def test_capture_replay_bypass_logic(): comp_config = CompilationConfig( mode=CompilationMode.VLLM_COMPILE, - cudagraph_mode="FULL", + cudagraph_mode=CUDAGraphMode.FULL, cudagraph_capture_sizes=[1, 2], ) vllm_config = _create_vllm_config(comp_config) @@ -489,7 +489,7 @@ def test_nested_wrappers(): """Tests a scenario with a PIECEWISE wrapper inside a FULL one.""" comp_config = CompilationConfig( mode=CompilationMode.VLLM_COMPILE, - cudagraph_mode="FULL", + cudagraph_mode=CUDAGraphMode.FULL, cudagraph_capture_sizes=[1], ) vllm_config = _create_vllm_config(comp_config) diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 7d2948cba4..e5f2f6369f 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -38,7 +38,7 @@ def _reset_graph_pool_id(): def _create_vllm_config() -> MagicMock: compilation_config = CompilationConfig( - cudagraph_mode="FULL", + cudagraph_mode=CUDAGraphMode.FULL, cudagraph_capture_sizes=[4], ) compilation_config.max_cudagraph_capture_size = 4 @@ -134,7 +134,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings(): device=torch.device("cpu"), max_num_reqs=num_reqs, max_num_tokens=num_tokens, - block_tables=pcp_block_tables, + block_tables=pcp_block_tables, # type: ignore[arg-type] ) block_tables = MagicMock() @@ -224,7 +224,7 @@ def test_speculator_capture_preserves_decode_query_bounds( ) manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager) manager.max_num_reqs = num_reqs - manager.vllm_config = SimpleNamespace( + manager.vllm_config = SimpleNamespace( # type: ignore[assignment] parallel_config=SimpleNamespace(data_parallel_size=1) ) buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu")) @@ -264,7 +264,7 @@ def _create_decode_vllm_config( use_kda_recoverssm: bool = False, ) -> MagicMock: compilation_config = CompilationConfig( - cudagraph_mode="FULL_AND_PIECEWISE", + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, cudagraph_capture_sizes=capture_sizes, ) compilation_config.max_cudagraph_capture_size = capture_sizes[-1] @@ -473,6 +473,7 @@ def test_dynamic_spec_decode_shared_token_count_stays_reachable(monkeypatch): assert any(len(descs) > 1 for descs in by_num_tokens.values()) for desc in full_descs: + assert desc.num_reqs is not None assert desc in manager._candidates[(desc.num_tokens, 0)], desc assert ( manager.dispatch( diff --git a/tests/v1/cudagraph/test_encoder_cudagraph.py b/tests/v1/cudagraph/test_encoder_cudagraph.py index 1f145bdc89..2f4de52b18 100644 --- a/tests/v1/cudagraph/test_encoder_cudagraph.py +++ b/tests/v1/cudagraph/test_encoder_cudagraph.py @@ -14,7 +14,7 @@ Test organization: from collections.abc import Hashable from types import SimpleNamespace -from typing import Any +from typing import Any, cast from unittest.mock import MagicMock, patch import pytest @@ -100,6 +100,50 @@ class _MockModel(SupportsEncoderCudaGraph): def get_encoder_cudagraph_budget_range(self, vllm_config): return (self._min_budget, self._max_budget) + def get_max_frames_per_video(self) -> int: + return 0 + + def get_encoder_cudagraph_item_specs( + self, mm_kwargs: dict[str, Any] + ) -> list[EncoderItemSpec]: + return [] + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + path: str = "default", + axis_keys: tuple[Hashable, ...] | None = None, + ) -> EncoderCudaGraphCaptureInputs: + return EncoderCudaGraphCaptureInputs(values={}) + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + path: str = "default", + ) -> EncoderCudaGraphReplayBuffers: + return EncoderCudaGraphReplayBuffers(values={}) + + def encoder_cudagraph_forward( + self, inputs: dict[str, torch.Tensor], path: str = "default" + ) -> torch.Tensor: + return torch.zeros(0) + + def encoder_eager_forward( + self, mm_kwargs: dict[str, Any], path: str = "default" + ) -> torch.Tensor: + return torch.zeros(0) + + def select_encoder_cudagraph_items( + self, mm_kwargs: dict[str, Any], indices: list[int] + ) -> dict[str, Any]: + return {} + def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager: """Create a minimal EncoderCudaGraphManager with only token_budgets set. @@ -594,6 +638,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only ECConnectorBase, ECConnectorMetadata, ) + from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.worker.gpu.ec_connector import ActiveECConnector from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner @@ -607,7 +652,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only encoder.device = device encoder.cudagraph_manager = manager runner = object.__new__(MMEncoderModelRunner) - runner.model_state = SimpleNamespace(encoder_runner=encoder) + runner.model_state = SimpleNamespace(encoder_runner=encoder) # type: ignore[assignment] # No decoder manager is installed: capture must be encoder-only. with patch( "vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace @@ -628,9 +673,12 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only with patch( "vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector ): - ec = ActiveECConnector(SimpleNamespace(), cache) - scheduled = SimpleNamespace( - ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset() + ec = ActiveECConnector(SimpleNamespace(), cache) # type: ignore[arg-type] + scheduled = cast( + SchedulerOutput, + SimpleNamespace( + ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset() + ), ) saved_outputs: list[tuple[torch.Tensor, torch.Tensor]] = [] # Mixed sizes exercise packing order; 64 is the boundary, 81 falls back. @@ -643,6 +691,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only outputs = manager.execute(inputs) assert outputs is not None cache[str(len(cache))] = outputs[0] + assert ec_output is not None assert ec_output.finished_sending is None assert outputs is not None for actual, eager in zip(outputs, expected): @@ -656,6 +705,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only connector.get_finished.return_value = (set(cache), None) with ec.maybe_get_output(scheduled) as ec_output: pass + assert ec_output is not None assert ec_output.finished_sending == set(cache) @@ -666,7 +716,7 @@ def test_eonly_without_encoder_graph_skips_capture(): encoder = object.__new__(EncoderRunner) encoder.cudagraph_manager = None runner = object.__new__(MMEncoderModelRunner) - runner.model_state = SimpleNamespace(encoder_runner=encoder) + runner.model_state = SimpleNamespace(encoder_runner=encoder) # type: ignore[assignment] assert runner.capture_model() == 0 @@ -973,7 +1023,7 @@ class TestInitInvariantValidation: vllm_config = _MockVllmConfig(token_budgets, max_mm_items) model = _MockModel(min_budget, max_budget) return EncoderCudaGraphManager( - vllm_config=vllm_config, + vllm_config=vllm_config, # type: ignore[arg-type] device=torch.device("cpu"), dtype=torch.float32, model=model, diff --git a/tests/v1/determinism/test_batch_invariance.py b/tests/v1/determinism/test_batch_invariance.py index 9af55b9fbb..b167078e06 100644 --- a/tests/v1/determinism/test_batch_invariance.py +++ b/tests/v1/determinism/test_batch_invariance.py @@ -155,7 +155,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle( # Ensure engines are shutdown to free GPU/VRAM across test sessions if llm is not None: with contextlib.suppress(Exception): - llm.shutdown() + llm.llm_engine.engine_core.shutdown() @skip_unsupported @@ -456,7 +456,7 @@ def test_logprobs_bitwise_batch_invariance_ragged_chunked_prefill(backend): ) finally: with contextlib.suppress(Exception): - llm.shutdown() + llm.llm_engine.engine_core.shutdown() @skip_unsupported @@ -505,7 +505,7 @@ def test_simple_generation(backend): finally: with contextlib.suppress(Exception): - llm.shutdown() + llm.llm_engine.engine_core.shutdown() @skip_unsupported @@ -815,15 +815,8 @@ def test_decode_logprobs_match_prefill_logprobs( if token_idx == 0: prefix_prompt = prompt else: - # Use the partial output text up to this token - # We'll need to construct this from the full output - prefix_output = decode_output.outputs[0] - # Get the text for tokens 0 to token_idx-1 - # Unfortunately, we don't have per-token text, so we'll use - # a different approach: run prefill with prompt + tokens[0:token_idx] - - # Actually, we need to get the actual text. Let's use a workaround: - # Run a generation with max_tokens = token_idx to get that prefix + # No per-token text, so regenerate with max_tokens = token_idx + # to get the prefix text prefix_sp = SamplingParams( temperature=0.0, max_tokens=token_idx, diff --git a/tests/v1/determinism/test_batch_invariance_vlm.py b/tests/v1/determinism/test_batch_invariance_vlm.py index 00f138d62c..fe95b03c61 100644 --- a/tests/v1/determinism/test_batch_invariance_vlm.py +++ b/tests/v1/determinism/test_batch_invariance_vlm.py @@ -12,6 +12,7 @@ from PIL import Image from utils import _extract_step_logprobs, skip_if_not_cuda from vllm import LLM, SamplingParams +from vllm.inputs import TextPrompt VLM_TEST_MODEL = os.getenv("VLLM_VLM_TEST_MODEL", "Qwen/Qwen3-VL-2B-Instruct") @@ -30,8 +31,8 @@ def _make_image(seed: int, size: int) -> Image.Image: return Image.fromarray(rng.integers(0, 255, (size, size, 3), dtype=np.uint8)) -def _make_inputs(input_type: str, num_reqs: int) -> list[dict]: - inputs = [] +def _make_inputs(input_type: str, num_reqs: int) -> list[TextPrompt]: + inputs: list[TextPrompt] = [] for i in range(num_reqs): if input_type == "image": prompt = IMAGE_PROMPT @@ -54,7 +55,7 @@ def _make_inputs(input_type: str, num_reqs: int) -> list[dict]: return inputs -def _assert_batch_invariant(llm: LLM, inputs: list[dict], sampling) -> None: +def _assert_batch_invariant(llm: LLM, inputs: list[TextPrompt], sampling) -> None: bs1 = [ _extract_step_logprobs(llm.generate([inp], sampling, use_tqdm=False)[0]) for inp in inputs @@ -134,4 +135,4 @@ def _run_vlm_batch_invariance( _assert_batch_invariant(llm, inputs, sampling) finally: with contextlib.suppress(Exception): - llm.shutdown() + llm.llm_engine.engine_core.shutdown() diff --git a/tests/v1/determinism/test_cutlass_batch_invariance.py b/tests/v1/determinism/test_cutlass_batch_invariance.py index 95403e50ff..c3384a62f6 100644 --- a/tests/v1/determinism/test_cutlass_batch_invariance.py +++ b/tests/v1/determinism/test_cutlass_batch_invariance.py @@ -221,13 +221,15 @@ def _make_cutlass_fp4_moe_batch_invariant_case( in_dtype=dtype, activation=activation, ) + prepare_finalize = maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ) + assert prepare_finalize is not None kernel = mk.FusedMoEKernel( - maybe_make_prepare_finalize( - moe=moe_config, - quant_config=quant_config, - allow_new_interface=True, - use_monolithic=False, - ), + prepare_finalize, CutlassExpertsFp4( moe_config=moe_config, quant_config=quant_config, diff --git a/tests/v1/determinism/test_nvfp4_batch_invariant.py b/tests/v1/determinism/test_nvfp4_batch_invariant.py index dafd1a1644..1e9b0cc8fc 100644 --- a/tests/v1/determinism/test_nvfp4_batch_invariant.py +++ b/tests/v1/determinism/test_nvfp4_batch_invariant.py @@ -97,4 +97,4 @@ def test_dense_nvfp4_generation_is_deterministic_across_batch_sizes_e2e(backend) finally: if llm is not None: with contextlib.suppress(Exception): - llm.shutdown() + llm.llm_engine.engine_core.shutdown() diff --git a/tests/v1/determinism/test_rms_norm_batch_invariant.py b/tests/v1/determinism/test_rms_norm_batch_invariant.py index 5699a661c1..1feb5e6741 100644 --- a/tests/v1/determinism/test_rms_norm_batch_invariant.py +++ b/tests/v1/determinism/test_rms_norm_batch_invariant.py @@ -524,6 +524,7 @@ def test_rms_norm_batch_invariance(dtype): batch[4] = row[0] out_batch = rms_norm_batch_invariant(batch, weight, eps=eps) + assert isinstance(out_single, torch.Tensor) and isinstance(out_batch, torch.Tensor) assert torch.equal(out_single[0], out_batch[4]), ( "rms_norm output for a row differs when batch context changes" ) @@ -547,6 +548,7 @@ if __name__ == "__main__": # Batch-invariant implementation triton_output = rms_norm_batch_invariant(input_tensor, weight, eps=eps) + assert isinstance(triton_output, torch.Tensor) # Compare max_diff = (triton_output - reference_output).abs().max().item() diff --git a/tests/v1/executor/test_executor.py b/tests/v1/executor/test_executor.py index a21a8dafd3..949282cffb 100644 --- a/tests/v1/executor/test_executor.py +++ b/tests/v1/executor/test_executor.py @@ -85,12 +85,12 @@ def test_multiproc_executor_worker_termination_timeout( monkeypatch.setattr(multiproc_executor_module.time, "sleep", clock.sleep) executor = MultiprocExecutor.__new__(MultiprocExecutor) proc = _FakeProcess(clock, exits_at=exits_at) - executor._ensure_worker_termination([proc]) + executor._ensure_worker_termination([proc]) # type: ignore[list-item] assert proc.terminate_called is expected_terminate class CustomMultiprocExecutor(MultiprocExecutor): - def collective_rpc( + def collective_rpc( # type: ignore[override] self, method: str | Callable, timeout: float | None = None, @@ -98,7 +98,7 @@ class CustomMultiprocExecutor(MultiprocExecutor): kwargs: dict | None = None, non_block: bool = False, unique_reply_rank: int | None = None, - kv_output_aggregator: KVOutputAggregator = None, + kv_output_aggregator: KVOutputAggregator | None = None, ec_output_aggregator: ECOutputAggregator | None = None, ) -> Any | list[Any] | Future[Any | list[Any]]: # Drop marker to show that this was run @@ -126,7 +126,7 @@ def test_custom_executor_type_checking(): model=MODEL, gpu_memory_utilization=0.2, max_model_len=8192, - distributed_executor_backend=Mock, + distributed_executor_backend=Mock, # type: ignore[arg-type] ) LLMEngine.from_engine_args(engine_args) with pytest.raises(ValueError): @@ -134,7 +134,7 @@ def test_custom_executor_type_checking(): model=MODEL, gpu_memory_utilization=0.2, max_model_len=8192, - distributed_executor_backend=Mock, + distributed_executor_backend=Mock, # type: ignore[arg-type] ) AsyncLLM.from_engine_args(engine_args) diff --git a/tests/v1/simple_kv_offload/test_integration.py b/tests/v1/simple_kv_offload/test_integration.py index 02f6360e08..ea6caa4215 100644 --- a/tests/v1/simple_kv_offload/test_integration.py +++ b/tests/v1/simple_kv_offload/test_integration.py @@ -55,6 +55,7 @@ def _flush_gpu_cache(llm: LLM, sampling_params: SamplingParams, seed: int = 0): block_size = cache_config.block_size # Use 1.2x GPU capacity to give the lazy cursor enough scheduling steps # to walk past all target blocks near the tail of the free queue. + assert num_gpu_blocks is not None total_tokens_needed = int(num_gpu_blocks * block_size * 1.5) # Use token-id prompts so each filler is unique (no prefix sharing). diff --git a/tests/v1/simple_kv_offload/test_kv_events.py b/tests/v1/simple_kv_offload/test_kv_events.py index 68121f17e0..4876ae4f38 100644 --- a/tests/v1/simple_kv_offload/test_kv_events.py +++ b/tests/v1/simple_kv_offload/test_kv_events.py @@ -260,6 +260,7 @@ def test_block_stored_per_group_metadata_full_attention() -> None: ) assert ev.block_hashes == [expected_hash_0] assert ev.parent_block_hash is None + assert req.prompt_token_ids is not None assert ev.token_ids == req.prompt_token_ids[0:BLOCK_SIZE] # Second block's parent_block_hash equals the first block's hash. @@ -296,7 +297,7 @@ def test_eager_store_lora_metadata() -> None: f"expected lora_name='test-lora', got {ev.lora_name!r}" ) assert ev.extra_keys is not None, "expected extra_keys for lora request" - assert any("test-lora" in keys for keys in ev.extra_keys), ( + assert any(keys is not None and "test-lora" in keys for keys in ev.extra_keys), ( f"expected 'test-lora' in extra_keys, got {ev.extra_keys}" ) assert len(ev.token_ids) == BLOCK_SIZE @@ -372,6 +373,7 @@ def test_finished_eager_store_emits_all_storage_events() -> None: secondary_event_hash = maybe_convert_block_hash(get_block_hash(secondary_hash)) by_hash = {event.block_hashes[0]: event for event in stored} assert set(by_hash) == {*primary_event_hashes, secondary_event_hash} + assert req.prompt_token_ids is not None assert ( by_hash[primary_event_hashes[0]].token_ids == req.prompt_token_ids[:BLOCK_SIZE] ) @@ -451,6 +453,7 @@ def test_secondary_hash_block_stored_metadata() -> None: by_hash = {event.block_hashes[0]: event for event in stored} assert set(by_hash) == {primary_event_hash, secondary_event_hash} assert by_hash[primary_event_hash].block_size == BLOCK_SIZE + assert req.prompt_token_ids is not None assert by_hash[primary_event_hash].token_ids == req.prompt_token_ids[:BLOCK_SIZE] assert by_hash[secondary_event_hash].block_size == BLOCK_SIZE // 2 assert ( @@ -752,6 +755,7 @@ def test_mamba_align_skips_positional_event_metadata() -> None: stored = [e for e in events if isinstance(e, BlockStored)] assert len(stored) == 2 assert {ev.group_idx for ev in stored} == {0} + assert req.prompt_token_ids is not None for idx, ev in enumerate(stored): assert ev.block_size == vbs assert ev.token_ids == req.prompt_token_ids[idx * vbs : (idx + 1) * vbs] diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index 3817a15e3e..8a85c88d70 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -19,6 +19,7 @@ from vllm.config import ( SchedulerConfig, VllmConfig, ) +from vllm.config.cache import MambaCacheMode from vllm.config.kv_events import KVEventsConfig from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import ( SimpleCPUOffloadConnector, @@ -1243,7 +1244,7 @@ def test_preemption_no_cpu_block_leak() -> None: block_hasher=req._block_hasher, ) hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens > 0 + assert hit_tokens is not None and hit_tokens > 0 gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) @@ -1338,7 +1339,7 @@ def test_inflight_finish_deferred_cleanup() -> None: block_hasher=req._block_hasher, ) hit_tokens, _ = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens > 0 + assert hit_tokens is not None and hit_tokens > 0 gpu_blocks2 = fix.gpu_block_pool.get_new_blocks(num_blocks) kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) @@ -1896,7 +1897,7 @@ def test_reset_pending_loads() -> None: block_hasher=req._block_hasher, ) hit_tokens, is_async = sched.get_num_new_matched_tokens(req2, num_computed_tokens=0) - assert hit_tokens > 0 + assert hit_tokens is not None and hit_tokens > 0 gpu_blocks2 = gpu_pool.get_new_blocks(num_blocks) kv_blocks2 = KVCacheBlocks(blocks=(gpu_blocks2,)) @@ -2246,7 +2247,7 @@ def _make_hybrid_attention_mamba_scheduler( hash_block_size: int | None = None, dcp_world_size: int = 4, lazy: bool = False, - mamba_cache_mode: str = "align", + mamba_cache_mode: MambaCacheMode = "align", enable_kv_cache_events: bool = False, ) -> SchedulerFixture: """Build a scheduler for one attention group plus one Mamba group.""" diff --git a/tests/v1/simple_kv_offload/test_worker.py b/tests/v1/simple_kv_offload/test_worker.py index 1739290051..2936d0eb87 100644 --- a/tests/v1/simple_kv_offload/test_worker.py +++ b/tests/v1/simple_kv_offload/test_worker.py @@ -36,6 +36,7 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.kv_cache_interface import ( FullAttentionSpec, + KVCacheConfig, KVCacheGroupSpec, KVCacheLayout, KVCacheSpec, @@ -50,7 +51,10 @@ from vllm.v1.simple_kv_offload.cuda_mem_ops import ( build_params, ) from vllm.v1.simple_kv_offload.disk_backend import DiskBackend -from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadMetadata +from vllm.v1.simple_kv_offload.metadata import ( + SimpleCPUOffloadMetadata, + SimpleCPUOffloadWorkerMetadata, +) from vllm.v1.simple_kv_offload.worker import SimpleCPUOffloadWorker from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector from vllm.v1.worker.kv_connector_model_runner_mixin import KVConnectorModelRunnerMixin @@ -64,6 +68,12 @@ ITERS = 30 SLEEP_CYCLES = 50_000_000 +def _make_worker( + kv_cache_config: KVCacheConfig | None, cpu_capacity_bytes: int +) -> SimpleCPUOffloadWorker: + return SimpleCPUOffloadWorker(None, kv_cache_config, cpu_capacity_bytes) # type: ignore[arg-type] + + def _make_backend() -> tuple[DmaCopyBackend, torch.Tensor, torch.Tensor]: gpu = {"k": torch.zeros((NUM_BLOCKS, BLOCK_BYTES), dtype=torch.int8, device="cuda")} cpu = { @@ -87,7 +97,7 @@ def _make_backend() -> tuple[DmaCopyBackend, torch.Tensor, torch.Tensor]: def test_no_forward_step_completes_cpu_store(monkeypatch, use_v2): """A finished request's final store must drain without another model step.""" backend, gpu, cpu = _make_backend() - worker = SimpleCPUOffloadWorker(None, None, cpu_capacity_bytes=0) + worker = _make_worker(None, 0) worker._backend = backend connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector) connector.worker_handler = worker @@ -105,13 +115,13 @@ def test_no_forward_step_completes_cpu_store(monkeypatch, use_v2): active = ActiveKVConnector.__new__(ActiveKVConnector) active.kv_connector = connector active._disabled = False - active._pending_load_start = False + active._pending_load_kwargs = None module = "vllm.v1.worker.gpu.kv_connector" monkeypatch.setattr(f"{module}.is_forward_context_available", lambda: True) monkeypatch.setattr(f"{module}.get_forward_context", lambda: None) result = active.no_forward(output) else: - result = KVConnectorModelRunnerMixin.kv_connector_no_forward(output, None) + result = KVConnectorModelRunnerMixin.kv_connector_no_forward(output, None) # type: ignore[arg-type] completion = ( result.kv_connector_output.kv_connector_worker_meta if result.kv_connector_output is not None @@ -123,6 +133,7 @@ def test_no_forward_step_completes_cpu_store(monkeypatch, use_v2): completion = connector.build_connector_worker_meta() time.sleep(0.001) assert completion is not None, "CPU store never completed on the empty step" + assert isinstance(completion, SimpleCPUOffloadWorkerMetadata) assert completion.completed_store_events == {0: 1} assert torch.equal(cpu[0], gpu[0].cpu()) finally: @@ -132,7 +143,7 @@ def test_no_forward_step_completes_cpu_store(monkeypatch, use_v2): def test_deferred_store_waits_for_draft_forward(monkeypatch): """Target completion must not submit the store before draft finalization.""" backend, gpu, cpu = _make_backend() - worker = SimpleCPUOffloadWorker(None, None, cpu_capacity_bytes=0) + worker = _make_worker(None, 0) worker._backend = backend connector = SimpleCPUOffloadConnector.__new__(SimpleCPUOffloadConnector) connector.worker_handler = worker @@ -267,11 +278,12 @@ class _RecordingBackend: def test_transfer_hooks_pass_wait_event_for_store_only(): """wait_for_save gates stores on a compute-done event; start_load_kv does not.""" - worker = SimpleCPUOffloadWorker( - vllm_config=None, kv_cache_config=None, cpu_capacity_bytes=0 + worker = _make_worker( + kv_cache_config=None, + cpu_capacity_bytes=0, ) recording = _RecordingBackend() - worker._backend = recording + worker._backend = recording # type: ignore[assignment] worker._connector_metadata = SimpleCPUOffloadMetadata( load_event=0, load_gpu_blocks=[0], @@ -332,8 +344,7 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout): ) ], ) - worker = SimpleCPUOffloadWorker( - vllm_config=None, + worker = _make_worker( kv_cache_config=cache_config, cpu_capacity_bytes=raw.nbytes, ) @@ -366,8 +377,7 @@ def test_register_kv_cache_storage_with_trailing_padding(monkeypatch): cache_bytes = num_blocks * block_bytes raw = torch.zeros(4096, dtype=torch.int8, device="cuda") cache = raw[:cache_bytes].view(num_blocks, block_bytes) - worker = SimpleCPUOffloadWorker( - vllm_config=None, + worker = _make_worker( kv_cache_config=MagicMock( num_blocks=num_blocks, kv_cache_tensors=[ @@ -412,8 +422,7 @@ def test_register_separate_kv_head_groups(monkeypatch): ) caches = dense_kv_cache_views(raw, spec, num_blocks, num_layers, layout) layer_names = [f"layer.{i}" for i in range(num_layers)] - worker = SimpleCPUOffloadWorker( - vllm_config=None, + worker = _make_worker( kv_cache_config=MagicMock( num_blocks=num_blocks, kv_cache_tensors=[ @@ -509,8 +518,7 @@ def test_register_mixed_page_sizes_in_one_cache_group(monkeypatch): assert len(set(pages)) > 1, "the mixed page sizes are what this test covers" kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout) - worker = SimpleCPUOffloadWorker( - vllm_config=None, + worker = _make_worker( kv_cache_config=kv_cache_config, cpu_capacity_bytes=sum(pages) * num_blocks, ) @@ -567,8 +575,7 @@ def test_register_mixed_page_sizes_odd_block_counts(monkeypatch, rank_blocks): assert kv_cache_config.num_blocks == rank_blocks kv_caches = allocate_kv_cache(kv_cache_config, torch.device("cuda"), layout) - worker = SimpleCPUOffloadWorker( - vllm_config=None, + worker = _make_worker( kv_cache_config=kv_cache_config, cpu_capacity_bytes=sum(pages) * rank_blocks, ) diff --git a/tests/v1/structured_output/test_backend_guidance.py b/tests/v1/structured_output/test_backend_guidance.py index edcdd983c0..055ac9c364 100644 --- a/tests/v1/structured_output/test_backend_guidance.py +++ b/tests/v1/structured_output/test_backend_guidance.py @@ -91,7 +91,7 @@ def test_grammar_bitmask_with_specdec(): json='{"type": "object"}', ), ) - sampling_params.structured_outputs._backend = "guidance" + sampling_params.structured_outputs._backend = "guidance" # type: ignore[union-attr] sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) my_req_id = f"my_req_id_{i}" @@ -107,19 +107,23 @@ def test_grammar_bitmask_with_specdec(): def grammar_bitmask(req: Request, tokens: list[int]) -> None: structured_output_manager.grammar_bitmask( requests={req.request_id: req}, - structured_output_request_ids={req.request_id: 0}, + structured_output_request_ids=[req.request_id], scheduled_spec_decode_tokens={req.request_id: tokens}, ) # At this point, we rolled-back, so should not be terminated - assert not req.structured_output_request.grammar.is_terminated() + assert req.structured_output_request is not None + grammar = req.structured_output_request.grammar + assert grammar is not None and not isinstance(grammar, Exception) + assert not grammar.is_terminated() # The grammar might not yet be compiled, so we wait for it + assert request.structured_output_request is not None while not request.structured_output_request._check_grammar_completion(): continue - assert request.structured_output_request.grammar.accept_tokens( - request.request_id, prompt[:i] - ) + grammar = request.structured_output_request.grammar + assert grammar is not None and not isinstance(grammar, Exception) + assert grammar.accept_tokens(request.request_id, prompt[:i]) grammar_bitmask(request, prompt[i:] + [tokenizer.eos_token_id]) grammar_bitmask( @@ -155,7 +159,7 @@ def test_grammar_init_async_and_sync(async_grammar): json='{"type": "object"}', ), ) - sampling_params.structured_outputs._backend = "guidance" + sampling_params.structured_outputs._backend = "guidance" # type: ignore[union-attr] sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) request = Request( @@ -169,6 +173,7 @@ def test_grammar_init_async_and_sync(async_grammar): # Check the internal _grammar type immediately after init # Before _check_grammar_completion is called, async mode should have a Future + assert request.structured_output_request is not None raw_grammar = request.structured_output_request._grammar if async_grammar: assert isinstance(raw_grammar, Future), ( @@ -191,7 +196,7 @@ def test_grammar_init_async_and_sync(async_grammar): # Verify grammar is properly initialized and functional grammar = request.structured_output_request.grammar - assert grammar is not None + assert grammar is not None and not isinstance(grammar, Exception) assert not grammar.is_terminated() # Verify the grammar can accept valid tokens diff --git a/tests/v1/structured_output/test_backend_xgrammar_lark.py b/tests/v1/structured_output/test_backend_xgrammar_lark.py index 6b13f270d9..9293b19296 100644 --- a/tests/v1/structured_output/test_backend_xgrammar_lark.py +++ b/tests/v1/structured_output/test_backend_xgrammar_lark.py @@ -13,6 +13,7 @@ from mistral_common.protocol.instruct.tool_calls import Tool, ToolChoiceEnum from transformers import AutoTokenizer from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.config.structured_outputs import StructuredOutputsBackend from vllm.exceptions import VLLMValidationError from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.tokenizers.mistral import MistralTokenizer @@ -41,13 +42,15 @@ EBNF_GRAMMAR = 'root ::= "id=" [0-9]+' def _validate( - grammar: str, backend: str = "xgrammar", tokenizer: object | None = None + grammar: str, + backend: StructuredOutputsBackend = "xgrammar", + tokenizer: object | None = None, ) -> StructuredOutputsParams: params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar)) params._validate_structured_outputs( - _StubModelConfig(), + _StubModelConfig(), # type: ignore[arg-type] StructuredOutputsConfig(backend=backend), - tokenizer=tokenizer or object(), + tokenizer=tokenizer or object(), # type: ignore[arg-type] ) assert params.structured_outputs is not None return params.structured_outputs diff --git a/tests/v1/structured_output/test_backend_xgrammar_plamo3.py b/tests/v1/structured_output/test_backend_xgrammar_plamo3.py index a6506dafaa..613daa5350 100644 --- a/tests/v1/structured_output/test_backend_xgrammar_plamo3.py +++ b/tests/v1/structured_output/test_backend_xgrammar_plamo3.py @@ -51,7 +51,7 @@ def make_vllm_config() -> VllmConfig: config = VllmConfig( structured_outputs_config=StructuredOutputsConfig(backend="xgrammar"), ) - config.model_config = SimpleNamespace( + config.model_config = SimpleNamespace( # type: ignore[assignment] hf_config=SimpleNamespace(model_type="plamo3") ) return config @@ -73,7 +73,7 @@ def test_plamo3_python_tokenizer_compiles_and_accepts_structural_tag(): tokenizer = Plamo3Tokenizer(["", *call_tokens]) backend = XgrammarBackend( make_vllm_config(), - tokenizer=tokenizer, + tokenizer=tokenizer, # type: ignore[arg-type] vocab_size=len(tokenizer.get_vocab()), ) tools = [ @@ -107,7 +107,7 @@ def test_plamo3_grammar_matches_decoded_bytes(value): tokenizer = Plamo3Tokenizer(list(dict.fromkeys(["", value, *pieces]))) backend = XgrammarBackend( make_vllm_config(), - tokenizer=tokenizer, + tokenizer=tokenizer, # type: ignore[arg-type] vocab_size=len(tokenizer.get_vocab()), ) for tokens in ([value], pieces): diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index a1da205fe0..15239ecb76 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -10,9 +10,11 @@ from transformers import AutoTokenizer from vllm.config import DeviceConfig, StructuredOutputsConfig, VllmConfig from vllm.config.model import ModelConfig from vllm.config.speculative import SpeculativeConfig +from vllm.config.structured_outputs import StructuredOutputsBackend from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.v1.request import Request from vllm.v1.structured_output import StructuredOutputManager +from vllm.v1.structured_output.backend_outlines import OutlinesGrammar TOKENIZER = "gpt2" THINK_END = "\n" # reasoning-end marker (single GPT-2 token) @@ -84,7 +86,7 @@ def _wait_for_grammar(request: Request) -> None: def _build_harness( tokenizer, - backend: str, + backend: StructuredOutputsBackend, prefix: str = "", use_reasoner: bool = True, reasoning_ended: bool | None = None, @@ -106,13 +108,13 @@ def _build_harness( ) manager = StructuredOutputManager(vllm_config) if use_reasoner: - manager.reasoner_cls = MockReasoner + manager.reasoner_cls = MockReasoner # type: ignore[assignment] sampling_params = SamplingParams( structured_outputs=structured_outputs or StructuredOutputsParams(json=JSON_SCHEMA) ) - sampling_params.structured_outputs._backend = backend + sampling_params.structured_outputs._backend = backend # type: ignore[union-attr] sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) prompt_ids = tokenizer.encode(prefix) if prefix else [] @@ -130,9 +132,10 @@ def _build_harness( structured_req = request.structured_output_request assert request.prompt_token_ids is not None assert structured_req is not None - assert structured_req.grammar is not None + grammar = structured_req.grammar + assert grammar is not None and not isinstance(grammar, Exception) if prompt_ids: - assert structured_req.grammar.accept_tokens( + assert grammar.accept_tokens( request.request_id, prompt_ids, ) @@ -173,7 +176,7 @@ def _run_real_flow( structured_req = request.structured_output_request assert structured_req is not None grammar = structured_req.grammar - assert grammar is not None + assert grammar is not None and not isinstance(grammar, Exception) assert not grammar.is_terminated() # `bitmask[i]` is the grammar state after the first `i` scheduled tokens. # These tests commit the validated prefix, so the matching post-accept @@ -335,7 +338,7 @@ FLOW_CASES = [ @pytest.mark.parametrize("case", FLOW_CASES) def test_real_flow( tokenizer, - backend: str, + backend: StructuredOutputsBackend, case: FlowCase, ): if backend == "guidance" and case.xfail_guidance: @@ -379,7 +382,7 @@ def test_real_flow( ) def test_initial_constraint_activation( tokenizer, - backend: str, + backend: StructuredOutputsBackend, use_reasoner: bool, reasoning_ended: bool | None, enable_in_reasoning: bool, @@ -431,7 +434,7 @@ def test_initial_constraint_activation( ) def test_regex_flow( tokenizer, - backend: str, + backend: StructuredOutputsBackend, raw_drafts: tuple[str, ...], expected_validated: tuple[str, ...] | None, expected_row_pattern: str, @@ -459,7 +462,9 @@ def test_regex_flow( @pytest.mark.parametrize("backend", REGEX_BACKENDS) -def test_rejected_draft_keeps_later_rows_constrained(tokenizer, backend: str): +def test_rejected_draft_keeps_later_rows_constrained( + tokenizer, backend: StructuredOutputsBackend +): """A draft the grammar rejects during bitmask fill must not unconstrain the remaining rows or the bonus row.""" manager, request = _build_harness( @@ -492,7 +497,7 @@ def test_outlines_termination(tokenizer): structured_req = request.structured_output_request assert structured_req is not None grammar = structured_req.grammar - assert grammar is not None + assert isinstance(grammar, OutlinesGrammar) one, eos = _to_token_ids(tokenizer, ("1", EOS)) assert not grammar.accept_tokens(request.request_id, [eos]) diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index 739279280c..2ce81534a4 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -35,7 +35,7 @@ def test_structured_outputs_rejected_for_diffusion_models(): ) with pytest.raises(VLLMValidationError, match="not yet supported for diffusion"): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=True), + _StubModelConfig(is_diffusion=True), # type: ignore[arg-type] StructuredOutputsConfig(), tokenizer=None, ) @@ -45,7 +45,7 @@ def test_plain_request_allowed_for_diffusion_models(): """Requests without structured outputs are unaffected by the guard.""" params = SamplingParams() params._validate_structured_outputs( - _StubModelConfig(is_diffusion=True), + _StubModelConfig(is_diffusion=True), # type: ignore[arg-type] StructuredOutputsConfig(), tokenizer=None, ) @@ -71,9 +71,9 @@ def test_degenerate_structured_outputs_rejected(structured_outputs, match): params = SamplingParams(structured_outputs=structured_outputs) with pytest.raises(VLLMValidationError, match=match): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), + _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] StructuredOutputsConfig(), - tokenizer=object(), + tokenizer=object(), # type: ignore[arg-type] ) @@ -97,9 +97,9 @@ def test_regex_with_nul_byte_rejected(regex): # (which would otherwise catch the error and fall back to another backend). with pytest.raises(VLLMValidationError, match="NUL"): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), + _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] StructuredOutputsConfig(), - tokenizer=object(), + tokenizer=object(), # type: ignore[arg-type] ) # The xgrammar backend also rejects it directly (defense in depth), before @@ -133,9 +133,9 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): params = SamplingParams(structured_outputs=structured_outputs) with pytest.raises(VLLMClientError): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), + _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] StructuredOutputsConfig(backend=backend), - tokenizer=object(), + tokenizer=object(), # type: ignore[arg-type] ) @@ -180,8 +180,9 @@ def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend) """`auto` falls back on rejection, so it must catch what the validators raise.""" params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema)) params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), + _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] StructuredOutputsConfig(backend="auto"), - tokenizer=object(), + tokenizer=object(), # type: ignore[arg-type] ) + assert params.structured_outputs is not None assert params.structured_outputs._backend == expected_backend diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 88015c6a53..dbef99b353 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -31,24 +31,37 @@ SILENT_GROUPS = [ "tests/compile/correctness_e2e", "tests/compile/fullgraph", "tests/compile/fusions_e2e", + "tests/compile/passes", "tests/config", "tests/entrypoints/generate", + "tests/entrypoints/serve", "tests/entrypoints/tool_parsers", + "tests/entrypoints/unit_tests", "tests/entrypoints/weight_transfer", "tests/kernels/core", "tests/kernels/mamba", + "tests/kernels/quantization", + "tests/model_executor/layers", "tests/models/language", "tests/models/quantization", "tests/plugins/bge_m3_sparse_plugin", "tests/plugins/prithvi_io_processor_plugin", "tests/plugins/vllm_add_dummy_platform", "tests/plugins/vllm_add_dummy_stat_logger", - "tests/plugins_tests/gguf", - "tests/plugins_tests/lora_resolvers", + "tests/plugins_tests", + "tests/samplers", "tests/spec_decode", + "tests/tokenizers_", + "tests/tool_use", "tests/transformers_utils", + "tests/utils_", + "tests/v1/cudagraph", + "tests/v1/determinism", "tests/v1/distributed", + "tests/v1/executor", "tests/v1/shutdown", + "tests/v1/simple_kv_offload", + "tests/v1/structured_output", ] # After fixing errors resulting from changing follow_imports @@ -57,57 +70,42 @@ SEPARATE_GROUPS = [ "tests", "tests/benchmarks", "tests/compile", - "tests/compile/passes", "tests/distributed", "tests/entrypoints/anthropic", "tests/entrypoints/llm", "tests/entrypoints/multimodal", "tests/entrypoints/openai", "tests/entrypoints/pooling", - "tests/entrypoints/serve", "tests/entrypoints/speech_to_text", - "tests/entrypoints/unit_tests", "tests/kernels", "tests/kernels/attention", "tests/kernels/helion", "tests/kernels/moe", - "tests/kernels/quantization", "tests/lora", "tests/model_executor", - "tests/model_executor/layers", "tests/model_executor/model_loader", "tests/models", "tests/models/test_initialization.py", "tests/models/multimodal", "tests/multimodal", "tests/parser", - "tests/plugins_tests", "tests/quantization", "tests/reasoning", "tests/renderers", - "tests/samplers", - "tests/tokenizers_", "tests/tool_parsers", - "tests/tool_use", - "tests/utils_", "tests/v1", "tests/v1/attention", "tests/v1/core", - "tests/v1/cudagraph", - "tests/v1/determinism", "tests/v1/e2e", "tests/v1/ec_connector", "tests/v1/engine", - "tests/v1/executor", "tests/v1/kv_connector", "tests/v1/kv_offload", "tests/v1/logits_processors", "tests/v1/metrics", "tests/v1/sample", - "tests/v1/simple_kv_offload", "tests/v1/spec_decode", "tests/v1/streaming_input", - "tests/v1/structured_output", "tests/v1/worker", ] From faacc13565312d29e9596d182fc808b452a4508e Mon Sep 17 00:00:00 2001 From: Stary Date: Tue, 29 Sep 2026 22:29:04 +0200 Subject: [PATCH 052/248] [Perf][KV Connector][Mooncake] Pack hybrid/MLA KV into coalesced transfer regions (#57952) Signed-off-by: staryxchen Co-authored-by: Cursor Co-authored-by: Yifan Qiao --- .../unit/test_mooncake_connector.py | 1030 ++++++++++++++--- .../test_mooncake_connector_hybrid_mamba.py | 2 + .../v1/mooncake/mooncake_connector.py | 559 ++++++++- 3 files changed, 1384 insertions(+), 207 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector.py b/tests/v1/kv_connector/unit/test_mooncake_connector.py index be70a46bf7..c306103288 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector.py @@ -23,6 +23,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake import ( mooncake_utils, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import ( + _SHARED_REGION_GROUP_ID, KVConnectorRole, MooncakeConnector, MooncakeConnectorMetadata, @@ -34,7 +35,11 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector im SendBlockMeta, TransferRegion, _align_transfer_regions, + _block_ids_for_region, + _coalesce_contiguous_transfer_regions, _compute_sender_transfer_plan, + _has_opaque_packed_row, + _pp_mismatch_hides_packed_layers, _validate_asymmetric_region_lengths, get_mooncake_bootstrap_addr, should_launch_bootstrap_server, @@ -48,6 +53,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, KVCacheLayout, + MLAAttentionSpec, ) from vllm.v1.request import RequestStatus @@ -164,6 +170,34 @@ def _make_test_kv_cache_config() -> KVCacheConfig: ) +def _make_packed_mla_kv_cache_config( + num_blocks: int, *, num_groups: int = 1 +) -> KVCacheConfig: + spec = MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=64, + dtype=torch.uint8, + ) + if num_groups == 1: + groups = [ + KVCacheGroupSpec( + ["model.layers.0.mla_attn", "model.layers.1.indexer"], + spec, + ) + ] + else: + groups = [ + KVCacheGroupSpec(["model.layers.0.mla_attn"], spec), + KVCacheGroupSpec(["model.layers.1.indexer"], spec), + ] + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[], + kv_cache_groups=groups, + ) + + class FakeMooncakeWrapper: """Mock Mooncake TransferEngine for unit testing environments.""" @@ -235,6 +269,37 @@ def test_align_transfer_regions_uses_layer_name_occurrences(): assert [r.base_addr for r in aligned_local] == [0x1000, 0x1100] assert [r.base_addr for r in aligned_remote] == [0xB000, 0xB100] + # Hetero PP with no shared layers is an empty success. Same PP fails closed. + p_regions = [ + _region( + 0x1000, + block_len=2048, + kv_block_len=1024, + row_offset=0, + layer_name="model.layers.2.mla_attn", + layer_index=2, + ) + ] + early_d = [ + _region( + 0xA000, + block_len=2048, + kv_block_len=1024, + row_offset=0, + layer_name="model.layers.0.mla_attn", + layer_index=0, + ) + ] + none_p, none_d, none_err = _align_transfer_regions( + p_regions, early_d, allow_partial_layers=True + ) + assert none_err is None + assert none_p == [] and none_d == [] + _, _, mismatch_err = _align_transfer_regions( + p_regions, early_d, allow_partial_layers=False + ) + assert mismatch_err is not None + @pytest.mark.asyncio async def test_build_transfer_params_separates_prefill_pp_layers(): @@ -323,18 +388,7 @@ async def test_build_transfer_params_separates_prefill_pp_layers(): local_block_ids=[[10, 11]], ready=asyncio.Event(), ) - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=1, - remote_tp_rank=0, - req_blocks={"d-req-pp": (transfer_id, [[20, 21]])}, - kv_caches_base_addr=[region.base_addr for region in remote_regions], - block_lens=[region.block_len for region in remote_regions], - kv_block_lens=[region.kv_block_len for region in remote_regions], - registered_layer_names=[region.layer_name for region in remote_regions], - registered_layer_indices=[region.layer_index for region in remote_regions], - ) + xfer_meta = _xfer_meta(remote_regions, {"d-req-pp": (transfer_id, [[20, 21]])}) for pp_rank, local_regions in producer_pp_regions.items(): aligned_local, aligned_remote, err = _align_transfer_regions( @@ -411,20 +465,24 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata( prefill_worker.reqs_need_send[transfer_id] = send_meta send_meta.ready.set() - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=1, - remote_tp_rank=0, - req_blocks={"d-req-layer-align": (transfer_id, [[20]])}, - kv_caches_base_addr=[0xA000, 0xB000], - block_lens=[block_len, block_len], - kv_block_lens=[block_len, block_len], - registered_layer_names=[ - "model.layers.0.self_attn", - "model.layers.1.self_attn", + xfer_meta = _xfer_meta( + [ + _region( + 0xA000, + block_len=block_len, + row_offset=-1, + layer_name="model.layers.0.self_attn", + layer_index=0, + ), + _region( + 0xB000, + block_len=block_len, + row_offset=-1, + layer_name="model.layers.1.self_attn", + layer_index=1, + ), ], - registered_layer_indices=[0, 1], + {"d-req-layer-align": (transfer_id, [[20]])}, ) mock_socket = AsyncMock(spec=zmq.asyncio.Socket) mock_socket.send_multipart = AsyncMock() @@ -1093,6 +1151,81 @@ def patch_worker_dependencies(): } +def _region( + base: int, + *, + block_len: int = 256, + kv_block_len: int | None = None, + row_offset: int | None = None, + layer_name: str = "layer", + layer_index: int = 0, + group_index: int = 0, + shared_group_ids: tuple[int, ...] = (), +) -> TransferRegion: + """One transfer slice. `row_offset` defaults to `base` for in-row addresses.""" + return TransferRegion( + layer_name=layer_name, + layer_index=layer_index, + base_addr=base, + block_len=block_len, + kv_block_len=block_len if kv_block_len is None else kv_block_len, + group_index=group_index, + shared_group_ids=shared_group_ids, + row_offset=base if row_offset is None else row_offset, + ) + + +def _xfer_meta( + regions: list[TransferRegion], + req_blocks: dict, + **overrides, +) -> MooncakeXferMetadata: + """Handshake metadata derived from the regions under test.""" + fields = dict( + remote_hostname="consumer-host", + remote_port=54321, + remote_tp_size=1, + remote_tp_rank=0, + req_blocks=req_blocks, + kv_caches_base_addr=[region.base_addr for region in regions], + block_lens=[region.block_len for region in regions], + kv_block_lens=[region.kv_block_len for region in regions], + registered_layer_names=[region.layer_name for region in regions], + registered_layer_indices=[region.layer_index for region in regions], + registered_group_indices=[region.group_index for region in regions], + registered_shared_group_ids=[ + list(region.shared_group_ids) for region in regions + ], + registered_row_offsets=[region.row_offset for region in regions], + ) + fields.update(overrides) + return MooncakeXferMetadata(**fields) + + +@contextlib.contextmanager +def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"): + """Build a worker whose engine registration and sender thread are patched.""" + vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role) + connector_mod = ( + "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector" + ) + with ( + set_current_vllm_config(vllm_config), + patch_worker_dependencies(), + patch(f"{connector_mod}.threading.Event"), + patch(f"{connector_mod}.threading.Thread") as mock_thread, + ): + connector = MooncakeConnector( + vllm_config, KVConnectorRole.WORKER, kv_cache_config + ) + worker = connector.connector_worker + mock_thread.return_value.is_alive.return_value = False + with patch.object( + worker.engine, "batch_register_memory", return_value=0 + ) as reg: + yield worker, reg + + @pytest.mark.asyncio @pytest.mark.parametrize( ("local_pp_size", "local_pp_rank", "expected_addrs"), @@ -1211,18 +1344,19 @@ async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers(): patch.object(worker, "_send_blocks", return_value=0) as send, ): for pp_rank in range(2): - metadata = MooncakeXferMetadata( + metadata = _xfer_meta( + [ + _region( + 0x2000, + row_offset=-1, + layer_name=f"model.layers.{pp_rank}.self_attn", + layer_index=pp_rank, + ) + ], + {"d-req": ("transfer", [[2]])}, remote_hostname="consumer", remote_port=1234 + pp_rank, - remote_tp_size=1, - remote_tp_rank=0, remote_pp_size=2, - req_blocks={"d-req": ("transfer", [[2]])}, - kv_caches_base_addr=[0x2000], - block_lens=[256], - kv_block_lens=[256], - registered_layer_names=[f"model.layers.{pp_rank}.self_attn"], - registered_layer_indices=[pp_rank], ) await worker.send_kv_to_decode(b"consumer", sock, metadata) response = worker._xfer_resp_decoder.decode( @@ -1286,17 +1420,17 @@ async def test_kv_producer(monkeypatch): send_meta.ready.set() # Remote consumer request metadata - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=1, - remote_tp_rank=0, - req_blocks={"d-req-1": (transfer_id, [[20, 21]])}, - kv_caches_base_addr=[0x2000], - block_lens=[block_len], - kv_block_lens=[block_len], - registered_layer_names=["model.layers.0.self_attn"], - registered_layer_indices=[0], + xfer_meta = _xfer_meta( + [ + _region( + 0x2000, + block_len=block_len, + row_offset=-1, + layer_name="model.layers.0.self_attn", + layer_index=0, + ) + ], + {"d-req-1": (transfer_id, [[20, 21]])}, ) mock_socket = AsyncMock(spec=zmq.asyncio.Socket) @@ -1544,139 +1678,703 @@ async def test_worker_get_finished_timeout(monkeypatch): ) def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool): """Tests the memory registration logic with the underlying Mooncake engine.""" - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_consumer" + spec = FullAttentionSpec( + block_size=16, + num_kv_heads=4, + head_size=64, + dtype=torch.float16, + num_head_slots=2 if separate_kv_head_groups else None, + state_content_bytes=4 * 64 * 2 if separate_kv_head_groups else None, ) - - with ( - set_current_vllm_config(vllm_config), - patch_worker_dependencies(), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Event" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Thread" - ) as mock_thread, - ): - connector = MooncakeConnector( - vllm_config, - KVConnectorRole.WORKER, - _make_test_kv_cache_config(), - ) - worker = connector.connector_worker - mock_thread.return_value.is_alive.return_value = False - - spec = FullAttentionSpec( - block_size=16, - num_kv_heads=4, - head_size=64, - dtype=torch.float16, - num_head_slots=2 if separate_kv_head_groups else None, - state_content_bytes=4 * 64 * 2 if separate_kv_head_groups else None, - ) - layer_names = [ - "model.layers.0.self_attn", - "model.layers.1.self_attn", - ] + layer_names = [ + "model.layers.0.self_attn", + "model.layers.1.self_attn", + ] + with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg): for layer_name in layer_names: worker._layer_specs[layer_name] = spec raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8) tensor1, tensor2 = dense_kv_cache_views(raw, spec, 2, 2, layout) kv_caches = dict(zip(layer_names, (tensor1, tensor2))) + worker.register_kv_caches(kv_caches) - with patch.object( - worker.engine, "batch_register_memory", return_value=0 - ) as mock_batch_register: - connector.register_kv_caches(kv_caches) + reg.assert_called_once() + registered_ptrs, registered_lens = reg.call_args[0] + assert registered_ptrs == [raw.data_ptr()] + assert registered_lens == [raw.nbytes] - mock_batch_register.assert_called_once() - registered_ptrs, registered_lens = mock_batch_register.call_args[0] - assert registered_ptrs == [raw.data_ptr()] - assert registered_lens == [raw.nbytes] + n_blocks = tensor1.shape[0] if tensor1.ndim > 1 else 1 + storage_end = raw.data_ptr() + raw.nbytes + for base, stride, kv_len in zip( + worker.kv_caches_base_addr, + worker.block_len_per_layer, + worker.kv_block_len_per_layer, + ): + assert base + (n_blocks - 1) * stride + kv_len <= storage_end - if not layout.is_block_compact: - expected_addrs = [ - cache[:, head_idx].data_ptr() - for cache in (tensor1, tensor2) - for head_idx in range(cache.shape[1]) - ] - head_block_bytes = tensor1.stride(0) * tensor1.element_size() - assert worker.kv_caches_base_addr == expected_addrs - assert worker.block_len_per_layer == [head_block_bytes] * len( - expected_addrs - ) - assert worker.kv_block_len_per_layer == [head_block_bytes] * len( - expected_addrs - ) - assert worker.registered_layer_names == [ - layer_name - for layer_name in layer_names - for _ in range(tensor1.shape[1]) - ] - else: - assert len(worker.block_len_per_layer) == len(kv_caches) - for bl in worker.block_len_per_layer: - assert bl == tensor1.stride(0) * tensor1.element_size() - assert worker.kv_block_len_per_layer == [spec.page_size_bytes] * 2 - assert worker.registered_layer_names == list(kv_caches) - assert worker.registered_layer_indices == [0, 1] + # Expected shape is the layout, not a recomputation of the pack predicate. + kind = { + KVCacheLayout.LBHNC: "page", + KVCacheLayout.BLHNC: "page", + KVCacheLayout.BHLNC: "opaque", + KVCacheLayout.LHBNC: "per_head", + }[layout] + if kind == "opaque": + packed_row = raw.nbytes // n_blocks + assert worker.kv_caches_base_addr == [raw.data_ptr()] + assert worker.block_len_per_layer == [packed_row] + assert worker.kv_block_len_per_layer == [packed_row] + assert worker.registered_layer_names == [layer_names[0]] + elif kind == "per_head": + expected_addrs = [ + cache[:, head_idx].data_ptr() + for cache in (tensor1, tensor2) + for head_idx in range(cache.shape[1]) + ] + head_block_bytes = tensor1.stride(0) * tensor1.element_size() + assert worker.kv_caches_base_addr == expected_addrs + assert worker.block_len_per_layer == [head_block_bytes] * len( + expected_addrs + ) + assert worker.kv_block_len_per_layer == [head_block_bytes] * len( + expected_addrs + ) + assert worker.registered_layer_names == [ + layer_name + for layer_name in layer_names + for _ in range(tensor1.shape[1]) + ] + else: + page_stride = tensor1.stride(0) * tensor1.element_size() + assert len(worker.block_len_per_layer) == len(kv_caches) + assert worker.block_len_per_layer == [page_stride, page_stride] + assert worker.kv_block_len_per_layer == [spec.page_size_bytes] * 2 + assert worker.registered_layer_names == list(kv_caches) + assert worker.registered_layer_indices == [0, 1] + + +def test_register_noncontiguous_packed_allows_matching_pp(): + """Same PP on both sides can still register a non-contiguous packed row. + + A remote PP mismatch is rejected at send time, once the peer PP size is + known. Rejecting every local PP>1 layout at startup is too strict. + """ + spec = FullAttentionSpec( + block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16 + ) + layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"] + with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg): + worker.pp_size = 2 + for layer_name in layer_names: + worker._layer_specs[layer_name] = spec + raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8) + tensor1, tensor2 = dense_kv_cache_views(raw, spec, 2, 2, KVCacheLayout.BHLNC) + assert not tensor1[0].is_contiguous() + worker.register_kv_caches(dict(zip(layer_names, (tensor1, tensor2)))) + assert worker.kv_caches_base_addr == [raw.data_ptr()] + assert worker.region_row_offsets == [0] def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes(): """Mixed MLA+Eagle caches should register by byte length, not shape.""" - vllm_config = create_vllm_config( - kv_connector="MooncakeConnector", kv_role="kv_consumer" - ) - - with ( - set_current_vllm_config(vllm_config), - patch_worker_dependencies(), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Event" - ), - patch( - "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Thread" - ) as mock_thread, - ): - connector = MooncakeConnector( - vllm_config, - KVConnectorRole.WORKER, - _make_test_kv_cache_config(), - ) - worker = connector.connector_worker - mock_thread.return_value.is_alive.return_value = False - + with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg): worker.use_mla = True worker.transfer_topo.is_mla = True - - # MLA cache tensor: shape[-2] is the block size and each block's - # byte stride matches the cache spec page size. mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16) - # Eagle3/GQA-like cache tensor: shape[-2] is num_kv_heads, not block size. eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16) kv_caches = { "model.layers.0.mla_attn": mla_cache, "model.layers.1.eagle_attn": eagle_cache, } + worker.register_kv_caches(kv_caches) - with patch.object( - worker.engine, "batch_register_memory", return_value=0 - ) as mock_batch_register: - connector.register_kv_caches(kv_caches) + reg.assert_called_once() + registered_ptrs, registered_lens = reg.call_args[0] + assert registered_ptrs == [mla_cache.data_ptr(), eagle_cache.data_ptr()] + assert registered_lens == [mla_cache.nbytes, eagle_cache.nbytes] + assert worker.block_len_per_layer == [ + mla_cache.nbytes // mla_cache.shape[0], + eagle_cache.nbytes // eagle_cache.shape[0], + ] + assert worker.registered_layer_names == [ + "model.layers.0.mla_attn", + "model.layers.1.eagle_attn", + ] + assert worker.registered_layer_indices == [0, 1] + assert len(worker.kv_caches_base_addr) == 2 - mock_batch_register.assert_called_once() - registered_ptrs, registered_lens = mock_batch_register.call_args[0] - assert registered_ptrs == [mla_cache.data_ptr(), eagle_cache.data_ptr()] - assert registered_lens == [mla_cache.nbytes, eagle_cache.nbytes] - assert worker.block_len_per_layer == [ - mla_cache.nbytes // mla_cache.shape[0], - eagle_cache.nbytes // eagle_cache.shape[0], + +def test_packed_and_unpacked_region_lengths_fail_homogeneous_tp_handshake(): + packed = TransferRegion( + layer_name="model.layers.0.mla_attn", + layer_index=0, + base_addr=0x1000, + block_len=2048, + kv_block_len=2048, + group_index=0, + ) + unpacked = TransferRegion( + layer_name="model.layers.0.mla_attn", + layer_index=0, + base_addr=0x1000, + block_len=64, + kv_block_len=64, + group_index=0, + ) + err = _validate_asymmetric_region_lengths( + local_regions=[packed], + remote_regions=[unpacked], + local_tp_size=1, + remote_tp_size=1, + producer_cache_replicated=False, + ) + assert err is not None + assert "length mismatch" in err + + count_err = _validate_asymmetric_region_lengths( + local_regions=[packed], + remote_regions=[unpacked, unpacked], + local_tp_size=1, + remote_tp_size=1, + producer_cache_replicated=False, + ) + assert count_err is not None + assert "region counts" in count_err + + +@pytest.mark.parametrize( + ("num_groups", "expected_group_index"), + [(1, 0), (2, _SHARED_REGION_GROUP_ID)], +) +def test_register_kv_caches_collapses_shared_mla_storage( + num_groups, expected_group_index +): + """Shared backing collapses to one packed region; two groups mark it -1.""" + num_blocks = 4 + packed_row = 256 + with mooncake_register_worker( + _make_packed_mla_kv_cache_config(num_blocks, num_groups=num_groups) + ) as (worker, reg): + backing = torch.zeros((num_blocks, packed_row), dtype=torch.uint8) + worker.register_kv_caches( + { + "model.layers.0.mla_attn": backing, + "model.layers.1.indexer": backing, + } + ) + + reg.assert_called_once() + registered_ptrs, registered_lens = reg.call_args[0] + assert registered_ptrs == [backing.data_ptr()] + assert registered_lens == [backing.untyped_storage().nbytes()] + assert worker.kv_caches_base_addr == [backing.data_ptr()] + assert worker.block_len_per_layer == [packed_row] + assert worker.kv_block_len_per_layer == [packed_row] + assert worker.registered_layer_names == ["model.layers.0.mla_attn"] + assert worker.registered_group_indices == [expected_group_index] + if expected_group_index == _SHARED_REGION_GROUP_ID: + assert worker.region_shared_groups == [(0, 1)] + + +def test_block_ids_for_region_flattens_shared_groups(): + ids = [[10, 11], [12]] + assert _block_ids_for_region(ids, 0) == [10, 11] + assert _block_ids_for_region(ids, 1) == [12] + assert _block_ids_for_region(ids, _SHARED_REGION_GROUP_ID, (0, 1)) == [ + 10, + 11, + 12, + ] + # A group that does not share the allocation stays out. + assert _block_ids_for_region( + [[10, 11], [12], [99]], _SHARED_REGION_GROUP_ID, (0, 1) + ) == [10, 11, 12] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("group_index", "local_ids", "remote_ids", "n_blocks"), + [ + (0, [[10, 11]], [[20, 21]], 2), + (_SHARED_REGION_GROUP_ID, [[10, 11], [12]], [[20, 21], [22]], 3), + ], +) +async def test_build_transfer_params_sends_packed_region_once( + group_index, local_ids, remote_ids, n_blocks +): + """Packed region coalesces contiguous blocks, including shared-group flatten.""" + worker = MooncakeConnectorWorker.__new__(MooncakeConnectorWorker) + worker.async_zmq_ctx = MagicMock() + worker.is_kv_consumer = True + worker.is_kv_producer = True + worker.tp_rank = 0 + worker.tp_size = 1 + worker.use_mla = True + worker.kv_cache_config = _make_packed_mla_kv_cache_config( + num_blocks=4, num_groups=len(local_ids) + ) + worker._physical_blocks_per_logical_kv_block = 1 + worker.transfer_topo = SimpleNamespace( + local_replicates_kv_cache=False, + total_num_kv_heads=1, + ) + + block_len = 256 + shared = (0, 1) if group_index == _SHARED_REGION_GROUP_ID else () + local_region = _region( + 0x1000, + block_len=block_len, + row_offset=0, + layer_name="model.layers.0.mla_attn", + group_index=group_index, + shared_group_ids=shared, + ) + remote_region = _region( + 0xA000, + block_len=block_len, + row_offset=0, + layer_name="model.layers.0.mla_attn", + group_index=group_index, + shared_group_ids=shared, + ) + transfer_id = "xfer-packed" + send_meta = SendBlockMeta( + p_req_id="p-packed", + transfer_id=transfer_id, + local_block_ids=local_ids, + ready=asyncio.Event(), + ) + xfer_meta = _xfer_meta([remote_region], {"d-packed": (transfer_id, remote_ids)}) + + ( + src_ptrs, + dst_ptrs, + lengths, + err_reqs, + err_msg, + ) = await worker._build_transfer_params( + ready_reqs=[("d-packed", send_meta)], + agent_meta=xfer_meta, + local_regions=[local_region], + remote_regions=[remote_region], + ) + + assert err_reqs == [] + assert err_msg is None + assert src_ptrs == [0x1000 + 10 * block_len] + assert dst_ptrs == [0xA000 + 20 * block_len] + assert lengths == [n_blocks * block_len] + + +def test_coalesce_promotes_padding_only_for_a_full_row(): + """A full-row merge includes the padding tail; a partial row does not.""" + page, row = 1024, 2560 + local = [ + _region(0, block_len=row, kv_block_len=page, row_offset=0), + _region(page, block_len=row, kv_block_len=page, row_offset=page), + ] + remote = [ + _region(100, block_len=row, kv_block_len=page, row_offset=0), + _region(100 + page, block_len=row, kv_block_len=page, row_offset=page), + ] + full_local, full_remote = _coalesce_contiguous_transfer_regions( + local, remote, promote_full_row=True + ) + assert len(full_local) == 1 + assert full_local[0].kv_block_len == full_remote[0].kv_block_len == row + + partial_local, _ = _coalesce_contiguous_transfer_regions(local, remote) + assert partial_local[0].kv_block_len == 2 * page + + +def test_coalesce_spans_padding_between_pages(): + """Unpadded pages merge across the padding gap without a full-row lift.""" + row = 2560 + merged, _ = _coalesce_contiguous_transfer_regions( + [ + _region(0, block_len=row, kv_block_len=1000), + _region(1024, block_len=row, kv_block_len=1000), + ], + [ + _region(5000, block_len=row, kv_block_len=1000, row_offset=0), + _region(6024, block_len=row, kv_block_len=1000, row_offset=1024), + ], + ) + assert len(merged) == 1 + assert merged[0].kv_block_len == 2024 + assert merged[0].row_offset == 0 + + +def test_coalesce_promotes_each_group_that_starts_at_row_zero(): + """Groups overlay one row, so each row-start run is its own full-row copy.""" + page, gap, row = 1000, 1024, 4096 + + def side(origin: int) -> list[TransferRegion]: + return [ + _region( + origin + base, + block_len=row, + kv_block_len=page, + row_offset=offset, + layer_name=f"group-{group}", + layer_index=group, + group_index=group, + ) + for group, base, offset in ( + (0, 0, 0), + (0, gap, gap), + (1, 0, 0), + (1, gap, gap), + ) ] - assert worker.registered_layer_names == [ - "model.layers.0.mla_attn", - "model.layers.1.eagle_attn", - ] - assert worker.registered_layer_indices == [0, 1] + + merged_local, merged_remote = _coalesce_contiguous_transfer_regions( + side(0), side(8000), promote_full_row=True + ) + assert [region.group_index for region in merged_local] == [0, 1] + assert [region.kv_block_len for region in merged_local] == [row, row] + assert [region.kv_block_len for region in merged_remote] == [row, row] + assert merged_local[1].base_addr == 0 + assert merged_remote[1].base_addr == 8000 + + +def test_coalesce_does_not_promote_a_mid_row_run(): + """A matched PP slice that does not start at row offset 0 stays partial.""" + page, row = 1000, 4096 + merged, _ = _coalesce_contiguous_transfer_regions( + [ + _region( + 1024, block_len=row, kv_block_len=page, row_offset=1024, layer_index=2 + ), + _region( + 2048, block_len=row, kv_block_len=page, row_offset=2048, layer_index=2 + ), + ], + [ + _region( + 9000 + 1024, + block_len=row, + kv_block_len=page, + row_offset=1024, + layer_index=2, + ), + _region( + 9000 + 2048, + block_len=row, + kv_block_len=page, + row_offset=2048, + layer_index=2, + ), + ], + promote_full_row=True, + ) + assert len(merged) == 1 + assert merged[0].row_offset == 1024 + assert merged[0].kv_block_len == 1024 + page + + +def test_opaque_packed_row_is_full_stride_at_offset_zero(): + """Branch B is row-sized at offset 0. A page view at offset 0 is not.""" + assert _has_opaque_packed_row([0], [4096], [4096]) + assert not _has_opaque_packed_row([0, 1024], [1000, 1000], [4096, 4096]) + assert not _has_opaque_packed_row([], [4096], [4096]) + + +def test_pp_mismatch_rejects_opaque_row_in_both_directions(): + """The smaller PP side can be either the producer or the consumer.""" + page_view = dict( + remote_row_offsets=[0, 1024], + remote_kv_block_lens=[1000, 1000], + remote_block_lens=[4096, 4096], + ) + opaque_row = dict( + remote_row_offsets=[0], + remote_kv_block_lens=[4096], + remote_block_lens=[4096], + ) + # P pp=2, D pp=1, local row is opaque. + assert _pp_mismatch_hides_packed_layers( + 2, 1, local_has_opaque_row=True, **page_view + ) + # P pp=1, D pp=2, remote row is opaque. + assert _pp_mismatch_hides_packed_layers( + 1, 2, local_has_opaque_row=False, **opaque_row + ) + # Matching PP still aligns by the shared first layer name. + assert not _pp_mismatch_hides_packed_layers( + 2, 2, local_has_opaque_row=True, **opaque_row + ) + # Hetero PP with page-contiguous views is not this rejection. + assert not _pp_mismatch_hides_packed_layers( + 2, 1, local_has_opaque_row=False, **page_view + ) + + +def test_prepare_transfer_regions_reuses_success_and_error(): + """A peer layout is built once. Re-registering drops the cache.""" + worker, _, _, _ = _register_sliced_packed_mla([0, 1]) + regions = worker._get_transfer_regions( + worker.kv_caches_base_addr, + worker.block_len_per_layer, + worker.kv_block_len_per_layer, + worker.registered_layer_names, + worker.registered_layer_indices, + worker.registered_group_indices, + [tuple(groups) for groups in worker.region_shared_groups], + worker.region_row_offsets, + ) + meta = _xfer_meta( + regions, + {}, + remote_hostname="peer", + remote_port=1, + remote_tp_size=worker.tp_size, + remote_pp_size=worker.pp_size, + ) + calls = 0 + original = worker._get_transfer_regions + + def counting(*args, **kwargs): + nonlocal calls + calls += 1 + return original(*args, **kwargs) + + worker._get_transfer_regions = counting + prepared = worker._prepare_transfer_regions(meta) + assert prepared[2] is None + assert calls == 2 + assert worker._prepare_transfer_regions(meta) == prepared + assert calls == 2 + + mismatch = _xfer_meta( + [ + _region( + 0x2000, + block_len=1024, + row_offset=0, + layer_name="model.layers.9.mla_attn", + layer_index=9, + ) + ], + {}, + remote_hostname="peer", + remote_port=2, + remote_tp_size=worker.tp_size, + remote_pp_size=worker.pp_size, + ) + failed = worker._prepare_transfer_regions(mismatch) + assert failed[2] is not None + assert calls == 4 + assert worker._prepare_transfer_regions(mismatch)[2] == failed[2] + assert calls == 4 + + page = 1024 + backing = torch.zeros((4, page * 2), dtype=torch.uint8) + kv_caches = { + "model.layers.0.mla_attn": backing[:, :page], + "model.layers.1.mla_attn": backing[:, page:], + } + worker.is_kv_consumer = True + # The first register built a producer. Its thread patch is already gone, + # so stay on the consumer path and return before the sender thread starts. + with patch.object(worker.engine, "batch_register_memory", return_value=0): + worker.register_kv_caches(kv_caches) + assert worker._prepared_transfer_regions == {} + worker._prepare_transfer_regions(meta) + assert calls == 6 + + +def test_coalesce_keeps_a_run_inside_its_starting_row(): + """The row bound is the run start, not the previous slice.""" + row = 2500 + slices = [ + _region(base, block_len=row, kv_block_len=1000) for base in (0, 1000, 2000) + ] + merged, _ = _coalesce_contiguous_transfer_regions(slices, slices) + assert len(merged) == 2 + assert merged[0].kv_block_len == 2000 + assert merged[1].base_addr == 2000 + assert merged[1].kv_block_len == 1000 + + +def _layer_name(idx: int) -> str: + return f"model.layers.{idx}.mla_attn" + + +def _register_sliced_packed_mla( + layer_idxs: list[int], *, page: int = 1024, num_blocks: int = 4 +): + """Register column slices of one MLA packed row. Returns worker fields.""" + spec = MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=64, + dtype=torch.uint8, + ) + kv_cache_config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec([_layer_name(i) for i in layer_idxs], spec), + ], + ) + backing = torch.zeros((num_blocks, page * len(layer_idxs)), dtype=torch.uint8) + kv_caches = { + _layer_name(idx): backing[:, i * page : (i + 1) * page] + for i, idx in enumerate(layer_idxs) + } + with mooncake_register_worker(kv_cache_config, kv_role="kv_producer") as ( + worker, + _, + ): + worker.register_kv_caches(kv_caches) + regions = worker._get_transfer_regions( + worker.kv_caches_base_addr, + worker.block_len_per_layer, + worker.kv_block_len_per_layer, + worker.registered_layer_names, + worker.registered_layer_indices, + worker.registered_group_indices, + row_offsets=worker.region_row_offsets, + ) + return worker, kv_cache_config, regions, backing.data_ptr() + + +def test_sliced_packed_mla_registers_each_layer_view(): + """Each column of a packed MLA row stays its own view, in row order.""" + page = 1024 + layer_idxs = [0, 1] + with mooncake_register_worker( + KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[], + kv_cache_groups=[ + KVCacheGroupSpec( + [_layer_name(i) for i in layer_idxs], + MLAAttentionSpec( + block_size=16, + num_kv_heads=1, + head_size=64, + dtype=torch.uint8, + ), + ) + ], + ), + kv_role="kv_producer", + ) as (worker, reg): + backing = torch.zeros((4, page * len(layer_idxs)), dtype=torch.uint8) + worker.register_kv_caches( + { + _layer_name(idx): backing[:, i * page : (i + 1) * page] + for i, idx in enumerate(layer_idxs) + } + ) + reg.assert_called_once() + assert reg.call_args[0][0] == [backing.data_ptr()] + assert worker.region_row_offsets == [0, page] + + +async def _copy_packed_slices( + p_layers: list[int], + d_layers: list[int], + *, + allow_partial: bool, + local_ids: list[list[int]], + remote_ids: list[list[int]], + page: int = 1024, +): + p_worker, p_config, p_regions, p_base = _register_sliced_packed_mla( + p_layers, page=page + ) + _, _, d_regions, d_base = _register_sliced_packed_mla(d_layers, page=page) + aligned_p, aligned_d, err = _align_transfer_regions( + p_regions, d_regions, allow_partial_layers=allow_partial + ) + assert err is None + coalesced_p, coalesced_d = _coalesce_contiguous_transfer_regions( + aligned_p, aligned_d + ) + transfer_id = "xfer-packed-slices" + send_meta = SendBlockMeta( + p_req_id="p-slices", + transfer_id=transfer_id, + local_block_ids=local_ids, + ready=asyncio.Event(), + ) + p_worker.kv_cache_config = p_config + src, dst, lengths, err_reqs, err_msg = await p_worker._build_transfer_params( + ready_reqs=[("d-slices", send_meta)], + agent_meta=_xfer_meta(coalesced_d, {"d-slices": (transfer_id, remote_ids)}), + local_regions=coalesced_p, + remote_regions=coalesced_d, + ) + return SimpleNamespace( + p_regions=p_regions, + d_regions=d_regions, + coalesced_p=coalesced_p, + coalesced_d=coalesced_d, + p_base=p_base, + d_base=d_base, + src=src, + dst=dst, + lengths=lengths, + err_reqs=err_reqs, + err_msg=err_msg, + ) + + +@pytest.mark.asyncio +async def test_packed_hetero_pp_keeps_shared_layers(): + """P-PP4/D-PP2 copies the shared MLA span, not either side's full row.""" + page = 1024 + copied = await _copy_packed_slices( + [2, 3], + [0, 1, 2, 3], + allow_partial=True, + local_ids=[[10, 11]], + remote_ids=[[10, 11]], + page=page, + ) + assert [region.layer_index for region in copied.p_regions] == [2, 3] + assert [region.kv_block_len for region in copied.p_regions] == [page, page] + assert copied.p_regions[0].block_len == 2 * page + assert copied.d_regions[0].block_len == 4 * page + assert len(copied.coalesced_p) == 1 + assert copied.coalesced_p[0].kv_block_len == 2 * page + assert copied.coalesced_d[0].base_addr == copied.d_base + 2 * page + assert copied.err_reqs == [] + assert copied.err_msg is None + assert copied.lengths == [2 * page, 2 * page] + assert copied.src == [ + copied.p_base + 10 * (2 * page), + copied.p_base + 11 * (2 * page), + ] + assert copied.dst == [ + copied.d_base + 2 * page + 10 * (4 * page), + copied.d_base + 2 * page + 11 * (4 * page), + ] + + +@pytest.mark.asyncio +async def test_packed_matching_layers_coalesce_to_one_copy(): + """Identical packed layer sets still emit one copy for contiguous blocks.""" + page = 1024 + copied = await _copy_packed_slices( + [0, 1], + [0, 1], + allow_partial=False, + local_ids=[[10, 11]], + remote_ids=[[20, 21]], + page=page, + ) + assert copied.err_reqs == [] + assert copied.err_msg is None + assert len(copied.coalesced_p) == 1 + assert copied.coalesced_p[0].kv_block_len == copied.coalesced_p[0].block_len + block_len = 2 * page + assert copied.src == [copied.p_base + 10 * block_len] + assert copied.dst == [copied.d_base + 20 * block_len] + assert copied.lengths == [2 * block_len] @pytest.mark.asyncio @@ -1760,22 +2458,24 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size): ) as mock_send_blocks: for d_rank in target_d_ranks: remote_block_ids = d_rank_remote_blocks[d_rank] - xfer_meta = MooncakeXferMetadata( - remote_hostname="consumer-host", - remote_port=54321, - remote_tp_size=d_tp_size, - remote_tp_rank=d_rank, - req_blocks={ + xfer_meta = _xfer_meta( + [ + _region( + 0x2000, + block_len=remote_block_len, + row_offset=-1, + layer_name="model.layers.0.self_attn", + layer_index=0, + ) + ], + { f"d-req-h1-r{d_rank}": ( transfer_id, remote_block_ids, ) }, - kv_caches_base_addr=[0x2000], - block_lens=[remote_block_len], - kv_block_lens=[remote_block_len], - registered_layer_names=["model.layers.0.self_attn"], - registered_layer_indices=[0], + remote_tp_size=d_tp_size, + remote_tp_rank=d_rank, ) mock_send_blocks.reset_mock() diff --git a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py index 9376745516..23856b4541 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py +++ b/tests/v1/kv_connector/unit/test_mooncake_connector_hybrid_mamba.py @@ -242,6 +242,8 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages(): kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16) worker = object.__new__(MooncakeConnectorWorker) worker.shutdown = noop_shutdown + # __new__ skips __init__, where this cache is created. register_kv_caches clears it. + worker._prepared_transfer_regions = {} worker.use_mla = False worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0) worker.is_kv_consumer = True diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py index e3fb5b95c9..46118f4056 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/mooncake_connector.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import itertools import logging import queue import threading @@ -62,6 +63,8 @@ from vllm.v1.kv_cache_interface import ( KpoolTailSpec, KVCacheSpec, MambaSpec, + MLAAttentionSpec, + SlidingWindowMLASpec, SlidingWindowSpec, ) from vllm.v1.request import RequestStatus @@ -90,6 +93,12 @@ if TYPE_CHECKING: ReqId = str # Internal scheduler request ID TransferId = str # KV transfer coordination ID (shared by P/D) +# Follow NIXL (`nixl/base_worker.py`): HMA packed pages can put more than one +# KV group on a single allocation. group_index=-1 flattens every group's block +# ids onto that region. Hybrid KV manager assigns distinct ids per group, so +# flatten does not copy the same block twice. +_SHARED_REGION_GROUP_ID = -1 + @dataclass(frozen=True) class TransferRegion: @@ -99,6 +108,34 @@ class TransferRegion: block_len: int kv_block_len: int group_index: int = 0 + # Groups that share this allocation. Empty unless group_index is -1. + shared_group_ids: tuple[int, ...] = () + # Byte offset of this view from the start of its block row. + row_offset: int = 0 + + +def _block_ids_for_region( + block_ids_by_group: list[list[int]], + group_index: int, + shared_group_ids: tuple[int, ...] = (), +) -> list[int]: + """Follow NIXL `_block_ids_by_region`. + + -1 flattens only the groups that share this allocation. NIXL can flatten + every group because its packed flag means they all share one storage; + Mooncake can still have a separate Mamba group in the same request. + """ + if group_index == _SHARED_REGION_GROUP_ID: + assert shared_group_ids, "Shared packed region is missing its group ids." + return list( + itertools.chain.from_iterable( + block_ids_by_group[group] for group in shared_group_ids + ) + ) + assert 0 <= group_index < len(block_ids_by_group), ( + "Transfer region references a missing KV group." + ) + return list(block_ids_by_group[group_index]) def _get_tp_ratio(local_tp_size: int, remote_tp_size: int) -> int: @@ -129,6 +166,8 @@ def _expand_transfer_regions( layer_names: list[str], layer_indices: list[int], group_indices: list[int] | None = None, + shared_group_ids: list[tuple[int, ...]] | None = None, + row_offsets: list[int] | None = None, ) -> list[TransferRegion]: """Expand registered KV tensors into the regions transferred by Mooncake.""" assert ( @@ -150,6 +189,14 @@ def _expand_transfer_regions( "Mooncake transfer regions require matching group metadata lengths, " f"got group_indices={len(group_indices)}, layer_names={len(layer_names)}." ) + if shared_group_ids is None: + shared_group_ids = [() for _ in layer_names] + assert len(shared_group_ids) == len(layer_names) + # Missing offsets must not look like row starts, or every run would be + # promoted to the full row. + if row_offsets is None: + row_offsets = [-1] * len(layer_names) + assert len(row_offsets) == len(layer_names) regions: list[TransferRegion] = [] for ( base_addr, @@ -158,6 +205,8 @@ def _expand_transfer_regions( layer_name, layer_index, group_index, + region_groups, + row_offset, ) in zip( base_addrs, block_lens, @@ -165,6 +214,8 @@ def _expand_transfer_regions( layer_names, layer_indices, group_indices, + shared_group_ids, + row_offsets, ): regions.append( TransferRegion( @@ -174,6 +225,8 @@ def _expand_transfer_regions( block_len=block_len, kv_block_len=kv_block_len, group_index=group_index, + shared_group_ids=region_groups, + row_offset=row_offset, ) ) return regions @@ -377,12 +430,168 @@ def _align_transfer_regions( f"{remote_region.group_index}." ), ) + if ( + local_region.shared_group_ids + and remote_region.shared_group_ids + and local_region.shared_group_ids != remote_region.shared_group_ids + ): + return ( + [], + [], + ( + "Mooncake shared-group set mismatch for " + f"{local_region.layer_name}: producer=" + f"{local_region.shared_group_ids}, consumer=" + f"{remote_region.shared_group_ids}." + ), + ) aligned_local.append(local_region) aligned_remote.append(remote_region) return aligned_local, aligned_remote, None +def _coalesce_contiguous_transfer_regions( + local_regions: list[TransferRegion], + remote_regions: list[TransferRegion], + *, + promote_full_row: bool = False, +) -> tuple[list[TransferRegion], list[TransferRegion]]: + """Merge aligned packed slices that are adjacent on both sides. + + Page-contiguous views stay addressable by layer name for hetero PP + (#56033). KV groups overlay one row and must stay separate: each group + owns a different block-id space. A run that starts at row offset 0 is + lifted to the full row so consecutive blocks can coalesce. + """ + if len(local_regions) != len(remote_regions): + return local_regions, remote_regions + + def adjacent( + run_start: TransferRegion, prev: TransferRegion, nxt: TransferRegion + ) -> bool: + # Page padding sits between unpadded payloads. Allow that gap, but + # keep the whole run inside the row that `run_start` began. Anchoring + # on `prev` would slide the bound forward as the run grows. + return ( + prev.group_index == nxt.group_index + and prev.shared_group_ids == nxt.shared_group_ids + and prev.block_len == nxt.block_len + and prev.base_addr + prev.kv_block_len <= nxt.base_addr + and nxt.base_addr + nxt.kv_block_len + <= run_start.base_addr + run_start.block_len + ) + + def span(region: TransferRegion, kv_block_len: int) -> TransferRegion: + return TransferRegion( + layer_name=region.layer_name, + layer_index=region.layer_index, + base_addr=region.base_addr, + block_len=region.block_len, + kv_block_len=kv_block_len, + group_index=region.group_index, + shared_group_ids=region.shared_group_ids, + row_offset=region.row_offset, + ) + + def promote( + merged_local: list[TransferRegion], merged_remote: list[TransferRegion] + ) -> tuple[list[TransferRegion], list[TransferRegion]]: + if not promote_full_row: + return merged_local, merged_remote + # Each KV group overlays the row from offset 0. Promoting that run to + # the full row copies bytes the group's block ids exclusively own. + # Equal block_len is the row-structure check: both peers packed the + # same stride, so row_offset does not need a second compare in align. + promoted_local: list[TransferRegion] = [] + promoted_remote: list[TransferRegion] = [] + for local_region, remote_region in zip(merged_local, merged_remote): + if ( + local_region.row_offset == 0 + and remote_region.row_offset == 0 + and local_region.block_len == remote_region.block_len + and local_region.kv_block_len <= local_region.block_len + and remote_region.kv_block_len <= remote_region.block_len + ): + promoted_local.append(span(local_region, local_region.block_len)) + promoted_remote.append(span(remote_region, remote_region.block_len)) + else: + promoted_local.append(local_region) + promoted_remote.append(remote_region) + return promoted_local, promoted_remote + + if len(local_regions) <= 1: + return promote(list(local_regions), list(remote_regions)) + + merged_local: list[TransferRegion] = [] + merged_remote: list[TransferRegion] = [] + run_local = [local_regions[0]] + run_remote = [remote_regions[0]] + + def flush() -> None: + if len(run_local) == 1: + merged_local.append(run_local[0]) + merged_remote.append(run_remote[0]) + return + for merged, run in ((merged_local, run_local), (merged_remote, run_remote)): + span_len = run[-1].base_addr + run[-1].kv_block_len - run[0].base_addr + merged.append(span(run[0], span_len)) + + for local_region, remote_region in zip(local_regions[1:], remote_regions[1:]): + if adjacent(run_local[0], run_local[-1], local_region) and adjacent( + run_remote[0], run_remote[-1], remote_region + ): + run_local.append(local_region) + run_remote.append(remote_region) + continue + flush() + run_local = [local_region] + run_remote = [remote_region] + flush() + return promote(merged_local, merged_remote) + + +def _has_opaque_packed_row( + row_offsets: list[int], kv_block_lens: list[int], block_lens: list[int] +) -> bool: + """Branch B is one row-sized region at offset 0, so later layer names are gone. + + Page-contiguous views keep kv_block_len below the stride until promotion, + and promotion happens after this handshake check. + """ + if ( + not row_offsets + or len(row_offsets) != len(kv_block_lens) + or len(row_offsets) != len(block_lens) + ): + return False + return any( + offset == 0 and length == stride + for offset, length, stride in zip(row_offsets, kv_block_lens, block_lens) + ) + + +def _pp_mismatch_hides_packed_layers( + local_pp_size: int, + remote_pp_size: int, + *, + local_has_opaque_row: bool, + remote_row_offsets: list[int], + remote_kv_block_lens: list[int], + remote_block_lens: list[int], +) -> bool: + """A row that dropped per-layer names cannot be aligned across PP sizes. + + Either side may be the one that collapsed the row. Matching PP still + lines up, because both stages keep the same first layer name. + """ + if local_pp_size == remote_pp_size: + return False + return local_has_opaque_row or _has_opaque_packed_row( + remote_row_offsets, remote_kv_block_lens, remote_block_lens + ) + + def _get_tensor_dense_flag(tensor: torch.Tensor) -> bool | None: is_dense = getattr(tensor, "is_non_overlapping_and_dense", None) if callable(is_dense): @@ -405,6 +614,12 @@ class MooncakeXferMetadata( registered_layer_names: list[str] = msgspec.field(default_factory=list) registered_layer_indices: list[int] = msgspec.field(default_factory=list) registered_group_indices: list[int] = msgspec.field(default_factory=list) + # Parallel to registered regions. Each entry is the groups that share + # that region. Empty means the peer did not send the field. + registered_shared_group_ids: list[list[int]] = msgspec.field(default_factory=list) + # Byte offset of each registered view from its block row. -1 means the + # peer did not send the field, so the run must not be promoted. + registered_row_offsets: list[int] = msgspec.field(default_factory=list) remote_pp_size: int = 1 @@ -999,7 +1214,18 @@ class MooncakeConnectorWorker: self.registered_layer_names: list[str] = [] self.registered_layer_indices: list[int] = [] self.registered_group_indices: list[int] = [] + # Parallel to registered regions. -1 flattens only these groups. + self.region_shared_groups: list[tuple[int, ...]] = [] + # Parallel to registered regions. -1 is not a packed row view. + self.region_row_offsets: list[int] = [] + self.opaque_packed_storages: set[int] = set() self.seen_base_addresses: list[int] = [] + # Aligned regions depend only on the peer's registered layout. + # The third item is an error string when alignment cannot proceed. + self._prepared_transfer_regions: dict[ + tuple, + tuple[list[TransferRegion], list[TransferRegion], str | None], + ] = {} assert (parallel_config := vllm_config.parallel_config) dp_rank = parallel_config.data_parallel_index @@ -1266,39 +1492,35 @@ class MooncakeConnectorWorker: ) await sock.send_multipart((identity, self._encoder.encode(response))) return - local_regions = self._get_transfer_regions( - self.kv_caches_base_addr, - self.block_len_per_layer, - self.kv_block_len_per_layer, - self.registered_layer_names, - self.registered_layer_indices, - self.registered_group_indices, - ) - remote_regions = self._get_transfer_regions( - meta.kv_caches_base_addr, - meta.block_lens, - meta.kv_block_lens, - meta.registered_layer_names, - meta.registered_layer_indices, - meta.registered_group_indices, - ) - local_regions, remote_regions, align_err = _align_transfer_regions( - local_regions, - remote_regions, - allow_partial_layers=( - meta.remote_pp_size > 1 and meta.remote_pp_size != self.pp_size - ), - ) - if align_err is not None: + if _pp_mismatch_hides_packed_layers( + self.pp_size, + meta.remote_pp_size, + local_has_opaque_row=bool(self.opaque_packed_storages), + remote_row_offsets=meta.registered_row_offsets, + remote_kv_block_lens=meta.kv_block_lens, + remote_block_lens=meta.block_lens, + ): + msg = ( + "Mooncake non-page-contiguous packed regions keep only the " + "first layer name, so they cannot be aligned across PP sizes " + f"{self.pp_size} and {meta.remote_pp_size}." + ) + logger.error(msg) response = MooncakeXferResponse( status=MooncakeXferResponseStatus.ERROR, - err_msg=align_err, + err_msg=msg, ) await sock.send_multipart((identity, self._encoder.encode(response))) return - validation_err = self._validate_head_resharding_layout( - meta.remote_tp_size, local_regions - ) or _validate_asymmetric_region_lengths( + local_regions, remote_regions, prep_err = self._prepare_transfer_regions(meta) + if prep_err is not None: + response = MooncakeXferResponse( + status=MooncakeXferResponseStatus.ERROR, + err_msg=prep_err, + ) + await sock.send_multipart((identity, self._encoder.encode(response))) + return + validation_err = _validate_asymmetric_region_lengths( local_regions=local_regions, remote_regions=remote_regions, local_tp_size=self.tp_size, @@ -1600,12 +1822,16 @@ class MooncakeConnectorWorker: "Aligned Mooncake transfer regions must belong to the same " "KV group." ) - group_index = local_region.group_index - assert group_index < len(local_block_ids_by_group), ( - "Transfer region references a missing KV group." + local_block_ids = _block_ids_for_region( + local_block_ids_by_group, + local_region.group_index, + local_region.shared_group_ids, + ) + remote_block_ids = _block_ids_for_region( + remote_block_ids_by_group, + remote_region.group_index, + local_region.shared_group_ids, ) - local_block_ids = local_block_ids_by_group[group_index] - remote_block_ids = remote_block_ids_by_group[group_index] if not local_block_ids: continue @@ -1738,6 +1964,17 @@ class MooncakeConnectorWorker: self.registered_layer_names = [] self.registered_layer_indices = [] self.registered_group_indices = [] + self.region_shared_groups = [] + self.region_row_offsets = [] + self.opaque_packed_storages = set() + self._prepared_transfer_regions.clear() + + packed_storage_to_region: dict[int, int] = {} + packed_view_to_region: dict[tuple[int, int, int], int] = {} + collapsed_views = 0 + num_blocks = ( + self.kv_cache_config.num_blocks * self._physical_blocks_per_logical_kv_block + ) for layer_name, cache in kv_caches.items(): layer_index = extract_layer_index(layer_name) @@ -1751,6 +1988,104 @@ class MooncakeConnectorWorker: # One raw page tensor per layer; for Mamba that page holds all the # recurrent states, unpacked only when binding the cache for execution. self._log_debug_cache_registration(layer_name, cache) + group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name] + storage = cache.untyped_storage() + storage_addr = storage.data_ptr() + if storage_addr not in seen_storage_ptrs: + seen_storage_ptrs.add(storage_addr) + kv_data_ptrs.append(storage_addr) + kv_data_lens.append(storage.nbytes()) + + is_mla_region = isinstance( + layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec) + ) + tensor_blocks = cache.shape[0] if cache.ndim > 1 else num_blocks + if cache.ndim == 1: + # A flat buffer is block-major only when it is an integral run + # of this layer's pages. NIXL uses the same remainder check. + hnc_contiguous = False + page = ( + layer_spec.page_size_bytes + if isinstance(layer_spec, AttentionSpec) + else 0 + ) + remainder = ( + storage.nbytes() - tensor_blocks * page + if tensor_blocks > 0 and page > 0 + else -1 + ) + if 0 <= remainder < page: + block_stride = page + storage_is_block_major = True + else: + block_stride = 0 + storage_is_block_major = False + else: + block_stride = cache.stride(0) * cache.element_size() + storage_is_block_major = ( + tensor_blocks > 0 + and tensor_blocks * block_stride == storage.nbytes() + ) + hnc_contiguous = ( + cache.ndim == 4 + and cache.stride(2) == cache.shape[3] + and cache.stride(1) == cache.shape[2] * cache.shape[3] + ) + # Follow NIXL packed pages. Page-contiguous slices stay per layer so + # hetero PP can align by name (#56033). A non-contiguous row cannot + # name a sub-span, so it is one region anchored at the allocation. + use_packed_row = ( + not isinstance(layer_spec, (MambaSpec, KpoolTailSpec)) + and storage_is_block_major + and block_stride > 0 + and (is_mla_region or not hnc_contiguous) + ) + if use_packed_row: + page_contiguous = ( + cache.ndim > 1 + and cache[0].is_contiguous() + and 0 < cache[0].nbytes <= block_stride + ) + if page_contiguous: + view_addr = cache.data_ptr() + kv_block_len = cache[0].nbytes + else: + # NIXL branch B: length is the row, base is the allocation. + view_addr = storage_addr + kv_block_len = block_stride + self.opaque_packed_storages.add(storage_addr) + # Same PP on both sides still aligns by the first layer name. + # A remote PP mismatch is rejected at send time, once the + # peer's PP size is known. + dedupe_key = (view_addr, block_stride, kv_block_len) + region_idx = packed_view_to_region.get(dedupe_key) + if region_idx is not None: + collapsed_views += 1 + # -1 means these groups share the view. Flatten only them. + groups = list(self.region_shared_groups[region_idx]) + if group_index not in groups: + groups.append(group_index) + self.region_shared_groups[region_idx] = tuple(groups) + if len(self.region_shared_groups[region_idx]) > 1: + self.registered_group_indices[region_idx] = ( + _SHARED_REGION_GROUP_ID + ) + continue + if not page_contiguous: + self.opaque_packed_storages.add(storage_addr) + if storage_addr not in packed_storage_to_region: + packed_storage_to_region[storage_addr] = len(region_base_addresses) + packed_view_to_region[dedupe_key] = len(region_base_addresses) + region_base_addresses.append(view_addr) + self.block_len_per_layer.append(block_stride) + self.kv_block_len_per_layer.append(kv_block_len) + self.registered_layer_names.append(layer_name) + self.registered_layer_indices.append(layer_index) + self.registered_group_indices.append(group_index) + self.region_shared_groups.append((group_index,)) + self.region_row_offsets.append(view_addr - storage_addr) + continue + block_is_contiguous = is_non_overlapping_and_dense(cache[0]) if not block_is_contiguous: # Non-block-compact layouts scatter a block across per-head @@ -1791,15 +2126,10 @@ class MooncakeConnectorWorker: self.kv_block_len_per_layer.append(kv_block_len) self.registered_layer_names.append(layer_name) self.registered_layer_indices.append(layer_index) - self.registered_group_indices.append( - self.kv_cache_config.transfer_group_index_by_layer[layer_name] - ) - storage = cache.untyped_storage() - storage_addr = storage.data_ptr() - if storage_addr not in seen_storage_ptrs: - seen_storage_ptrs.add(storage_addr) - kv_data_ptrs.append(storage_addr) - kv_data_lens.append(storage.nbytes()) + self.registered_group_indices.append(group_index) + self.region_shared_groups.append((group_index,)) + # Not a view into a shared packed row, so do not promote it. + self.region_row_offsets.append(-1) self.kv_caches_base_addr = region_base_addresses self.seen_base_addresses = kv_data_ptrs @@ -1807,6 +2137,18 @@ class MooncakeConnectorWorker: if not kv_data_ptrs: raise RuntimeError("No KV cache tensors were registered with Mooncake.") + if packed_storage_to_region and ( + collapsed_views + or len(packed_view_to_region) != len(packed_storage_to_region) + ): + logger.info( + "Mooncake packed KV xfer: %s layer regions over %s storage " + "regions (%s duplicate views collapsed).", + len(packed_view_to_region), + len(packed_storage_to_region), + collapsed_views, + ) + ret_value = self.engine.batch_register_memory(kv_data_ptrs, kv_data_lens) if ret_value != 0: raise RuntimeError("Mooncake batch memory registration failed.") @@ -1962,6 +2304,10 @@ class MooncakeConnectorWorker: registered_layer_names=self.registered_layer_names, registered_layer_indices=self.registered_layer_indices, registered_group_indices=self.registered_group_indices, + registered_shared_group_ids=[ + list(groups) for groups in self.region_shared_groups + ], + registered_row_offsets=self.region_row_offsets, ) encoded_data = self._encoder.encode(metadata) @@ -2209,6 +2555,131 @@ class MooncakeConnectorWorker: def _producer_cache_is_replicated(self) -> bool: return self.transfer_topo.local_replicates_kv_cache + def _prepare_transfer_regions( + self, meta: MooncakeXferMetadata + ) -> tuple[list[TransferRegion], list[TransferRegion], str | None]: + """Align and coalesce once per peer layout. + + ``send_kv_to_decode`` runs on every scheduler batch. The registered + layout does not change until the next ``register_kv_caches``. + """ + cache_key = ( + meta.remote_hostname, + meta.remote_port, + meta.remote_tp_size, + meta.remote_tp_rank, + meta.remote_pp_size, + tuple(meta.kv_caches_base_addr), + tuple(meta.block_lens), + tuple(meta.kv_block_lens), + tuple(meta.registered_layer_names), + tuple(meta.registered_layer_indices), + tuple(meta.registered_group_indices), + tuple(tuple(groups) for groups in meta.registered_shared_group_ids), + tuple(meta.registered_row_offsets), + ) + cached = self._prepared_transfer_regions.get(cache_key) + if cached is not None: + return cached + + def finish( + local: list[TransferRegion], + remote: list[TransferRegion], + err: str | None, + ) -> tuple[list[TransferRegion], list[TransferRegion], str | None]: + prepared = (local, remote, err) + self._prepared_transfer_regions[cache_key] = prepared + return prepared + + shared_groups: list[tuple[int, ...]] | None = self.region_shared_groups + if len(self.region_shared_groups) != len(self.registered_layer_names): + if _SHARED_REGION_GROUP_ID in self.registered_group_indices: + return finish( + [], + [], + ( + "Mooncake shared-group metadata does not match " + "registered layers, so a shared packed region cannot " + "be transferred." + ), + ) + shared_groups = None + local_rows: list[int] | None = self.region_row_offsets + if len(self.region_row_offsets) != len(self.registered_layer_names): + local_rows = None + local_regions = self._get_transfer_regions( + self.kv_caches_base_addr, + self.block_len_per_layer, + self.kv_block_len_per_layer, + self.registered_layer_names, + self.registered_layer_indices, + self.registered_group_indices, + shared_groups, + local_rows, + ) + remote_shared_groups = [ + tuple(groups) for groups in meta.registered_shared_group_ids + ] + remote_shared: list[tuple[int, ...]] | None = remote_shared_groups + if len(remote_shared_groups) != len(meta.registered_layer_names): + remote_shared = None + remote_rows = meta.registered_row_offsets + remote_row_offsets: list[int] | None = remote_rows + if len(remote_rows) != len(meta.registered_layer_names): + remote_row_offsets = None + remote_regions = self._get_transfer_regions( + meta.kv_caches_base_addr, + meta.block_lens, + meta.kv_block_lens, + meta.registered_layer_names, + meta.registered_layer_indices, + meta.registered_group_indices, + remote_shared, + remote_row_offsets, + ) + pre_align_local = local_regions + pre_align_remote = remote_regions + local_regions, remote_regions, align_err = _align_transfer_regions( + local_regions, + remote_regions, + allow_partial_layers=meta.remote_pp_size != self.pp_size, + ) + if align_err is not None: + return finish([], [], align_err) + # Head checks must see every layer name. Coalesce keeps only the first. + head_err = self._validate_head_resharding_layout( + meta.remote_tp_size, local_regions + ) + if head_err is not None: + return finish([], [], head_err) + # Hetero TP indexes kv_block_len as one head-sharded payload. A merged + # multi-layer span would land that offset on the wrong layer. + # Replicated KV keeps that offset at 0. MLA is replicated in practice; + # gating on use_mla alone would promote and then fail the length check + # for a non-replicated MLA rank with tp_ratio != 1. + tp_ratio = _get_tp_ratio(self.tp_size, meta.remote_tp_size) + if tp_ratio == 1 or self._producer_cache_is_replicated(): + covered_both_sides = len(local_regions) == len(pre_align_local) and len( + remote_regions + ) == len(pre_align_remote) + n_aligned = len(local_regions) + local_regions, remote_regions = _coalesce_contiguous_transfer_regions( + local_regions, + remote_regions, + promote_full_row=covered_both_sides, + ) + logger.info( + "Mooncake packed coalesce: %s aligned regions -> %s transfer regions.", + n_aligned, + len(local_regions), + ) + self._prepared_transfer_regions[cache_key] = ( + local_regions, + remote_regions, + None, + ) + return local_regions, remote_regions, None + def _validate_head_resharding_layout( self, remote_tp_size: int, local_regions: list[TransferRegion] ) -> str | None: @@ -2242,6 +2713,8 @@ class MooncakeConnectorWorker: layer_names: list[str], layer_indices: list[int], group_indices: list[int] | None = None, + shared_group_ids: list[tuple[int, ...]] | None = None, + row_offsets: list[int] | None = None, ) -> list[TransferRegion]: if not group_indices: group_indices = [ @@ -2255,6 +2728,8 @@ class MooncakeConnectorWorker: layer_names=layer_names, layer_indices=layer_indices, group_indices=group_indices, + shared_group_ids=shared_group_ids, + row_offsets=row_offsets, ) def _get_sender_transfer_plan( From 1d331f8341f09636c0182a4b48112f8e4f7d027c Mon Sep 17 00:00:00 2001 From: yzong-rh Date: Tue, 29 Sep 2026 16:40:15 -0400 Subject: [PATCH 053/248] [Bugfix] Suppress HarmonyError Unexpected token while expecting start token 200006 (#59254) Signed-off-by: Yifan Zong --- tests/parser/test_harmony.py | 9 +++++++++ vllm/parser/harmony.py | 10 ++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index d8db7f4057..00294afb88 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -872,6 +872,15 @@ class TestProcessChunk: ("final", "Two"), ] + def test_malformed_token_does_not_raise(self, harmony_parser): + malformed = encode_output( + "<|channel|>analysis<|message|>think<|end|><|return|>" + "<|start|>assistant<|channel|>final<|message|>answer<|return|>" + ) + + result = harmony_parser.process_chunk(malformed) + assert "".join(s.delta for s in result.segments if s.delta) == "thinkanswer" + class TestCountReasoningTokens: def test_matches_process_chunk(self, harmony_parser, gpt_oss_tokenizer): diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 26f311c7e9..6a1399db66 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -335,7 +335,10 @@ class HarmonyParser(DelegatingParser): segments: list[Segment] = [] reasoning_token_count = 0 for token_id in token_ids: - self._harmony_parser.process(token_id) + try: + self._harmony_parser.process(token_id) + except HarmonyError: + continue channel = self._harmony_parser.current_channel recipient = self._normalize_recipient( self._harmony_parser.current_recipient @@ -376,7 +379,10 @@ class HarmonyParser(DelegatingParser): parser = get_streamable_parser_for_assistant() count = 0 for token_id in token_ids: - parser.process(token_id) + try: + parser.process(token_id) + except HarmonyError: + continue recipient = self._normalize_recipient(parser.current_recipient) if self._is_reasoning_token(token_id, parser.current_channel, recipient): count += 1 From d882bddbeab6b4a0d5861dfcb171bf61ce2109d6 Mon Sep 17 00:00:00 2001 From: Jared Wen Date: Wed, 30 Sep 2026 05:00:16 +0800 Subject: [PATCH 054/248] [Bugfix][Mamba] Keep the prompt-end prefill checkpoint under sparse retention (#59146) Signed-off-by: Jared Wen Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Jiangyun Zhu --- .../test_partial_prefix_cache_hits.py | 34 +++++++++++++++++++ vllm/v1/core/single_type_kv_cache_manager.py | 12 +++++++ 2 files changed, 46 insertions(+) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index f1301d9288..3b5fd218a6 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -2531,3 +2531,37 @@ def test_boundary_states_offered_past_prompt_for_resumed_prefill(): offered = [b for _, _, b in drain_boundary_state_offloads(manager).get("0", [])] assert 2 * block_size in offered assert req0.num_prompt_tokens < 2 * block_size + + +def test_prompt_end_checkpoint_survives_sparse_retention_with_coarse_hash(): + """With hash_block_size == mamba block size (no fine-grained matching), the + hybrid coordinator caches the final chunk only up to its last full block, + so the managers see ``num_tokens < num_prompt_tokens`` there too. The + prompt-end checkpoint must still be published under retention_interval=0, + otherwise the next request that extends this prompt misses the prefix. + """ + block_size = 32 + manager = make_full_mamba_manager( + dcp_world_size=1, + hash_block_size=block_size, + full_block_size=block_size, + mamba_block_size=block_size, + num_prefill_checkpoint_blocks=1, + ) + manager.coordinator.retention_interval = 0 + + # 240 tokens: not block aligned, prompt-end checkpoint at 224. + request = make_request("producer", list(range(240)), block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(request) + assert manager.allocate_slots(request, 128, num_computed, computed_blocks) + request.num_computed_tokens = 128 + manager.new_step_starts() + assert manager.allocate_slots(request, 112) is not None + + end_hash = request.block_hashes[224 // block_size - 1] + assert manager.block_pool.get_cached_block(end_hash, [1]) is not None + + # A request extending the prompt resumes from that checkpoint. + follower = make_request("follower", list(range(240)) + [7] * 40, block_size, sha256) + _, num_hit, _ = manager.get_computed_blocks(follower) + assert num_hit == 224 diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 7352f71cb6..707786f17f 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2042,9 +2042,21 @@ class MambaManager(SingleTypeKVCacheManager): blocks = self.req_to_blocks[request.request_id] assert 0 <= checkpoint_idx < len(blocks) checkpoint_block = blocks[checkpoint_idx] + # The prompt's final chunk caches only up to its last full block, + # so ``num_tokens < num_prompt_tokens`` holds there too; its + # checkpoint is the prompt's reusable boundary, not a transient one. + is_prompt_checkpoint = ( + checkpoint_position + == get_mamba_prefill_checkpoint_position( + request.num_prompt_tokens, + hash_block_size, + self.drop_eagle_checkpoint_block, + ) + ) if ( retention_interval == 0 and num_tokens < request.num_prompt_tokens + and not is_prompt_checkpoint and checkpoint_position != request.shared_prefix_boundary ): # retention_interval == 0 keeps this transient checkpoint From 1ee7f78e806b5e9b7476c74edf453b58fe108157 Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Tue, 29 Sep 2026 17:23:27 -0400 Subject: [PATCH 055/248] [Perf] Reduce redundant Triton sampler warmup specializations (#58605) Signed-off-by: mgoin Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/v1/sample/ops/topk_topp_triton.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/v1/sample/ops/topk_topp_triton.py b/vllm/v1/sample/ops/topk_topp_triton.py index 20b636e745..151f641d21 100755 --- a/vllm/v1/sample/ops/topk_topp_triton.py +++ b/vllm/v1/sample/ops/topk_topp_triton.py @@ -103,7 +103,7 @@ def _update_min_larger_stats(data, above_mask, min_larger, num_min_larger, senti return min_larger, num_min_larger -@triton.jit +@triton.jit(do_not_specialize_on_alignment=["BATCH_SIZE"]) def _topk_topp_kernel( LOGITS, LOGITS_STRIDE_0, @@ -1225,7 +1225,7 @@ def _topp_sb_combine( ) -@triton.jit +@triton.jit(do_not_specialize_on_alignment=["ROUND"]) def _topp_sb_step_kernel( LOGITS, LOGITS_STRIDE_0, From 4e7af3b0de4cae4f70056da590f2f3d676223978 Mon Sep 17 00:00:00 2001 From: Ostring24 <163995157+Ostring24@users.noreply.github.com> Date: Wed, 30 Sep 2026 05:33:28 +0800 Subject: [PATCH 056/248] [Bugfix][Frontend] Enforce parallel_tool_calls=false in the required-tool grammar (#50502) Signed-off-by: ostring Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../test_functiongemma_tool_parser.py | 1 + .../test_structural_tag_registry.py | 152 ++++++++++++++++-- tests/tool_use/test_tool_choice_required.py | 67 +++++++- vllm/parser/abstract_parser.py | 18 ++- vllm/tool_parsers/abstract_tool_parser.py | 4 +- vllm/tool_parsers/mimo_tool_parser.py | 17 -- vllm/tool_parsers/structural_tag_registry.py | 30 ++++ vllm/tool_parsers/utils.py | 10 +- 8 files changed, 261 insertions(+), 38 deletions(-) diff --git a/tests/tool_parsers/test_functiongemma_tool_parser.py b/tests/tool_parsers/test_functiongemma_tool_parser.py index d32aba3085..10792c1660 100644 --- a/tests/tool_parsers/test_functiongemma_tool_parser.py +++ b/tests/tool_parsers/test_functiongemma_tool_parser.py @@ -27,6 +27,7 @@ def mock_request(): request = MagicMock(spec=ChatCompletionRequest) request.tools = [] request.tool_choice = "auto" + request.parallel_tool_calls = True return request diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 3f22da17cd..2c4de60a9d 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -1317,27 +1317,153 @@ def test_mimo_strict_compact_xml(policy): ) -@pytest.mark.parametrize("policy", ["auto", "required"]) -def test_mimo_request_limits_parallel_calls(sample_tools_strict, policy): +_DS_CALLS_BEGIN = "<\uff5ctool\u2581calls\u2581begin\uff5c>" +_DS_CALL_BEGIN = "<\uff5ctool\u2581call\u2581begin\uff5c>" +_DS_SEP = "<\uff5ctool\u2581sep\uff5c>" +_DS_CALL_END = "<\uff5ctool\u2581call\u2581end\uff5c>" +_DS_CALLS_END = "<\uff5ctool\u2581calls\u2581end\uff5c>" + +# One entry per tool-call list shape: top-level triggered tags (mimo, qwen3), +# tags with a separator (hermes required), and a calls block wrapping the list +# (deepseek). Each is (tool_parser, list prefix, one call, list suffix). +_SINGLE_CALL_CASES = [ + ( + "mimo", + "", + "Paris" + "", + "", + ), + ( + "hermes", + "", + '\n{"name": "get_weather", "arguments": {"city": "Paris"}}' + "\n", + "", + ), + ( + "qwen3_xml", + "", + "\n\n\nParis\n" + "\n\n", + "", + ), + ( + "deepseek_v31", + _DS_CALLS_BEGIN, + f'{_DS_CALL_BEGIN}get_weather{_DS_SEP}{{"city": "Paris"}}{_DS_CALL_END}', + _DS_CALLS_END, + ), +] + +_SINGLE_CALL_IDS = [case[0] for case in _SINGLE_CALL_CASES] + + +def _tool_calling_grammar( + tool_parser: str, + tools: list[ChatCompletionToolsParam], + request: ChatCompletionRequest | ResponsesRequest, +) -> Grammar | None: from vllm.parser.parser_manager import ParserManager + parser_cls = ParserManager.get_parser( + tool_parser_name=tool_parser, enable_auto_tools=True + ) + adjusted = parser_cls(MagicMock(), tools=tools).adjust_request(request) + structured_outputs = adjusted.structured_outputs + if structured_outputs is None or structured_outputs.structural_tag is None: + return None + return Grammar.from_structural_tag(structured_outputs.structural_tag) + + +@pytest.mark.parametrize("policy", ["auto", "required"]) +@pytest.mark.parametrize( + "tool_parser,prefix,call,suffix", _SINGLE_CALL_CASES, ids=_SINGLE_CALL_IDS +) +def test_parallel_tool_calls_false_limits_grammar_to_one_call( + sample_tools, tool_parser, prefix, call, suffix, policy +): + # Tools are not strict, so auto gets a grammar only because of the limit. request = ChatCompletionRequest( - model="mimo", + model="m", messages=[], - tools=[t.model_dump() for t in sample_tools_strict], + tools=[t.model_dump() for t in sample_tools], tool_choice=policy, parallel_tool_calls=False, ) - parser_cls = ParserManager.get_parser( - tool_parser_name="mimo", enable_auto_tools=True + + grammar = _tool_calling_grammar(tool_parser, sample_tools, request) + + assert grammar is not None + assert _is_grammar_accept_string(grammar, prefix + call + suffix) + assert not _is_grammar_accept_string(grammar, prefix + call * 2 + suffix) + assert not _is_grammar_accept_string(grammar, prefix + call + suffix + "text") + + +@pytest.mark.parametrize( + "tool_parser,prefix,call,suffix", _SINGLE_CALL_CASES, ids=_SINGLE_CALL_IDS +) +def test_parallel_tool_calls_default_keeps_grammar_unlimited( + sample_tools, tool_parser, prefix, call, suffix +): + def request(policy: str) -> ChatCompletionRequest: + return ChatCompletionRequest( + model="m", + messages=[], + tools=[t.model_dump() for t in sample_tools], + tool_choice=policy, + ) + + assert _tool_calling_grammar(tool_parser, sample_tools, request("auto")) is None + grammar = _tool_calling_grammar(tool_parser, sample_tools, request("required")) + assert grammar is not None + assert _is_grammar_accept_string(grammar, prefix + call * 2 + suffix) + + +def test_parallel_tool_calls_false_does_not_enable_calls_with_response_format( + sample_tools, +): + # With the flag unset, auto + response_format + non-strict tools is + # format-only. Setting the flag to false must not make tool calls possible. + request = ChatCompletionRequest( + model="m", + messages=[], + tools=[t.model_dump() for t in sample_tools], + tool_choice="auto", + parallel_tool_calls=False, + response_format={ + "type": "json_schema", + "json_schema": {"name": "answer", "schema": {"type": "object"}}, + }, ) - parser = parser_cls(MagicMock(), tools=sample_tools_strict) - adjusted = parser.adjust_request(request) - grammar = Grammar.from_structural_tag(adjusted.structured_outputs.structural_tag) - call = ( - "北京" - "" + + assert _tool_calling_grammar("hermes", sample_tools, request) is None + + +def test_responses_parallel_tool_calls_false_limits_grammar_to_one_call(): + # The Responses API has no post-hoc filter, so the grammar is the only limit. + request = ResponsesRequest.model_validate( + { + "input": "hi", + "tools": [ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ], + "tool_choice": "required", + "parallel_tool_calls": False, + } ) + _, _, call, _ = _SINGLE_CALL_CASES[1] + + grammar = _tool_calling_grammar("hermes", request.tools, request) + + assert grammar is not None assert _is_grammar_accept_string(grammar, call) assert not _is_grammar_accept_string(grammar, call * 2) - assert not _is_grammar_accept_string(grammar, call + "extra text") diff --git a/tests/tool_use/test_tool_choice_required.py b/tests/tool_use/test_tool_choice_required.py index f2b98eb17b..aaa499da16 100644 --- a/tests/tool_use/test_tool_choice_required.py +++ b/tests/tool_use/test_tool_choice_required.py @@ -9,6 +9,8 @@ from openai.types.responses import FunctionTool, WebSearchTool from pydantic import TypeAdapter from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedFunction, + ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) from vllm.tool_parsers.streaming import extract_required_tool_call_streaming @@ -68,11 +70,18 @@ EXAMPLE_TOOLS = [ def _compile_and_check( - tools: list[ChatCompletionToolsParam], sample_output, should_match: bool + tools: list[ChatCompletionToolsParam], + sample_output, + should_match: bool, + parallel_tool_calls: bool | None = None, ): # self = MagicMock(tool_choice="required", tools=tools) # schema = ChatCompletionRequest._get_json_schema_from_tool(self) - schema = get_json_schema_from_tools(tools=tools, tool_choice="required") + schema = get_json_schema_from_tools( + tools=tools, + tool_choice="required", + parallel_tool_calls=parallel_tool_calls, + ) assert isinstance(schema, dict) # use build_regex_from_schema used in JSONLogitsProcessor to create Guide @@ -393,3 +402,57 @@ class TestNonFunctionToolsSkipped: any_of = schema["items"]["anyOf"] assert len(any_of) == 1 assert any_of[0]["properties"]["name"]["enum"] == ["get_weather"] + + +class TestParallelToolCallsConstraint: + """`parallel_tool_calls=false` must be enforced by the decoding grammar. + + Without a `maxItems` bound the model is free to emit an unbounded run of + tool calls that are only discarded afterwards, wasting the token budget and + risking truncation of the one call the client actually receives.""" + + TOOLS = TypeAdapter(list[ChatCompletionToolsParam]).validate_python(EXAMPLE_TOOLS) + ONE_CALL = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}] + TWO_CALLS = [ + {"name": "get_current_weather", "parameters": {"city": "Vienna"}}, + {"name": "get_current_weather", "parameters": {"city": "Berlin"}}, + ] + + def test_disabled_rejects_a_second_tool_call(self): + _compile_and_check(self.TOOLS, self.ONE_CALL, True, parallel_tool_calls=False) + _compile_and_check(self.TOOLS, self.TWO_CALLS, False, parallel_tool_calls=False) + + @pytest.mark.parametrize("parallel_tool_calls", [True, None]) + def test_enabled_or_unset_still_allows_multiple(self, parallel_tool_calls): + _compile_and_check( + self.TOOLS, + self.TWO_CALLS, + True, + parallel_tool_calls=parallel_tool_calls, + ) + + @pytest.mark.parametrize( + "parallel_tool_calls,expected", + [(False, 1), (True, None), (None, None)], + ) + def test_max_items_bound(self, parallel_tool_calls, expected): + schema = get_json_schema_from_tools( + tools=self.TOOLS, + tool_choice="required", + parallel_tool_calls=parallel_tool_calls, + ) + assert isinstance(schema, dict) + assert schema["minItems"] == 1 + assert schema.get("maxItems") == expected + + def test_forced_named_tool_is_unaffected(self): + # Named tool choice yields a bare parameters object, never an array. + schema = get_json_schema_from_tools( + tools=self.TOOLS, + tool_choice=ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_current_weather") + ), + parallel_tool_calls=False, + ) + assert isinstance(schema, dict) + assert "maxItems" not in schema diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index b15d86fd07..2a348697c7 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -49,7 +49,10 @@ from vllm.tool_parsers.streaming import ( extract_named_tool_call_streaming, extract_required_tool_call_streaming, ) -from vllm.tool_parsers.structural_tag_registry import resolve_tool_strictness +from vllm.tool_parsers.structural_tag_registry import ( + limit_to_single_tool_call, + resolve_tool_strictness, +) from vllm.tool_parsers.tool_strict_level import ToolStrictLevel logger = init_logger(__name__) @@ -496,13 +499,20 @@ class DelegatingParser(Parser): structured_outputs = request.extract_structured_outputs() is_auto = request.tool_choice == "auto" + single_call = request.parallel_tool_calls is False + strict_level = self.tool_strict_level + if single_call and not (is_auto and structured_outputs is not None): + # Limiting the call count needs the call envelope in the grammar. + # Auto with structured outputs keeps its format-only grammar, so the + # flag never makes calls possible that are impossible without it. + strict_level = max(strict_level, ToolStrictLevel.FUNCTION) resolved_tools = None if tool_parser.structural_tag_model is not None: resolved_tools = resolve_tool_strictness( request.tools, request.tool_choice, - self.tool_strict_level, + strict_level, ) output_format = None @@ -529,7 +539,7 @@ class DelegatingParser(Parser): tools_structural_tag = tool_parser.get_structural_tag( tag_request, reasoning=False, - strict_level=self.tool_strict_level, + strict_level=strict_level, ) else: tools_structural_tag = None @@ -544,6 +554,8 @@ class DelegatingParser(Parser): ) return request + if single_call: + limit_to_single_tool_call(tools_structural_tag) structural_tag = tools_structural_tag if output_format is not None: structural_tag = StructuralTag( diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index bcbb5757c1..bc5abea3a2 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -141,7 +141,9 @@ class ToolParser: return request json_schema_from_tool = get_json_schema_from_tools( - tool_choice=request.tool_choice, tools=request.tools + tool_choice=request.tool_choice, + tools=request.tools, + parallel_tool_calls=request.parallel_tool_calls, ) # Set structured output params for tool calling if json_schema_from_tool is not None: diff --git a/vllm/tool_parsers/mimo_tool_parser.py b/vllm/tool_parsers/mimo_tool_parser.py index cca77eb56b..3b44c5d989 100644 --- a/vllm/tool_parsers/mimo_tool_parser.py +++ b/vllm/tool_parsers/mimo_tool_parser.py @@ -1,25 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from xgrammar.structural_tag import TriggeredTagsFormat - from vllm.parser.engine.registered_adapters import MiMoParserToolAdapter -from vllm.tool_parsers.tool_strict_level import ToolStrictLevel class MiMoToolParser(MiMoParserToolAdapter): # type: ignore[valid-type, misc] structural_tag_model = "mimo" - - def get_structural_tag( - self, request, *, reasoning=False, strict_level=ToolStrictLevel.AUTO - ): - tag = super().get_structural_tag( - request, reasoning=reasoning, strict_level=strict_level - ) - if ( - tag is not None - and request.parallel_tool_calls is False - and isinstance(tag.format, TriggeredTagsFormat) - ): - tag.format.stop_after_first = True - return tag diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index bc8fab8fe6..ef57e85fab 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -9,6 +9,7 @@ from openai.types.responses.response import ToolChoice as ResponsesToolChoice from openai.types.responses.tool import Tool as ResponsesTool from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed from openai.types.responses.tool_choice_function import ToolChoiceFunction +from pydantic import BaseModel from xgrammar import StructuralTag, normalize_tool_choice from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag from xgrammar.openai_tool_call_schema import ( @@ -27,6 +28,7 @@ from xgrammar.structural_tag import ( StarFormat, TagFormat, TagsWithSeparatorFormat, + TokenTriggeredTagsFormat, TriggeredTagsFormat, ) @@ -197,6 +199,34 @@ def get_model_structural_tag( ) +_TOOL_CALL_LIST_FORMATS = ( + TriggeredTagsFormat, + TokenTriggeredTagsFormat, + TagsWithSeparatorFormat, +) + + +def limit_to_single_tool_call(tag: StructuralTag) -> StructuralTag: + """Stop every tool-call list in ``tag`` after its first call, in place. + + Enforces ``parallel_tool_calls=false`` in the grammar. In every registered + tag, ``_TOOL_CALL_LIST_FORMATS`` repeat tool calls, never arguments. + """ + + def visit(node: Any) -> None: + if isinstance(node, BaseModel): + if isinstance(node, _TOOL_CALL_LIST_FORMATS): + node.stop_after_first = True + for name in type(node).model_fields: + visit(getattr(node, name)) + elif isinstance(node, (list, tuple)): + for item in node: + visit(item) + + visit(tag.format) + return tag + + def _dump_tool_for_xgrammar( tool: ChatCompletionToolsParam | ResponsesTool, ) -> dict[str, Any]: diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py index d2c8c6f412..081e58da04 100644 --- a/vllm/tool_parsers/utils.py +++ b/vllm/tool_parsers/utils.py @@ -347,6 +347,7 @@ def _get_tool_schema_defs( def _get_json_schema_from_tools( tools: list[Tool], + parallel_tool_calls: bool | None = None, ) -> dict: fn_tool_schemas: list[dict[str, Any]] = [] fn_tools: list[Tool] = [] @@ -361,7 +362,7 @@ def _get_json_schema_from_tools( elif _is_function_tool(tool): fn_tool_schemas.append(_get_tool_schema_from_tool(tool)) fn_tools.append(tool) - json_schema = { + json_schema: dict[str, Any] = { "type": "array", "minItems": 1, "items": { @@ -369,6 +370,10 @@ def _get_json_schema_from_tools( "anyOf": fn_tool_schemas, }, } + # Only an explicit `false` opts out of parallel tool calls; `None` means the + # field was never set and the default (true) applies. + if parallel_tool_calls is False: + json_schema["maxItems"] = 1 json_schema_defs = _get_tool_schema_defs(fn_tools) if json_schema_defs: json_schema["$defs"] = json_schema_defs @@ -378,6 +383,7 @@ def _get_json_schema_from_tools( def get_json_schema_from_tools( tool_choice: str | ToolChoiceFunction | ChatCompletionNamedToolChoiceParam, tools: list[Tool] | None, + parallel_tool_calls: bool | None = None, ) -> str | dict | None: # tool_choice: "none" if tool_choice in ("none", None) or tools is None: @@ -413,7 +419,7 @@ def get_json_schema_from_tools( return chat_tool_map[tool_name].function.parameters # tool_choice: "required" if tool_choice == "required": - return _get_json_schema_from_tools(tools) + return _get_json_schema_from_tools(tools, parallel_tool_calls) # tool_choice: "auto" return None From 5e887a078b764e69b6579926619e25eb4f0066dc Mon Sep 17 00:00:00 2001 From: cherry77-cloud <1615405@qq.com> Date: Wed, 30 Sep 2026 05:58:46 +0800 Subject: [PATCH 057/248] [Bugfix] Route Step3p5 forced tool choices through XML parser (#51810) Signed-off-by: taking-lying-flat <1615405@qq.com> Signed-off-by: Flora Feng <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../tool_parsers/test_step3p5_tool_parser.py | 46 ++++++----- .../test_structural_tag_registry.py | 43 +++++++++++ vllm/tool_parsers/step3p5_tool_parser.py | 76 ++++--------------- 3 files changed, 83 insertions(+), 82 deletions(-) diff --git a/tests/tool_parsers/test_step3p5_tool_parser.py b/tests/tool_parsers/test_step3p5_tool_parser.py index de21aab797..0dbe114545 100644 --- a/tests/tool_parsers/test_step3p5_tool_parser.py +++ b/tests/tool_parsers/test_step3p5_tool_parser.py @@ -5,6 +5,7 @@ import json from collections.abc import Generator import pytest +from openai.types.responses import FunctionTool from vllm.entrypoints.generate.base.protocol import ( DeltaMessage, @@ -386,26 +387,29 @@ TX assert extracted_tool_calls.tool_calls[0].function.name == "get_current_weather" -def test_extract_tool_calls_type_conversion(step3p5_tokenizer): - """Test parameter type conversion based on tool schema.""" - tools = [ - ChatCompletionToolsParam( - type="function", - function={ - "name": "test_types", - "parameters": { - "type": "object", - "properties": { - "int_param": {"type": "integer"}, - "float_param": {"type": "float"}, - "bool_param": {"type": "boolean"}, - "str_param": {"type": "string"}, - "obj_param": {"type": "object"}, - }, - }, - }, - ) - ] +@pytest.mark.parametrize("api", ["chat", "responses"]) +def test_extract_tool_calls_type_conversion(step3p5_tokenizer, api): + """Test parameter type conversion based on Chat and Responses tool schemas.""" + parameters = { + "type": "object", + "properties": { + "int_param": {"type": "integer"}, + "float_param": {"type": "float"}, + "bool_param": {"type": "boolean"}, + "str_param": {"type": "string"}, + "obj_param": {"type": "object"}, + }, + } + tools: list[ChatCompletionToolsParam | FunctionTool] = ( + [ + ChatCompletionToolsParam( + type="function", + function={"name": "test_types", "parameters": parameters}, + ) + ] + if api == "chat" + else [FunctionTool(type="function", name="test_types", parameters=parameters)] + ) model_output = """ @@ -428,7 +432,7 @@ hello world """ parser = Step3p5ToolParser(step3p5_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) + request = ChatCompletionRequest(model=MODEL, messages=[]) extracted_tool_calls = parser.extract_tool_calls(model_output, request=request) args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments) diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 2c4de60a9d..50f8751f90 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -47,6 +47,7 @@ from vllm.tool_parsers.mimo_tool_parser import MiMoToolParser from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser +from vllm.tool_parsers.step3p5_tool_parser import Step3p5ToolParser from vllm.tool_parsers.structural_tag_registry import ( SUPPORTED_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, @@ -669,6 +670,7 @@ def test_get_model_structural_tag_supports_named_tool_choice( (Llama3JsonToolParser, "llama"), (MinimaxM2ToolParser, "minimax"), (Qwen3EngineToolParser, "qwen_3_coder"), + (Step3p5ToolParser, "qwen_3_coder"), (MiMoToolParser, "mimo"), ], ) @@ -677,6 +679,47 @@ def test_tool_parsers_declare_matching_xgrammar_builtin_model(parser_cls, model) assert not parser_cls.supports_required_and_named +@pytest.mark.parametrize( + "tool_choice", + ["required", {"type": "function", "function": {"name": "get_weather"}}], +) +def test_step3p5_forced_tool_choice_round_trips_native_xml( + sample_tools: list[ChatCompletionToolsParam], tool_choice +): + """Forced Step calls must be constrained to, and parsed from, Step XML.""" + + class StepParser(DelegatingParser): + tool_parser_cls = Step3p5ToolParser + + request = ChatCompletionRequest( + messages=[], + model="m", + # Named tool choice validation reads tools in their JSON form. + tools=[tool.model_dump(exclude_none=True) for tool in sample_tools], + tool_choice=tool_choice, + ) + parser = StepParser(MagicMock(), tools=sample_tools) + request = parser.adjust_request(request) + + assert request.structured_outputs.json is None + grammar = Grammar.from_structural_tag(request.structured_outputs.structural_tag) + # Rendered exactly as the Step-3.5/3.7 chat templates render tool calls. + output = ( + "\n\n" + "\nDallas\n\n" + "\n" + ) + assert _is_grammar_accept_string(grammar, output) + assert not _is_grammar_accept_string(grammar, "It is sunny in Dallas.") + + _, content, tool_calls = parser.parse(output, request, enable_auto_tools=True) + + assert not content + assert [(call.name, json.loads(call.arguments)) for call in tool_calls] == [ + ("get_weather", {"city": "Dallas"}) + ] + + def test_tool_parsers_without_structural_tag_support_required_and_named(): class NonStructuralTagToolParser(ToolParser): pass diff --git a/vllm/tool_parsers/step3p5_tool_parser.py b/vllm/tool_parsers/step3p5_tool_parser.py index 2639d2f204..ca0d86a9e0 100644 --- a/vllm/tool_parsers/step3p5_tool_parser.py +++ b/vllm/tool_parsers/step3p5_tool_parser.py @@ -22,7 +22,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser -from vllm.tool_parsers.utils import safe_literal_eval +from vllm.tool_parsers.utils import ( + find_tool_name, + find_tool_properties, + safe_literal_eval, +) logger = init_logger(__name__) @@ -421,43 +425,14 @@ class StreamingXMLToolCallParser: def _validate_function_name(self, func_name: str) -> bool: """Check if function name exists in tool definitions.""" - if not self.tools: - return False - - for tool in self.tools: - if ( - hasattr(tool, "type") - and tool.type == "function" - and hasattr(tool, "function") - and hasattr(tool.function, "name") - and tool.function.name == func_name - ): - return True - - return False + return find_tool_name(self.tools, func_name) def _validate_parameter_name(self, param_name: str) -> bool: """Check if parameter exists in current function's tool definition.""" if not self.tools or not self.current_function_name: return True - - for tool in self.tools: - if ( - hasattr(tool, "type") - and tool.type == "function" - and hasattr(tool, "function") - and hasattr(tool.function, "name") - and tool.function.name == self.current_function_name - ): - if not hasattr(tool.function, "parameters"): - return True - params = tool.function.parameters - if isinstance(params, dict): - properties = params.get("properties", params) - return param_name in properties - break - - return True + properties = find_tool_properties(self.tools, self.current_function_name) + return not properties or param_name in properties def _should_skip_element(self, element: str) -> bool: """Determine whether an element should be skipped. @@ -1190,34 +1165,10 @@ class StreamingXMLToolCallParser: """ if not self.tools or not self.current_function_name: return "string" - - for tool in self.tools: - if not hasattr(tool, "type") or not ( - hasattr(tool, "function") and hasattr(tool.function, "name") - ): - continue - if ( - tool.type == "function" - and tool.function.name == self.current_function_name - ): - if not hasattr(tool.function, "parameters"): - return "string" - params = tool.function.parameters - if isinstance(params, dict) and "properties" in params: - properties = params["properties"] - if param_name in properties and isinstance( - properties[param_name], dict - ): - return self.repair_param_type( - str(properties[param_name].get("type", "string")) - ) - elif isinstance(params, dict) and param_name in params: - param_config = params[param_name] - if isinstance(param_config, dict): - return self.repair_param_type( - str(param_config.get("type", "string")) - ) - break + properties = find_tool_properties(self.tools, self.current_function_name) + param_config = properties.get(param_name) + if isinstance(param_config, dict): + return self.repair_param_type(str(param_config.get("type", "string"))) return "string" def repair_param_type(self, param_type: str) -> str: @@ -1357,6 +1308,9 @@ class StreamingXMLToolCallParser: class Step3p5ToolParser(ToolParser): + # Step-3.5/3.7 chat templates use the Qwen3-Coder XML tool-call format. + structural_tag_model = "qwen_3_coder" + def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): super().__init__(tokenizer, tools) self.parser = StreamingXMLToolCallParser() From a7a9ac2745f47e55fa21ffe6bb5c94d2e086efa1 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 30 Sep 2026 06:20:51 +0800 Subject: [PATCH 058/248] [Core] Combine per-engine utility results like the Rust client (#59240) Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- vllm/v1/engine/core_client.py | 38 +++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 32770a9072..b03c40f440 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1239,6 +1239,23 @@ class AsyncMPClient(MPClient): async def call_utility_async(self, method: str, *args) -> Any: return await self._call_utility_async(method, *args, engine=self.core_engine) + async def call_utility_all_async(self, method: str, *args) -> list[Any]: + # Like the Rust client's call_utility: one result per managed engine. + return await asyncio.gather( + *[ + self._call_utility_async(method, *args, engine=engine) + for engine in self.core_engines + ] + ) + + async def call_utility_consensus_async(self, method: str, *args) -> Any: + results = await self.call_utility_all_async(method, *args) + if any(result != results[0] for result in results): + raise RuntimeError( + f"Engines returned different {method} results: {results}" + ) + return results[0] + async def _call_utility_async( self, method: str, *args, engine: EngineIdentity ) -> Any: @@ -1274,7 +1291,7 @@ class AsyncMPClient(MPClient): await self.call_utility_async("resume_scheduler") async def is_scheduler_paused_async(self) -> bool: - return await self.call_utility_async("is_scheduler_paused") + return await self.call_utility_consensus_async("is_scheduler_paused") async def profile_async( self, is_start: bool = True, profile_prefix: str | None = None @@ -1287,8 +1304,10 @@ class AsyncMPClient(MPClient): async def reset_prefix_cache_async( self, reset_running_requests: bool = False, reset_connector: bool = False ) -> bool: - return await self.call_utility_async( - "reset_prefix_cache", reset_running_requests, reset_connector + return all( + await self.call_utility_all_async( + "reset_prefix_cache", reset_running_requests, reset_connector + ) ) async def reset_encoder_cache_async(self) -> None: @@ -1304,7 +1323,7 @@ class AsyncMPClient(MPClient): return await self.call_utility_async("wake_up", tags) async def is_sleeping_async(self) -> bool: - return await self.call_utility_async("is_sleeping") + return await self.call_utility_consensus_async("is_sleeping") async def execute_dummy_batch_async(self) -> None: await self.call_utility_async("execute_dummy_batch") @@ -1313,7 +1332,7 @@ class AsyncMPClient(MPClient): await self.call_utility_async("set_weight_version", weight_version) async def get_weight_version_async(self) -> str: - return await self.call_utility_async("get_weight_version") + return await self.call_utility_consensus_async("get_weight_version") async def add_lora_async(self, lora_request: LoRARequest) -> bool: return await self.call_utility_async("add_lora", lora_request) @@ -1645,14 +1664,7 @@ class DPLBAsyncMPClient(DPAsyncMPClient): async def call_utility_async(self, method: str, *args) -> Any: # Only the result from the first engine is returned. - return ( - await asyncio.gather( - *[ - self._call_utility_async(method, *args, engine=engine) - for engine in self.core_engines - ] - ) - )[0] + return (await self.call_utility_all_async(method, *args))[0] @staticmethod async def process_engine_outputs( From 797505747f7847e4415fb1db604421810de1865e Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Tue, 29 Sep 2026 18:36:11 -0400 Subject: [PATCH 059/248] [Bugfix][Frontend] Honor parallel_tool_calls=false in the Responses API (#59298) Signed-off-by: sfeng33 <4florafeng@gmail.com> --- .../responses/test_serving_responses.py | 56 +++++++++++++++++++ vllm/entrypoints/openai/responses/context.py | 7 ++- vllm/entrypoints/openai/responses/serving.py | 6 +- .../serve/utils/tool_calls_utils.py | 37 +++++++----- 4 files changed, 89 insertions(+), 17 deletions(-) diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 89c188ec8a..c06e2b5915 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -1359,6 +1359,7 @@ def _make_simple_context_with_output( def _make_serving_instance( *, reasoning_parser: str = "", + tool_parser: str | None = None, enable_per_request_metrics: bool = False, ) -> OpenAIServingResponses: engine_client = MagicMock() @@ -1379,6 +1380,8 @@ def _make_serving_instance( chat_template=None, chat_template_content_format="auto", reasoning_parser=reasoning_parser, + enable_auto_tools=tool_parser is not None, + tool_parser=tool_parser, enable_per_request_metrics=enable_per_request_metrics, ) @@ -1977,3 +1980,56 @@ class TestAutoToolStreaming: assert len(function_done) == 1 assert function_done[0].item.name == "get_weather" assert function_done[0].item.arguments == tool_args + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)]) +async def test_streaming_parallel_tool_calls( + monkeypatch, parallel_tool_calls, num_calls +): + """parallel_tool_calls=False keeps only the first call even when the grammar + does not limit the model; None keeps every call.""" + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) + serving = _make_serving_instance(tool_parser="hermes") + request = ResponsesRequest( + input="hi", + tools=[{"type": "function", "name": "get_weather", "parameters": {}}], + parallel_tool_calls=parallel_tool_calls, + stream=True, + ) + context = SimpleContext( + response_parser=serving._make_response_parser(request, MagicMock(), {}) + ) + args = [f'{{"city": "{city}"}}' for city in ("Paris", "Berlin", "Tokyo")] + + async def result_generator(): + for i, arg in enumerate(args): + text = f'\n{{"name": "get_weather", "arguments": {arg}}}' + context.append_output(_make_request_output(f"{text}\n\n", [i])) + yield context + + events = [ + event + async for event in serving.responses_stream_generator( + request=request, + sampling_params=SamplingParams(max_tokens=64), + result_generator=result_generator(), + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + ] + + assert [ + event.output_index + for event in events + if event.type == "response.output_item.added" + ] == list(range(num_calls)) + assert all(getattr(event, "output_index", 0) < num_calls for event in events) + assert [ + event.item.arguments + for event in events + if event.type == "response.output_item.done" + ] == args[:num_calls] + assert [item.arguments for item in events[-1].response.output] == args[:num_calls] diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index fa233dcce1..726a955a55 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -37,6 +37,9 @@ from vllm.entrypoints.openai.responses.utils import ( build_response_output_items, construct_tool_dicts, ) +from vllm.entrypoints.serve.utils.tool_calls_utils import ( + maybe_filter_parallel_tool_calls, +) from vllm.outputs import RequestOutput from vllm.parser.abstract_parser import Parser from vllm.tokenizers import TokenizerLike @@ -369,7 +372,9 @@ class ParsableContext(ConversationContext): build_response_output_items( reasoning=reasoning, content=content, - tool_calls=tool_calls, + tool_calls=maybe_filter_parallel_tool_calls( + tool_calls or [], self.request + ), tools=self.request.tools, ) ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index f18c493259..4facd2fb3b 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -73,6 +73,9 @@ from vllm.entrypoints.openai.responses.utils import ( from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.api_utils import get_max_tokens from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.entrypoints.serve.utils.tool_calls_utils import ( + maybe_filter_parallel_tool_calls, +) from vllm.exceptions import GenerationError, VLLMValidationError from vllm.inputs import EngineInput from vllm.logger import init_logger @@ -1028,7 +1031,7 @@ class OpenAIServingResponses(GenerateBaseServing): return build_response_output_items( reasoning=reasoning, content=content, - tool_calls=tool_calls, + tool_calls=maybe_filter_parallel_tool_calls(tool_calls or [], request), logprobs=logprobs, tools=request.tools, ) @@ -1231,6 +1234,7 @@ class OpenAIServingResponses(GenerateBaseServing): if not delta_message: continue + delta_message = maybe_filter_parallel_tool_calls(delta_message, request) for dm in split_delta(delta_message): target_state, tool_call = processor.resolve_target_state(dm) diff --git a/vllm/entrypoints/serve/utils/tool_calls_utils.py b/vllm/entrypoints/serve/utils/tool_calls_utils.py index ae2fa24f7f..1252218127 100644 --- a/vllm/entrypoints/serve/utils/tool_calls_utils.py +++ b/vllm/entrypoints/serve/utils/tool_calls_utils.py @@ -2,35 +2,42 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import TypeVar +from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest # Used internally -_ChatCompletionResponseChoiceT = TypeVar( - "_ChatCompletionResponseChoiceT", +_ToolCallsContainerT = TypeVar( + "_ToolCallsContainerT", ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, + DeltaMessage, + list[FunctionCall], ) def maybe_filter_parallel_tool_calls( - choice: _ChatCompletionResponseChoiceT, request: ChatCompletionRequest -) -> _ChatCompletionResponseChoiceT: + item: _ToolCallsContainerT, + request: ChatCompletionRequest | ResponsesRequest, +) -> _ToolCallsContainerT: """Filter to first tool call only when parallel_tool_calls is explicitly False.""" if request.parallel_tool_calls is not False: - return choice + return item - if isinstance(choice, ChatCompletionResponseChoice) and choice.message.tool_calls: - choice.message.tool_calls = choice.message.tool_calls[:1] - elif ( - isinstance(choice, ChatCompletionResponseStreamChoice) - and choice.delta.tool_calls - ): - choice.delta.tool_calls = [ - tool_call for tool_call in choice.delta.tool_calls if tool_call.index == 0 + if isinstance(item, list): + return item[:1] + if isinstance(item, ChatCompletionResponseChoice): + if item.message.tool_calls: + item.message.tool_calls = item.message.tool_calls[:1] + return item + + delta = item.delta if isinstance(item, ChatCompletionResponseStreamChoice) else item + if delta.tool_calls: + delta.tool_calls = [ + tool_call for tool_call in delta.tool_calls if tool_call.index == 0 ] - - return choice + return item From 50239dfc4d10f007887f2cbc042809e3f4c2ceeb Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Tue, 29 Sep 2026 18:36:30 -0400 Subject: [PATCH 060/248] [WideEP] Change DeepEPv2 to auto select hybrid mode by default (#57991) Signed-off-by: Tyler Michael Smith Co-authored-by: Codex --- vllm/distributed/device_communicators/all2all.py | 4 +++- vllm/envs.py | 8 ++++++-- vllm/utils/nccl.py | 8 +++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/vllm/distributed/device_communicators/all2all.py b/vllm/distributed/device_communicators/all2all.py index 21a1f00da4..5f339540b4 100644 --- a/vllm/distributed/device_communicators/all2all.py +++ b/vllm/distributed/device_communicators/all2all.py @@ -1065,7 +1065,9 @@ class DeepEPV2All2AllManager(All2AllManagerBase): if os.environ.get("EP_DISABLE_GIN", "0") != "0": return - gin_type = query_nccl_gin_type(group) + gin_type = query_nccl_gin_type( + group, railed=envs.VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE + ) if gin_type is None: raise RuntimeError( "DeepEPv2 communicator properties query failed; " diff --git a/vllm/envs.py b/vllm/envs.py index c8e1a8b708..428d6d3c94 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -2009,9 +2009,13 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool( int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0")) ), - # DeepEP v2: enable two-tier NVLink+RDMA hybrid mode + # DeepEP v2: use NVLink+RDMA hybrid communication across NVLink domains. + # * hybrid mode = multi-node RDMA + # * non hybrid mode = nvlink + # Set to 1 (the default) to allow DeepEP to autoselect hybrid vs non-hybrid. + # Set to 0 to disable hybrid mode. "VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE": lambda: bool( - int(os.getenv("VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE", "0")) + int(os.getenv("VLLM_DEEPEP_V2_ALLOW_HYBRID_MODE", "1")) ), # DeepEP v2: use fewer SMs at slight throughput cost "VLLM_DEEPEP_V2_PREFER_OVERLAP": lambda: bool( diff --git a/vllm/utils/nccl.py b/vllm/utils/nccl.py index 5a033f1852..90064d4184 100644 --- a/vllm/utils/nccl.py +++ b/vllm/utils/nccl.py @@ -83,8 +83,10 @@ def find_nccl_library_paths() -> list[str] | None: return paths or None -def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None: - """Return the GIN type for an initialized group, or ``None`` on failure.""" +def query_nccl_gin_type( + group: torch.distributed.ProcessGroup, *, railed: bool = False +) -> int | None: + """Return the full or railed GIN type, or ``None`` on query failure.""" from vllm.distributed.device_communicators.pynccl_wrapper import ( NCCL_COMM_PROPERTIES_LAYOUT_VERSION, NCCLLibrary, @@ -126,4 +128,4 @@ def query_nccl_gin_type(group: torch.distributed.ProcessGroup) -> int | None: if result != 0: logger.warning("ncclCommQueryProperties returned error %d", result) return None - return props.ginType + return props.railedGinType if railed else props.ginType From c37f86e5721f089a27c8e4358c30e435805fcaab Mon Sep 17 00:00:00 2001 From: drakosha Date: Wed, 30 Sep 2026 01:36:39 +0300 Subject: [PATCH 061/248] [Bugfix] GLM-5.3-Flash: fp8 plan dtype on SM90 sparse MLA, and right-size the indexer prefill workspace (#55222) Signed-off-by: Mikhail Kostryukov Signed-off-by: NickLucche Co-authored-by: Mikhail Kostryukov Co-authored-by: NickLucche --- .../test_flashinfer_mla_sparse_sm90.py | 31 ++++++++++---- .../test_indexer_deepseek_v4_slot_mapping.py | 41 +++++++++++++++++++ vllm/models/glm5next/common/attention.py | 4 +- .../mla/flashinfer_mla_sparse_sm90.py | 8 +++- vllm/v1/attention/backends/mla/indexer.py | 19 +++++---- 5 files changed, 84 insertions(+), 19 deletions(-) diff --git a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py index b25656cf6e..735748ae94 100644 --- a/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py +++ b/tests/v1/attention/test_flashinfer_mla_sparse_sm90.py @@ -208,12 +208,14 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec assert builder.state.plan_calls[0][1].tolist() == expected_lens -def test_plan_uses_state_params(monkeypatch): +@pytest.mark.parametrize("kv_dtype", [torch.bfloat16, torch.float8_e4m3fn]) +def test_plan_uses_state_params(monkeypatch, kv_dtype): """The NoPE/rope dims and scale live on the builder state, not the layer. Without a device batch layout plan() takes exact per-row KV lengths and rebuilds the schedule on every call; the indptrs are always full-size - with zero-query padding rows past num_tokens. + with zero-query padding rows past num_tokens. An fp8 cache plans as + float8_e4m3fn. """ wrapper = FakeWrapper() plan_info = [0] * sm90_mod._PLAN_INFO_LEN @@ -226,7 +228,7 @@ def test_plan_uses_state_params(monkeypatch): state.device = torch.device("cpu") state.wrapper = wrapper state.num_heads = 4 - state.kv_dtype = torch.bfloat16 + state.kv_dtype = kv_dtype state.kv_lora_rank = HEAD state.qk_rope_head_dim = 64 state.sm_scale = 576**-0.5 @@ -249,7 +251,7 @@ def test_plan_uses_state_params(monkeypatch): assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False) assert scale == 576**-0.5 assert kwargs["q_data_type"] == torch.bfloat16 - assert kwargs["kv_data_type"] == torch.bfloat16 + assert kwargs["kv_data_type"] == kv_dtype # Replanning a smaller batch must clear the previous rows' lengths. state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None) @@ -339,6 +341,14 @@ def test_kv_lens_host_empty(): _NO_KPOOL = object() +@pytest.mark.parametrize( + "spec_dtype,expected_kv_dtype", + [ + (torch.bfloat16, torch.bfloat16), + (torch.uint8, torch.float8_e4m3fn), + ], + ids=["bf16", "fp8"], +) @pytest.mark.parametrize( "index_kpool,prefill_lens", [ @@ -348,9 +358,12 @@ _NO_KPOOL = object() ], ids=["kpool4", "kpool_none", "no_kpool_attr"], ) -def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens): +def test_builder_kpool_from_model_config( + monkeypatch, index_kpool, prefill_lens, spec_dtype, expected_kv_dtype +): """The builder took kpool from the KV cache spec, whose tokens_per_state is - 1, so the tail pool was never read.""" + 1, so the tail pool was never read. The state plans the cache's logical + dtype: uint8 fp8 storage becomes float8_e4m3fn.""" monkeypatch.setattr( sm90_mod.FlashInferMLASparseMetadataBuilder, "__init__", @@ -358,14 +371,15 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) ) class _RecordingState: - def __init__(self, *_args, **kwargs): + def __init__(self, *args, **kwargs): + self.kv_dtype = args[2] self.index_topk = kwargs["index_topk"] self.index_kpool = kwargs["index_kpool"] monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState) impl, _ = make_impl(64) spec = MLAAttentionSpec( - block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16 + block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=spec_dtype ) assert spec.tokens_per_state == 1 hf_config = SimpleNamespace(index_topk=2048) @@ -384,6 +398,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens) builder = FlashInferMLASparseSM90Builder( spec, ["attn"], vllm_config, torch.device("cpu") ) + assert builder.state.kv_dtype == expected_kv_dtype # req0: prefill chunk ending at context 42295; req1: context <= topk. cam = SimpleNamespace( num_reqs=2, diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index 9c92ae554e..bf775078c8 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -24,6 +24,7 @@ from vllm.v1.attention.backends.mla.indexer import ( DeepseekV4IndexerBackend, DeepseekV32IndexerMetadataBuilder, DeepseekV41IndexerBackend, + get_max_prefill_buffer_size, ) from vllm.v1.attention.backends.mla.sparse_utils import ( ConvertReqIndexToGlobalIndexKernel, @@ -377,3 +378,43 @@ def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states(): device=device, ) torch.testing.assert_close(valid_slots, expected) + + +@pytest.mark.parametrize("compress_ratio", [1, 4]) +def test_indexer_prefill_budget_matches_compressed_workspace(compress_ratio): + """The chunker budget is in compressed rows, like the K-gather workspace.""" + max_model_len = 1024 + kv_cache_spec = MLAAttentionSpec( + block_size=256, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + tokens_per_state=compress_ratio, + ) + vllm_config = create_vllm_config(max_model_len=max_model_len) + max_num_blocks = kv_cache_spec.max_num_blocks_per_req(vllm_config, max_model_len) + block_table_width = get_block_table_width(max_num_blocks, kv_cache_spec.block_size) + builder = DeepseekV32IndexerMetadataBuilder( + kv_cache_spec=kv_cache_spec, + layer_names=["dummy"], + vllm_config=vllm_config, + device=torch.device("cpu"), + block_table_width=block_table_width, + ) + + workspace_rows = get_max_prefill_buffer_size(vllm_config) // compress_ratio + assert builder.max_prefill_buffer_size == workspace_rows + + # Overflows the workspace unless the chunker splits. + num_reqs = 3 * (workspace_rows // (max_model_len // compress_ratio)) + 1 + compressed_seq_lens = torch.full((num_reqs,), max_model_len // compress_ratio) + query_lens = torch.ones(num_reqs, dtype=torch.int64) + chunks = builder._split_indexer_prefill_chunks( + compressed_seq_lens, + query_lens, + builder.max_prefill_buffer_size, + max_logits_bytes=10**15, + ) + assert len(chunks) > 1 + for req_slice, _ in chunks: + assert int(compressed_seq_lens[req_slice].sum()) <= workspace_rows diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index 25f51117a8..f39c3a7cd0 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -312,7 +312,9 @@ class Indexer(nn.Module): self.prefix = prefix from vllm.v1.attention.backends.mla.indexer import get_max_prefill_buffer_size - self.max_total_seq_len = get_max_prefill_buffer_size(vllm_config) + self.max_total_seq_len = ( + get_max_prefill_buffer_size(vllm_config) // self.index_kpool + ) self.indexer_op = SparseAttnIndexerKpool( self.k_cache, self.quant_block_size, diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py index a5f7757d38..16d7c12e67 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm90.py @@ -443,10 +443,16 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder): assert topk_indices_buffer is not None hf_config = vllm_config.model_config.hf_text_config assert hf_config.index_topk is not None + # fp8 KV cache is stored as uint8; plan() needs the logical dtype. + kv_plan_dtype = ( + torch.float8_e4m3fn + if kv_cache_spec.dtype == torch.uint8 + else kv_cache_spec.dtype + ) self.state = _SM90State( device, impl.num_heads, - kv_cache_spec.dtype, + kv_plan_dtype, vllm_config.scheduler_config.max_num_batched_tokens, topk_indices_buffer.shape[1], kv_lora_rank=impl.kv_lora_rank, diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index f2f2c51925..afaae5e35f 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -901,8 +901,17 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): self.use_pcp = self.pcp_world_size > 1 self.pcp_rank = get_pcp_group().rank_in_group if self.use_pcp else 0 self.cp_kv_cache_interleave_size = parallel_config.cp_kv_cache_interleave_size + # KV compression (DeepseekV4). Default to 1 for no compression. + self.compress_ratio = 1 + if isinstance(self.kv_cache_spec, MLAAttentionSpec): + assert isinstance(self.kv_cache_spec.tokens_per_state, int) + self.compress_ratio = self.kv_cache_spec.tokens_per_state # NOTE(Chen):an estimated max size of flattened_kv. Need to double check. - self.max_prefill_buffer_size = get_max_prefill_buffer_size(self.vllm_config) + # Counted in compressed rows, like the chunker's seq_lens and the + # workspace. + self.max_prefill_buffer_size = ( + get_max_prefill_buffer_size(self.vllm_config) // self.compress_ratio + ) self.num_speculative_tokens = ( self.vllm_config.speculative_config.num_speculative_tokens if self.vllm_config.speculative_config @@ -984,14 +993,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): (self.num_sms + 1, 2), dtype=torch.int32, device=self.device ) - # KV compression. Default to 1 for no compression. - self.compress_ratio = 1 - # Get compress_ratio for DeepseekV4 support - if isinstance(self.kv_cache_spec, MLAAttentionSpec): - # MLA compression is a whole number of tokens per state (fractions - # are whisper block pooling and never reach MLA). - assert isinstance(self.kv_cache_spec.tokens_per_state, int) - self.compress_ratio = self.kv_cache_spec.tokens_per_state if self.dcp_world_size > 1 and self.compress_ratio > 1: raise NotImplementedError( "DCP is not supported with sparse indexer KV compression " From 0755e69e75838babf3eca3fcdb74799b92aa3ea2 Mon Sep 17 00:00:00 2001 From: RongJie G <111257566+CorgiBoyG@users.noreply.github.com> Date: Wed, 30 Sep 2026 06:45:14 +0800 Subject: [PATCH 062/248] [Bugfix][Responses API] Preserve built-in tool output call IDs (#55596) Signed-off-by: CorgiBoyG <111257566+CorgiBoyG@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../responses/test_parsable_context_unit.py | 65 ++++++++++++++++++- vllm/entrypoints/mcp/tool.py | 2 +- vllm/entrypoints/openai/responses/context.py | 21 +++--- 3 files changed, 78 insertions(+), 10 deletions(-) diff --git a/tests/entrypoints/openai/responses/test_parsable_context_unit.py b/tests/entrypoints/openai/responses/test_parsable_context_unit.py index 0398723184..710cd2d04c 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context_unit.py +++ b/tests/entrypoints/openai/responses/test_parsable_context_unit.py @@ -6,10 +6,13 @@ These tests verify that ParsableContext correctly delegates to the unified Parser (via parse) and properly builds response output items. """ +import json from collections.abc import Sequence -from unittest.mock import MagicMock +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock import pytest +from openai.types.responses import ResponseFunctionToolCall from vllm.entrypoints.generate.base.protocol import ( DeltaMessage, @@ -17,6 +20,7 @@ from vllm.entrypoints.generate.base.protocol import ( FunctionCall, ToolCall, ) +from vllm.entrypoints.mcp.tool import HarmonyPythonTool from vllm.entrypoints.openai.responses.context import ParsableContext from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.outputs import CompletionOutput, RequestOutput @@ -317,6 +321,65 @@ def test_process_extracts_tool_calls(): assert tool_item.status == "completed" +@pytest.mark.parametrize( + ("tool_name", "session_name", "dispatched_name", "arguments"), + [ + ("code_interpreter", "python", "python", {"code": "print(42)"}), + ("web_search_preview", "browser", "search", {"query": "vLLM"}), + ("container.exec", "container", "exec", {"cmd": ["pwd"]}), + ], +) +@pytest.mark.asyncio +async def test_builtin_tool_output_preserves_function_call_id( + tool_name, session_name, dispatched_name, arguments +): + """Built-in tool outputs remain correlated with their originating call.""" + tool_session = MagicMock() + tool_session.call_tool = AsyncMock( + return_value=SimpleNamespace(content=[SimpleNamespace(text="result")]) + ) + context = _make_context(None, available_tools=[session_name]) + tool_call = ResponseFunctionToolCall( + id=f"fc_{session_name}", + call_id=f"call_{session_name}", + type="function_call", + name=tool_name, + arguments=json.dumps(arguments), + ) + context.response_messages.append(tool_call) + context._tool_sessions[session_name] = tool_session + + output = await context.call_tool() + + tool_session.call_tool.assert_awaited_once_with(dispatched_name, arguments) + assert output[0].call_id == tool_call.call_id + + +@pytest.mark.asyncio +async def test_local_python_tool_output_preserves_function_call_id(): + """Local code-interpreter output remains correlated with its call.""" + + async def process(_): + yield SimpleNamespace(content=[SimpleNamespace(text="result")]) + + python_tool = object.__new__(HarmonyPythonTool) + python_tool.python_tool = MagicMock(process=process) + context = _make_context(None, available_tools=["python"]) + tool_call = ResponseFunctionToolCall( + id="fc_python", + call_id="call_python", + type="function_call", + name="code_interpreter", + arguments='{"code": "print(42)"}', + ) + context.response_messages.append(tool_call) + context._tool_sessions["python"] = python_tool + + output = await context.call_tool() + + assert output[0].call_id == tool_call.call_id + + # --------------------------------------------------------------------------- # Tests: finish_reason tracking # --------------------------------------------------------------------------- diff --git a/vllm/entrypoints/mcp/tool.py b/vllm/entrypoints/mcp/tool.py index d1f9130bdf..935f7f784d 100644 --- a/vllm/entrypoints/mcp/tool.py +++ b/vllm/entrypoints/mcp/tool.py @@ -173,7 +173,7 @@ class HarmonyPythonTool(Tool): processed = ResponseFunctionToolCallOutputItem( id=f"fco_{random_uuid()}", type="function_call_output", - call_id=f"call_{random_uuid()}", + call_id=last_msg.call_id, output=msg.content[0].text, status="completed", ) diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 726a955a55..1a218f1692 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -24,7 +24,6 @@ from vllm import envs from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ) -from vllm.entrypoints.generate.base.protocol import FunctionCall from vllm.entrypoints.mcp.tool import Tool from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.parser.harmony_utils import render_for_completion @@ -440,7 +439,9 @@ class ParsableContext(ConversationContext): return False async def call_python_tool( - self, tool_session: Union["ClientSession", Tool], last_msg: FunctionCall + self, + tool_session: Union["ClientSession", Tool], + last_msg: ResponseFunctionToolCall, ) -> list[ResponseInputOutputItem]: self.called_tools.add("python") if isinstance(tool_session, Tool): @@ -455,7 +456,7 @@ class ParsableContext(ConversationContext): message = ResponseFunctionToolCallOutputItem( id=f"mcpo_{random_uuid()}", type="function_call_output", - call_id=f"call_{random_uuid()}", + call_id=last_msg.call_id, output=result_str, status="completed", ) @@ -463,7 +464,9 @@ class ParsableContext(ConversationContext): return [message] async def call_search_tool( - self, tool_session: Union["ClientSession", Tool], last_msg: FunctionCall + self, + tool_session: Union["ClientSession", Tool], + last_msg: ResponseFunctionToolCall, ) -> list[ResponseInputOutputItem]: self.called_tools.add("browser") if isinstance(tool_session, Tool): @@ -481,7 +484,7 @@ class ParsableContext(ConversationContext): message = ResponseFunctionToolCallOutputItem( id=f"fco_{random_uuid()}", type="function_call_output", - call_id=f"call_{random_uuid()}", + call_id=last_msg.call_id, output=result_str, status="completed", ) @@ -489,8 +492,10 @@ class ParsableContext(ConversationContext): return [message] async def call_container_tool( - self, tool_session: Union["ClientSession", Tool], last_msg: Message - ) -> list[Message]: + self, + tool_session: Union["ClientSession", Tool], + last_msg: ResponseFunctionToolCall, + ) -> list[ResponseInputOutputItem]: """Call container tool. Expect this to be run in a stateful docker with command line terminal. The official container tool would at least @@ -523,7 +528,7 @@ class ParsableContext(ConversationContext): message = ResponseFunctionToolCallOutputItem( id=f"fco_{random_uuid()}", type="function_call_output", - call_id=f"call_{random_uuid()}", + call_id=last_msg.call_id, output=result_str, status="completed", ) From eec2a86b1ed7d76b89f7afe25594acd9b0dd88a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20P=C3=A9rez=20de=20Algaba?= <124347725+jperezdealgaba@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:00:36 +0200 Subject: [PATCH 063/248] [Security] Bump nltk, aiohttp, pillow, and datamodel-code-generator (#59249) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Pérez de Algaba Co-authored-by: Cursor --- requirements/common.txt | 4 ++-- requirements/test/cpu.txt | 16 ++++++++++------ requirements/test/cuda.in | 3 ++- requirements/test/cuda.txt | 17 +++++++++++------ requirements/test/nightly-torch.txt | 2 +- requirements/test/rocm.in | 3 ++- requirements/test/rocm.txt | 16 +++++++++++----- requirements/test/xpu.in | 1 + requirements/test/xpu.txt | 5 ++++- 9 files changed, 44 insertions(+), 23 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index 1958173fd7..669efc4900 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -15,11 +15,11 @@ protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33 protobuf == 7.36.1; platform_machine == "s390x" fastapi[standard] >= 0.133.0, < 0.137.0 # First version supporting Starlette 1.0; < 0.137.0 avoids route-tree change that breaks model-hosting-container-standards handler overrides. starlette >= 1.0.1 # CVE-2026-48710: Host header injection in < 1.0.1 -aiohttp >= 3.13.3 +aiohttp >= 3.14.3 openai >= 2.25.0 # For Responses API with namespace tools pydantic >= 2.12.0 prometheus_client >= 0.18.0 -pillow # Required for image processing +pillow >= 12.3.0 # Required for image processing prometheus-fastapi-instrumentator >= 8.0.0 # v8 unblocks starlette >= 1.0 tiktoken >= 0.6.0 # Required for DBRX tokenizer lm-format-enforcer == 0.11.3 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index e8dd282121..df685200f3 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -8,7 +8,7 @@ accelerate==1.13.0 # via peft aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.13.3 +aiohttp==3.14.3 # via # -r requirements/test/../common.txt # aiohttp-cors @@ -157,7 +157,7 @@ cupy-cuda12x==13.6.0 # via ray cycler==0.12.1 # via matplotlib -datamodel-code-generator==0.26.3 +datamodel-code-generator==0.79.0 # via -r requirements/test/cuda.in dataproperty==1.0.1 # via @@ -173,6 +173,8 @@ decorator==5.1.1 # via librosa decord==0.6.0 # via -r requirements/test/cuda.in +defusedxml==0.7.1 + # via nltk depyf==0.20.0 # via -r requirements/test/../common.txt detect-installer==0.1.0 @@ -533,8 +535,10 @@ networkx==3.2.1 # torch ninja==1.13.0 # via -r requirements/test/../common.txt -nltk==3.9.1 - # via rouge-score +nltk==3.10.3 + # via + # -r requirements/test/cuda.in + # rouge-score num2words==0.5.14 # via -r requirements/test/cuda.in numba==0.65.0 @@ -655,7 +659,6 @@ packaging==24.2 # via # accelerate # black - # datamodel-code-generator # datasets # evaluate # fastparquet @@ -696,7 +699,7 @@ perceptron==0.1.4 # via -r requirements/test/cuda.in perf-analyzer==0.1.0 # via genai-perf -pillow==10.4.0 +pillow==12.3.0 # via # -r requirements/test/../common.txt # genai-perf @@ -1231,6 +1234,7 @@ types-requests==2.33.0.20260712 typing-extensions==4.15.0 # via # -r requirements/test/../common.txt + # aiohttp # aiosignal # albumentations # alembic diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 271cdba155..87d67fa9ff 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -36,7 +36,8 @@ matplotlib # required for qwen-vl test mistral_common[image,audio] >= 1.11.6 # required for voxtral test num2words # required for smolvlm test open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py -datamodel_code_generator # required for minicpm3 test +datamodel_code_generator>=0.64.0 # required for minicpm3 test +nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test transformers==5.17.0 diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index b7711a10cf..a3ff938b55 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -8,7 +8,7 @@ accelerate==1.13.0 # via peft aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.13.3 +aiohttp==3.14.3 # via # -c requirements/common.txt # -r requirements/test/../common.txt @@ -170,7 +170,7 @@ cupy-cuda12x==13.6.0 # via ray cycler==0.12.1 # via matplotlib -datamodel-code-generator==0.26.3 +datamodel-code-generator==0.79.0 # via -r requirements/test/cuda.in dataproperty==1.0.1 # via @@ -186,6 +186,8 @@ decorator==5.1.1 # via librosa decord==0.6.0 # via -r requirements/test/cuda.in +defusedxml==0.7.1 + # via nltk depyf==0.20.0 # via # -c requirements/common.txt @@ -567,8 +569,10 @@ networkx==3.2.1 # torch ninja==1.13.0 # via -r requirements/test/../common.txt -nltk==3.9.1 - # via rouge-score +nltk==3.10.3 + # via + # -r requirements/test/cuda.in + # rouge-score num2words==0.5.14 # via -r requirements/test/cuda.in numba==0.65.0 @@ -743,7 +747,6 @@ packaging==24.2 # via # accelerate # black - # datamodel-code-generator # datasets # evaluate # fastparquet @@ -784,8 +787,9 @@ perceptron==0.1.4 # via -r requirements/test/cuda.in perf-analyzer==0.1.0 # via genai-perf -pillow==10.4.0 +pillow==12.3.0 # via + # -c requirements/common.txt # -r requirements/test/../common.txt # genai-perf # imagehash @@ -1341,6 +1345,7 @@ typing-extensions==4.15.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # aiohttp # aiosignal # albumentations # alembic diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index cebf3b782b..256683967e 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -26,7 +26,7 @@ matplotlib # required for qwen-vl test mistral_common[image,audio] >= 1.11.6 # required for voxtral test num2words # required for smolvlm test opencv-python-headless >= 4.13.0 # required for video test -datamodel_code_generator # required for minicpm3 test +datamodel_code_generator>=0.64.0 # required for minicpm3 test lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test transformers==5.17.0 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index ffe72643aa..42c2b38d0f 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -32,7 +32,8 @@ matplotlib # required for qwen-vl test mistral_common[image,audio]>=1.11.6 # required for voxtral test num2words # required for smolvlm test open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py -datamodel_code_generator # required for minicpm3 test +datamodel_code_generator>=0.64.0 # required for minicpm3 test +nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test transformers==5.17.0 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 3581e5fea9..41234d29fd 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -6,7 +6,7 @@ accelerate==1.13.0 # via peft aiohappyeyeballs==2.6.1 # via aiohttp -aiohttp==3.13.3 +aiohttp==3.14.3 # via # -c requirements/common.txt # -r requirements/test/../common.txt @@ -165,7 +165,7 @@ cryptography==46.0.0 # pyjwt cycler==0.12.1 # via matplotlib -datamodel-code-generator==0.55.0 +datamodel-code-generator==0.79.0 # via -r requirements/test/rocm.in dataproperty==1.1.0 # via @@ -181,6 +181,8 @@ decorator==5.2.1 # via librosa decord==0.6.0 # via -r requirements/test/rocm.in +defusedxml==0.7.1 + # via nltk depyf==0.20.0 # via # -c requirements/common.txt @@ -569,8 +571,10 @@ networkx==3.6.1 # torch ninja==1.13.0 # via -r requirements/test/../common.txt -nltk==3.9.3 - # via rouge-score +nltk==3.10.3 + # via + # -r requirements/test/rocm.in + # rouge-score num2words==0.5.14 # via -r requirements/test/rocm.in numba==0.65.0 @@ -758,8 +762,9 @@ perceptron==0.1.4 # via -r requirements/test/rocm.in perf-analyzer==0.1.0 # via genai-perf -pillow==12.1.1 +pillow==12.3.0 # via + # -c requirements/common.txt # -r requirements/test/../common.txt # genai-perf # imagehash @@ -1291,6 +1296,7 @@ typing-extensions==4.15.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # aiohttp # aiosignal # albumentations # alembic diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index 08a81cd20d..a5e7470d6d 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -16,6 +16,7 @@ absl-py accelerate arctic-inference lm_eval[api]>=0.4.12 +nltk>=3.10.3 # pin transitive rouge-score/lm-eval dep for known GHSAs modelscope<1.38 runai-model-streamer[s3,gcs,azure]>=0.15.7 tensorizer==2.10.1 diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 2bcd3664c1..b2a6eb6da1 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -515,7 +515,9 @@ networkx==3.6.1 ninja==1.13.0 # via -r requirements/test/../common.txt nltk==3.10.3 - # via rouge-score + # via + # -r requirements/test/xpu.in + # rouge-score num2words==0.5.14 # via -r requirements/test/xpu.in numba==0.65.0 @@ -671,6 +673,7 @@ pathvalidate==3.3.1 # via pytablewriter pillow==12.3.0 # via + # -c requirements/common.txt # -r requirements/test/../common.txt # matplotlib # mistral-common From be255076d0498c163441d4aa3fc9d2da142f30fd Mon Sep 17 00:00:00 2001 From: Harshil Date: Tue, 29 Sep 2026 16:12:15 -0700 Subject: [PATCH 064/248] [Bugfix] Reject a prefix_match_unit that a single KV cache group cannot honor (#58021) Signed-off-by: QHarshil Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../unit/test_mooncake_store_worker.py | 4 +++- vllm/v1/core/kv_cache_utils.py | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index f08213444e..1ef77e285a 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -310,7 +310,9 @@ def _make_vllm_config( kv_cache_layout: KVCacheLayout = KVCacheLayout.LBHNC, disable_hybrid_kv_cache_manager: bool = True, ) -> SimpleNamespace: - cache_config = SimpleNamespace(block_size=16, num_gpu_blocks=10) + cache_config = SimpleNamespace( + block_size=16, num_gpu_blocks=10, prefix_match_unit=None + ) cache_config.get_resolved_kv_cache_layout = lambda: kv_cache_layout return SimpleNamespace( model_config=_FakeModelConfig(), diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 3f4f4e7bb1..ddeda3ed19 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -714,7 +714,9 @@ def resolve_kv_cache_block_sizes( group's effective block size. Attention groups are scaled by DCP; Mamba groups keep their full per-rank state and are not scaled. - ``hash_block_size`` is the granularity at which ``Request.block_hashes`` - is computed. Single group: equals scheduler block size. Multiple groups: + is computed. Single group: equals scheduler block size, and any other + ``cache_config.prefix_match_unit`` is rejected while block hashing is + active. Multiple groups: ``cache_config.prefix_match_unit`` override if set, else the GCD of group block sizes; every group's block size must be divisible by it. Returns the scheduler block size (i.e. disables finer hashing) if block @@ -728,6 +730,21 @@ def resolve_kv_cache_block_sizes( if groups and not groups[0].kv_cache_spec.dcp_sharded: dcp = 1 bs = cache_config.block_size * dcp + # The Mamba prefill checkpoint builder reads prefix_match_unit directly, + # so a value dropped here puts its checkpoints off the scheduler's grid. + if ( + cache_config.prefix_match_unit not in (None, bs) + and len(groups) == 1 + and ( + cache_config.enable_prefix_caching + or vllm_config.kv_transfer_config is not None + ) + ): + raise ValueError( + f"Invalid prefix_match_unit={cache_config.prefix_match_unit}; " + "with a single KV cache group, prefix-cache hits land on the " + f"block size ({bs}). Unset it or set it to {bs}." + ) return bs, bs group_block_sizes = [ From 4e0a414c440e489d472a797ff1028cfc41565693 Mon Sep 17 00:00:00 2001 From: Lio Einaudi Date: Wed, 30 Sep 2026 07:51:50 +0800 Subject: [PATCH 065/248] [Kernel][Perf] Add TP=2/4/8 per-rank shapes to the sm_120 batch-invariant matmul table (#58495) Signed-off-by: LioEinaudi Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../determinism/batch_invariant_configs.py | 615 ++++++++++++++++++ 1 file changed, 615 insertions(+) diff --git a/vllm/model_executor/determinism/batch_invariant_configs.py b/vllm/model_executor/determinism/batch_invariant_configs.py index 874447bb82..4fff768c1f 100644 --- a/vllm/model_executor/determinism/batch_invariant_configs.py +++ b/vllm/model_executor/determinism/batch_invariant_configs.py @@ -480,6 +480,621 @@ _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[ (2048, _MatmulMConfig(64, 128, 4, 3)), ), ), + (6144, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp2_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 3)), + (4, _MatmulMConfig(16, 128, 8, 5)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(32, 128, 8, 5)), + (64, _MatmulMConfig(32, 128, 4, 4)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 5)), + (1024, _MatmulMConfig(128, 64, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2048, 3072): _MatmulShapeConfig( # Qwen3-1.7B tp2_down_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 5)), + (4, _MatmulMConfig(16, 64, 4, 3)), + (8, _MatmulMConfig(16, 64, 8, 4)), + (16, _MatmulMConfig(16, 64, 8, 3)), + (32, _MatmulMConfig(16, 64, 8, 3)), + (64, _MatmulMConfig(16, 64, 4, 5)), + (256, _MatmulMConfig(64, 64, 8, 3)), + (512, _MatmulMConfig(64, 128, 4, 3)), + (1024, _MatmulMConfig(64, 64, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (2048, 1024): _MatmulShapeConfig( # Qwen3-1.7B tp2_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 8, 2)), + (4, _MatmulMConfig(16, 32, 8, 4)), + (8, _MatmulMConfig(32, 128, 8, 5)), + (16, _MatmulMConfig(32, 128, 4, 4)), + (32, _MatmulMConfig(64, 128, 4, 3)), + (64, _MatmulMConfig(16, 256, 4, 3)), + (256, _MatmulMConfig(32, 128, 4, 3)), + (512, _MatmulMConfig(64, 128, 4, 4)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(64, 128, 4, 5)), + ), + ), + (75968, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp2_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 8, 3)), + (4, _MatmulMConfig(16, 256, 8, 3)), + (8, _MatmulMConfig(16, 256, 8, 3)), + (16, _MatmulMConfig(16, 256, 4, 3)), + (32, _MatmulMConfig(32, 256, 8, 3)), + (64, _MatmulMConfig(64, 256, 8, 3)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (3072, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp4_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 4, 3)), + (4, _MatmulMConfig(32, 128, 4, 5)), + (8, _MatmulMConfig(16, 128, 8, 3)), + (16, _MatmulMConfig(32, 128, 8, 4)), + (32, _MatmulMConfig(32, 128, 8, 5)), + (64, _MatmulMConfig(64, 32, 8, 3)), + (256, _MatmulMConfig(128, 64, 8, 3)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(64, 64, 4, 5)), + (2048, _MatmulMConfig(64, 128, 8, 4)), + ), + ), + (2048, 1536): _MatmulShapeConfig( # Qwen3-1.7B tp4_down_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 2)), + (4, _MatmulMConfig(16, 256, 4, 3)), + (8, _MatmulMConfig(32, 128, 4, 2)), + (16, _MatmulMConfig(32, 32, 8, 5)), + (32, _MatmulMConfig(32, 32, 8, 4)), + (64, _MatmulMConfig(32, 128, 4, 4)), + (256, _MatmulMConfig(64, 64, 4, 3)), + (512, _MatmulMConfig(128, 64, 8, 4)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 64, 8, 4)), + ), + ), + (1024, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp4_qkv_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(32, 128, 8, 4)), + (4, _MatmulMConfig(16, 32, 8, 4)), + (8, _MatmulMConfig(16, 32, 4, 4)), + (16, _MatmulMConfig(32, 128, 8, 3)), + (32, _MatmulMConfig(32, 128, 8, 4)), + (64, _MatmulMConfig(32, 64, 4, 5)), + (256, _MatmulMConfig(64, 32, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 5)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + (2048, 512): _MatmulShapeConfig( # Qwen3-1.7B tp4_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(32, 32, 4, 5)), + (4, _MatmulMConfig(64, 128, 4, 2)), + (8, _MatmulMConfig(16, 128, 4, 2)), + (16, _MatmulMConfig(16, 256, 8, 3)), + (32, _MatmulMConfig(64, 32, 8, 4)), + (64, _MatmulMConfig(16, 128, 8, 4)), + (256, _MatmulMConfig(64, 64, 8, 5)), + (512, _MatmulMConfig(32, 128, 8, 4)), + (1024, _MatmulMConfig(64, 64, 8, 3)), + (2048, _MatmulMConfig(64, 128, 8, 3)), + ), + ), + (37984, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp4_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 4)), + (4, _MatmulMConfig(16, 128, 8, 5)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 128, 8, 4)), + (32, _MatmulMConfig(32, 128, 4, 5)), + (64, _MatmulMConfig(64, 128, 4, 5)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (1536, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp8_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(32, 128, 8, 3)), + (4, _MatmulMConfig(16, 256, 4, 3)), + (8, _MatmulMConfig(16, 32, 8, 5)), + (16, _MatmulMConfig(32, 32, 8, 3)), + (32, _MatmulMConfig(32, 128, 8, 4)), + (64, _MatmulMConfig(16, 128, 8, 4)), + (256, _MatmulMConfig(32, 128, 8, 3)), + (512, _MatmulMConfig(64, 128, 8, 4)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(64, 64, 4, 4)), + ), + ), + (2048, 768): _MatmulShapeConfig( # Qwen3-1.7B tp8_down_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(32, 64, 4, 3)), + (4, _MatmulMConfig(32, 64, 8, 3)), + (8, _MatmulMConfig(16, 32, 4, 5)), + (16, _MatmulMConfig(32, 64, 4, 2)), + (32, _MatmulMConfig(16, 64, 8, 3)), + (64, _MatmulMConfig(16, 32, 8, 2)), + (256, _MatmulMConfig(32, 32, 4, 4)), + (512, _MatmulMConfig(32, 64, 4, 5)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (512, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp8_qkv_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(64, 64, 8, 5)), + (4, _MatmulMConfig(16, 32, 4, 4)), + (8, _MatmulMConfig(16, 32, 4, 5)), + (16, _MatmulMConfig(32, 128, 8, 3)), + (32, _MatmulMConfig(64, 64, 8, 5)), + (64, _MatmulMConfig(32, 128, 8, 4)), + (256, _MatmulMConfig(16, 128, 4, 4)), + (512, _MatmulMConfig(32, 64, 8, 3)), + (1024, _MatmulMConfig(64, 64, 4, 4)), + (2048, _MatmulMConfig(64, 128, 4, 4)), + ), + ), + (2048, 256): _MatmulShapeConfig( # Qwen3-1.7B tp8_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(32, 128, 8, 2)), + (4, _MatmulMConfig(16, 128, 4, 2)), + (8, _MatmulMConfig(32, 256, 4, 2)), + (16, _MatmulMConfig(32, 32, 8, 4)), + (32, _MatmulMConfig(16, 256, 4, 3)), + (64, _MatmulMConfig(128, 32, 8, 3)), + (256, _MatmulMConfig(16, 32, 8, 2)), + (512, _MatmulMConfig(16, 32, 4, 3)), + (1024, _MatmulMConfig(64, 256, 4, 3)), + (2048, _MatmulMConfig(64, 128, 8, 4)), + ), + ), + (18992, 2048): _MatmulShapeConfig( # Qwen3-1.7B tp8_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 5)), + (4, _MatmulMConfig(16, 128, 8, 5)), + (8, _MatmulMConfig(16, 128, 8, 3)), + (16, _MatmulMConfig(16, 128, 4, 5)), + (32, _MatmulMConfig(32, 128, 8, 5)), + (64, _MatmulMConfig(64, 128, 8, 4)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (9728, 2560): _MatmulShapeConfig( # Qwen3-4B tp2_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 4)), + (4, _MatmulMConfig(16, 64, 8, 3)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(32, 64, 8, 4)), + (64, _MatmulMConfig(32, 128, 4, 3)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 5)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2560, 4864): _MatmulShapeConfig( # Qwen3-4B tp2_down_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 32, 4, 3)), + (4, _MatmulMConfig(16, 128, 8, 3)), + (8, _MatmulMConfig(16, 64, 8, 4)), + (16, _MatmulMConfig(32, 64, 8, 4)), + (32, _MatmulMConfig(16, 64, 4, 3)), + (64, _MatmulMConfig(32, 32, 8, 4)), + (256, _MatmulMConfig(64, 64, 4, 3)), + (512, _MatmulMConfig(64, 128, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (3072, 2560): _MatmulShapeConfig( # Qwen3-4B tp2_qkv_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 3)), + (4, _MatmulMConfig(16, 32, 8, 2)), + (8, _MatmulMConfig(16, 32, 8, 4)), + (16, _MatmulMConfig(16, 64, 4, 3)), + (32, _MatmulMConfig(32, 64, 4, 4)), + (64, _MatmulMConfig(16, 128, 4, 3)), + (256, _MatmulMConfig(64, 128, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (2560, 2048): _MatmulShapeConfig( # Qwen3-4B tp2_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(32, 128, 8, 4)), + (4, _MatmulMConfig(32, 128, 4, 4)), + (8, _MatmulMConfig(16, 128, 4, 3)), + (16, _MatmulMConfig(32, 128, 4, 5)), + (32, _MatmulMConfig(16, 128, 8, 4)), + (64, _MatmulMConfig(16, 128, 8, 4)), + (256, _MatmulMConfig(128, 32, 8, 5)), + (512, _MatmulMConfig(128, 64, 8, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + (75968, 2560): _MatmulShapeConfig( # Qwen3-4B tp2_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 5)), + (4, _MatmulMConfig(16, 64, 8, 5)), + (8, _MatmulMConfig(16, 64, 4, 5)), + (16, _MatmulMConfig(16, 64, 4, 5)), + (32, _MatmulMConfig(32, 64, 4, 5)), + (64, _MatmulMConfig(64, 256, 4, 3)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + (4864, 2560): _MatmulShapeConfig( # Qwen3-4B tp4_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 5)), + (4, _MatmulMConfig(16, 128, 4, 4)), + (8, _MatmulMConfig(16, 32, 8, 4)), + (16, _MatmulMConfig(16, 128, 8, 4)), + (32, _MatmulMConfig(16, 128, 4, 5)), + (64, _MatmulMConfig(16, 128, 4, 3)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2560, 2432): _MatmulShapeConfig( # Qwen3-4B tp4_down_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 4)), + (4, _MatmulMConfig(16, 32, 4, 5)), + (8, _MatmulMConfig(32, 64, 8, 4)), + (16, _MatmulMConfig(32, 128, 8, 5)), + (32, _MatmulMConfig(16, 128, 4, 4)), + (64, _MatmulMConfig(64, 32, 4, 3)), + (256, _MatmulMConfig(32, 128, 8, 4)), + (512, _MatmulMConfig(64, 128, 4, 5)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (1536, 2560): _MatmulShapeConfig( # Qwen3-4B tp4_qkv_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 4)), + (4, _MatmulMConfig(16, 32, 8, 3)), + (8, _MatmulMConfig(16, 32, 4, 4)), + (16, _MatmulMConfig(32, 128, 8, 3)), + (32, _MatmulMConfig(16, 128, 4, 3)), + (64, _MatmulMConfig(16, 64, 8, 4)), + (256, _MatmulMConfig(128, 32, 4, 5)), + (512, _MatmulMConfig(64, 128, 4, 5)), + (1024, _MatmulMConfig(64, 256, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 4)), + ), + ), + (2560, 1024): _MatmulShapeConfig( # Qwen3-4B tp4_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(32, 128, 8, 5)), + (4, _MatmulMConfig(16, 32, 8, 3)), + (8, _MatmulMConfig(16, 128, 4, 3)), + (16, _MatmulMConfig(32, 256, 4, 3)), + (32, _MatmulMConfig(16, 128, 4, 2)), + (64, _MatmulMConfig(32, 256, 8, 3)), + (256, _MatmulMConfig(32, 128, 4, 5)), + (512, _MatmulMConfig(64, 128, 8, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (37984, 2560): _MatmulShapeConfig( # Qwen3-4B tp4_lm_head + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 3)), + (4, _MatmulMConfig(16, 128, 8, 3)), + (8, _MatmulMConfig(16, 128, 4, 3)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(16, 32, 4, 5)), + (64, _MatmulMConfig(32, 32, 4, 5)), + (256, _MatmulMConfig(64, 128, 4, 3)), + (512, _MatmulMConfig(64, 128, 4, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (2432, 2560): _MatmulShapeConfig( # Qwen3-4B tp8_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 4, 5)), + (4, _MatmulMConfig(32, 128, 4, 3)), + (8, _MatmulMConfig(16, 128, 8, 3)), + (16, _MatmulMConfig(16, 128, 4, 4)), + (32, _MatmulMConfig(16, 128, 8, 4)), + (64, _MatmulMConfig(16, 128, 4, 3)), + (256, _MatmulMConfig(64, 64, 4, 5)), + (512, _MatmulMConfig(64, 128, 4, 4)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (2560, 1216): _MatmulShapeConfig( # Qwen3-4B tp8_down_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 8, 2)), + (4, _MatmulMConfig(16, 128, 8, 4)), + (8, _MatmulMConfig(16, 32, 4, 3)), + (16, _MatmulMConfig(32, 32, 4, 3)), + (32, _MatmulMConfig(16, 128, 8, 4)), + (64, _MatmulMConfig(32, 128, 8, 4)), + (256, _MatmulMConfig(32, 128, 4, 5)), + (512, _MatmulMConfig(64, 128, 8, 4)), + (1024, _MatmulMConfig(64, 256, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (768, 2560): _MatmulShapeConfig( # Qwen3-4B tp8_qkv_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(32, 32, 4, 2)), + (4, _MatmulMConfig(16, 64, 8, 3)), + (8, _MatmulMConfig(16, 64, 8, 4)), + (16, _MatmulMConfig(16, 64, 4, 5)), + (32, _MatmulMConfig(32, 64, 8, 4)), + (64, _MatmulMConfig(16, 64, 8, 4)), + (256, _MatmulMConfig(64, 32, 4, 5)), + (512, _MatmulMConfig(64, 64, 8, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 64, 4, 3)), + ), + ), + (2560, 512): _MatmulShapeConfig( # Qwen3-4B tp8_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 4)), + (4, _MatmulMConfig(16, 32, 4, 2)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(32, 32, 4, 5)), + (32, _MatmulMConfig(32, 256, 4, 3)), + (64, _MatmulMConfig(32, 128, 8, 5)), + (256, _MatmulMConfig(16, 256, 4, 2)), + (512, _MatmulMConfig(32, 128, 8, 4)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (18992, 2560): _MatmulShapeConfig( # Qwen3-4B tp8_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 4)), + (4, _MatmulMConfig(16, 64, 8, 5)), + (8, _MatmulMConfig(16, 128, 8, 3)), + (16, _MatmulMConfig(16, 128, 8, 5)), + (32, _MatmulMConfig(32, 128, 8, 5)), + (64, _MatmulMConfig(64, 128, 8, 4)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (12288, 4096): _MatmulShapeConfig( # Qwen3-8B tp2_gate_up_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 3)), + (4, _MatmulMConfig(16, 128, 8, 5)), + (8, _MatmulMConfig(16, 128, 8, 4)), + (16, _MatmulMConfig(16, 128, 4, 4)), + (32, _MatmulMConfig(32, 128, 4, 4)), + (64, _MatmulMConfig(64, 128, 4, 5)), + (256, _MatmulMConfig(64, 64, 4, 5)), + (512, _MatmulMConfig(128, 64, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (4096, 6144): _MatmulShapeConfig( # Qwen3-8B tp2_down_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 5)), + (4, _MatmulMConfig(16, 32, 8, 5)), + (8, _MatmulMConfig(16, 32, 8, 3)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(16, 64, 8, 4)), + (64, _MatmulMConfig(32, 64, 4, 5)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 64, 8, 4)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (3072, 4096): _MatmulShapeConfig( # Qwen3-8B tp2_qkv_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 4)), + (4, _MatmulMConfig(16, 32, 8, 3)), + (8, _MatmulMConfig(16, 128, 4, 3)), + (16, _MatmulMConfig(16, 32, 4, 4)), + (32, _MatmulMConfig(32, 32, 8, 5)), + (64, _MatmulMConfig(16, 128, 8, 4)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 8, 3)), + (1024, _MatmulMConfig(64, 64, 4, 5)), + (2048, _MatmulMConfig(128, 64, 4, 3)), + ), + ), + (75968, 4096): _MatmulShapeConfig( # Qwen3-8B tp2_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 8, 3)), + (4, _MatmulMConfig(16, 256, 8, 3)), + (8, _MatmulMConfig(16, 256, 8, 3)), + (16, _MatmulMConfig(16, 256, 8, 3)), + (32, _MatmulMConfig(32, 256, 8, 3)), + (64, _MatmulMConfig(64, 256, 8, 3)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 8, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (4096, 3072): _MatmulShapeConfig( # Qwen3-8B tp4_down_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 8, 3)), + (4, _MatmulMConfig(16, 128, 4, 5)), + (8, _MatmulMConfig(16, 128, 4, 4)), + (16, _MatmulMConfig(16, 64, 4, 5)), + (32, _MatmulMConfig(32, 128, 4, 3)), + (64, _MatmulMConfig(16, 128, 4, 4)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 64, 8, 4)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (1536, 4096): _MatmulShapeConfig( # Qwen3-8B tp4_qkv_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 4, 5)), + (4, _MatmulMConfig(16, 64, 8, 3)), + (8, _MatmulMConfig(32, 64, 4, 5)), + (16, _MatmulMConfig(16, 64, 8, 4)), + (32, _MatmulMConfig(16, 32, 8, 3)), + (64, _MatmulMConfig(16, 64, 8, 5)), + (256, _MatmulMConfig(64, 64, 4, 3)), + (512, _MatmulMConfig(64, 128, 4, 3)), + (1024, _MatmulMConfig(64, 64, 4, 3)), + (2048, _MatmulMConfig(64, 128, 4, 3)), + ), + ), + (4096, 1024): _MatmulShapeConfig( # Qwen3-8B tp4_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 4)), + (4, _MatmulMConfig(32, 128, 8, 4)), + (8, _MatmulMConfig(16, 256, 8, 3)), + (16, _MatmulMConfig(16, 128, 8, 3)), + (32, _MatmulMConfig(16, 128, 4, 4)), + (64, _MatmulMConfig(64, 64, 4, 3)), + (256, _MatmulMConfig(128, 64, 4, 3)), + (512, _MatmulMConfig(64, 64, 4, 5)), + (1024, _MatmulMConfig(128, 64, 4, 4)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (37984, 4096): _MatmulShapeConfig( # Qwen3-8B tp4_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 3)), + (4, _MatmulMConfig(16, 128, 4, 5)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 128, 8, 3)), + (32, _MatmulMConfig(32, 128, 8, 3)), + (64, _MatmulMConfig(64, 128, 8, 3)), + (256, _MatmulMConfig(64, 128, 4, 5)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), + (4096, 1536): _MatmulShapeConfig( # Qwen3-8B tp8_down_proj + block_k=32, + m_buckets=( + (1, _MatmulMConfig(32, 128, 8, 5)), + (4, _MatmulMConfig(16, 256, 4, 5)), + (8, _MatmulMConfig(16, 64, 4, 4)), + (16, _MatmulMConfig(16, 128, 8, 3)), + (32, _MatmulMConfig(16, 64, 4, 3)), + (64, _MatmulMConfig(16, 128, 4, 4)), + (256, _MatmulMConfig(64, 128, 4, 4)), + (512, _MatmulMConfig(128, 128, 8, 5)), + (1024, _MatmulMConfig(128, 256, 8, 4)), + (2048, _MatmulMConfig(128, 128, 8, 4)), + ), + ), + (768, 4096): _MatmulShapeConfig( # Qwen3-8B tp8_qkv_proj + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 32, 8, 5)), + (4, _MatmulMConfig(16, 32, 8, 3)), + (8, _MatmulMConfig(32, 64, 4, 5)), + (16, _MatmulMConfig(32, 64, 8, 3)), + (32, _MatmulMConfig(32, 32, 8, 5)), + (64, _MatmulMConfig(64, 32, 8, 3)), + (256, _MatmulMConfig(32, 64, 4, 4)), + (512, _MatmulMConfig(64, 64, 8, 3)), + (1024, _MatmulMConfig(64, 128, 4, 3)), + (2048, _MatmulMConfig(64, 64, 4, 3)), + ), + ), + (4096, 512): _MatmulShapeConfig( # Qwen3-8B tp8_o_proj + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 2)), + (4, _MatmulMConfig(32, 128, 4, 2)), + (8, _MatmulMConfig(16, 256, 4, 2)), + (16, _MatmulMConfig(32, 256, 4, 3)), + (32, _MatmulMConfig(32, 256, 8, 3)), + (64, _MatmulMConfig(32, 128, 4, 5)), + (256, _MatmulMConfig(64, 32, 8, 4)), + (512, _MatmulMConfig(128, 128, 8, 3)), + (1024, _MatmulMConfig(128, 64, 4, 4)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + (18992, 4096): _MatmulShapeConfig( # Qwen3-8B tp8_lm_head + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 64, 8, 5)), + (4, _MatmulMConfig(16, 64, 8, 4)), + (8, _MatmulMConfig(16, 64, 8, 5)), + (16, _MatmulMConfig(16, 128, 4, 3)), + (32, _MatmulMConfig(32, 64, 8, 5)), + (64, _MatmulMConfig(64, 128, 8, 3)), + (256, _MatmulMConfig(128, 128, 4, 3)), + (512, _MatmulMConfig(128, 128, 4, 3)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 4, 3)), + ), + ), }, } From 7ca31ef5993dc4426d912f32b8eb3cd70fb7a078 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Tue, 29 Sep 2026 20:16:05 -0400 Subject: [PATCH 066/248] [Bugfix][Responses API] Build streamed final response from streamed items (#59307) Signed-off-by: sfeng33 <4florafeng@gmail.com> --- .../entrypoints/openai/responses/conftest.py | 5 ++ .../responses/test_serving_responses.py | 58 +++++++++++++ vllm/entrypoints/openai/responses/context.py | 3 + vllm/entrypoints/openai/responses/serving.py | 77 +++++++---------- .../openai/responses/streaming_events.py | 83 ++++++++++++------- 5 files changed, 151 insertions(+), 75 deletions(-) diff --git a/tests/entrypoints/openai/responses/conftest.py b/tests/entrypoints/openai/responses/conftest.py index 5bba59781f..7ee4629df3 100644 --- a/tests/entrypoints/openai/responses/conftest.py +++ b/tests/entrypoints/openai/responses/conftest.py @@ -126,6 +126,11 @@ def _validate_event_pairing(events: list, pairs_of_event_types: dict[str, str]) etype = event.type if etype in end_events: expected_start = pairs_of_event_types[etype] + # Deltas are optional, e.g. a function call with no arguments. + if expected_start.endswith(".delta") and ( + not stack or stack[-1] != expected_start + ): + continue assert stack and stack[-1] == expected_start, ( f"Stack mismatch for {etype}: " f"expected {expected_start}, " diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index c06e2b5915..33e5eaa323 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -64,6 +64,7 @@ from vllm.entrypoints.openai.responses.streaming_events import ( ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.inputs import tokens_input +from vllm.logprobs import Logprob as SampleLogprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser.harmony import Segment from vllm.renderers import TokenizeParams @@ -1982,6 +1983,63 @@ class TestAutoToolStreaming: assert function_done[0].item.arguments == tool_args +@pytest.mark.skip_global_cleanup +@pytest.mark.asyncio +async def test_stream_completed_response_reuses_streamed_items(monkeypatch): + """response.completed must carry the streamed items (same ids, + call_id, logprobs) rather than a reparse of the full output.""" + monkeypatch.setattr(envs, "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", False) + serving = _make_serving_instance_with_reasoning() + call = DeltaToolCall( + id="chatcmpl-tool-parser-id", + index=0, + function=DeltaFunctionCall(name="get_weather", arguments="{}"), + ) + response_parser = _mock_parser_with_reasoning( + serving, + [ + DeltaMessage(reasoning="think"), + DeltaMessage(content="Hi"), + DeltaMessage(tool_calls=[call]), + ], + ) + response_parser.count_reasoning_tokens = MagicMock(return_value=1) + context = SimpleContext(response_parser=response_parser) + + async def result_generator(): + for token_id, text in [(10, "think"), (20, "Hi"), (30, "call")]: + output = _make_request_output(text, [token_id]) + output.outputs[0].logprobs = [ + {token_id: SampleLogprob(logprob=-0.5, decoded_token=text)} + ] + context.append_output(output) + yield context + + events = [ + event + async for event in serving.responses_stream_generator( + request=ResponsesRequest( + input="hi", + include=["message.output_text.logprobs"], + stream=True, + store=False, + ), + sampling_params=SamplingParams(max_tokens=16), + result_generator=result_generator(), + context=context, + model_name="test-model", + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="req"), + ) + ] + + streamed = [e.item for e in events if e.type == "response.output_item.done"] + assert events[-1].response.output == streamed + _, message, function_call = streamed + assert [lp.token for lp in message.content[0].logprobs] == ["Hi"] + assert function_call.call_id == "chatcmpl-tool-parser-id" + + @pytest.mark.asyncio @pytest.mark.parametrize(("parallel_tool_calls", "num_calls"), [(False, 1), (None, 3)]) async def test_streaming_parallel_tool_calls( diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 1a218f1692..e71af75a0e 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -194,6 +194,9 @@ class SimpleContext(ConversationContext): self._accumulated_text: str = "" self._accumulated_token_ids: list[int] = [] self._accumulated_logprobs: list = [] + # Items already sent as output_item.done events; the final + # streaming response reuses them instead of reparsing. + self.streamed_output_items: list[ResponseOutputItem] | None = None self.num_prompt_tokens = 0 self.num_output_tokens = 0 diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 4facd2fb3b..34f6fb44a2 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -15,7 +15,6 @@ from openai.types.responses import ( ResponseOutputMessage, ResponseOutputText, ResponseStatus, - response_text_delta_event, ) from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob from pydantic import TypeAdapter @@ -813,14 +812,19 @@ class OpenAIServingResponses(GenerateBaseServing): if final_output.finish_reason == "length": status = "incomplete" - # TODO: Build final response items from the accumulated streaming - # parser results instead of reparsing the complete output. - output = self._make_response_output_items( - request, - final_output, - tokenizer, - parser=context.response_parser, + streamed_items = context.streamed_output_items + self._log_final_output( + request, final_output, is_streaming=streamed_items is not None ) + if streamed_items is not None: + output = streamed_items + else: + output = self._make_response_output_items( + request, + final_output, + tokenizer, + parser=context.response_parser, + ) if request.enable_response_messages: input_messages = context.input_messages @@ -962,32 +966,21 @@ class OpenAIServingResponses(GenerateBaseServing): ) return out - def _create_stream_response_logprobs( + def _log_final_output( self, - token_ids: Sequence[int], - logprobs: SampleLogprobs | None, - tokenizer: TokenizerLike, - top_logprobs: int | None = None, - ) -> list[response_text_delta_event.Logprob]: - lgs = self._create_response_logprobs( - token_ids=token_ids, - logprobs=logprobs, - tokenizer=tokenizer, - top_logprobs=top_logprobs, - ) - return [ - response_text_delta_event.Logprob( - token=lg.token, - logprob=lg.logprob, - top_logprobs=[ - response_text_delta_event.LogprobTopLogprob( - token=tl.token, logprob=tl.logprob - ) - for tl in lg.top_logprobs - ], + request: ResponsesRequest, + final_output: CompletionOutput, + is_streaming: bool, + ) -> None: + if self.enable_log_outputs and self.request_logger: + self.request_logger.log_outputs( + request_id=request.request_id, + outputs=final_output.text, + output_token_ids=final_output.token_ids, + finish_reason=final_output.finish_reason, + is_streaming=is_streaming, + delta=False, ) - for lg in lgs - ] def _make_response_output_items( self, @@ -996,17 +989,6 @@ class OpenAIServingResponses(GenerateBaseServing): tokenizer: TokenizerLike, parser: Parser | None = None, ) -> list[ResponseOutputItem]: - # Log complete response if output logging is enabled - if self.enable_log_outputs and self.request_logger: - self.request_logger.log_outputs( - request_id=request.request_id, - outputs=final_output.text, - output_token_ids=final_output.token_ids, - finish_reason=final_output.finish_reason, - is_streaming=False, - delta=False, - ) - # Compute logprobs if requested logprobs = None if request.is_include_output_logprobs() and final_output.logprobs: @@ -1197,14 +1179,12 @@ class OpenAIServingResponses(GenerateBaseServing): hide_stream_metadata = not request.include_reasoning and self.parser is not None - def _get_logprobs( - output: CompletionOutput, - ) -> list[response_text_delta_event.Logprob]: + def _get_logprobs(output: CompletionOutput) -> list[Logprob]: if not request.is_include_output_logprobs(): return [] if hide_stream_metadata: return [] - return self._create_stream_response_logprobs( + return self._create_response_logprobs( token_ids=output.token_ids, logprobs=output.logprobs, tokenizer=tokenizer, @@ -1253,6 +1233,9 @@ class OpenAIServingResponses(GenerateBaseServing): for event in processor.close_current(): yield _increment_sequence_number_and_return(event) + assert isinstance(context, SimpleContext) + context.streamed_output_items = processor.output_items + async def _process_harmony_streaming_events( self, request: ResponsesRequest, diff --git a/vllm/entrypoints/openai/responses/streaming_events.py b/vllm/entrypoints/openai/responses/streaming_events.py index 4651bb7205..ae9fc2b25e 100644 --- a/vllm/entrypoints/openai/responses/streaming_events.py +++ b/vllm/entrypoints/openai/responses/streaming_events.py @@ -38,6 +38,7 @@ from openai.types.responses import ( ResponseMcpCallArgumentsDoneEvent, ResponseMcpCallCompletedEvent, ResponseMcpCallInProgressEvent, + ResponseOutputItem, ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent, ResponseOutputMessage, @@ -51,6 +52,7 @@ from openai.types.responses import ( ResponseWebSearchCallInProgressEvent, ResponseWebSearchCallSearchingEvent, response_function_web_search, + response_output_text, response_text_delta_event, ) from openai.types.responses.response_output_item import McpCall @@ -60,6 +62,7 @@ from openai.types.responses.response_reasoning_item import ( from openai.types.responses.tool import Tool from openai_harmony import Message as HarmonyMessage +from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.generate.base.protocol import DeltaMessage, DeltaToolCall from vllm.entrypoints.mcp.tool_server import ToolServer from vllm.entrypoints.openai.parser.harmony_utils import ( @@ -816,11 +819,13 @@ class SimpleStreamingState: current_item_id: str = "" content_index: int = 0 accumulated_text: str = "" + accumulated_logprobs: list[response_output_text.Logprob] = field( + default_factory=list + ) tool_call_id: str = "" tool_call_name: str = "" tool_call_namespace: str | None = None tool_call_index: int | None = None - has_emitted_tool_call_delta: bool = False current_state: _StateType = field(default_factory=lambda: _StateType.NONE) @@ -828,9 +833,10 @@ def emit_simple_content_open( state: SimpleStreamingState, ) -> list[StreamingResponsesResponse]: state.current_state = _StateType.CONTENT - state.current_item_id = random_uuid() + state.current_item_id = f"msg_{random_uuid()}" state.content_index = 0 state.accumulated_text = "" + state.accumulated_logprobs = [] return [ ResponseOutputItemAddedEvent( type="response.output_item.added", @@ -860,12 +866,32 @@ def emit_simple_content_open( ] +def _to_stream_logprobs( + logprobs: list[response_output_text.Logprob], +) -> list[response_text_delta_event.Logprob]: + return [ + response_text_delta_event.Logprob( + token=lp.token, + logprob=lp.logprob, + top_logprobs=[ + response_text_delta_event.LogprobTopLogprob( + token=tl.token, logprob=tl.logprob + ) + for tl in lp.top_logprobs + ], + ) + for lp in logprobs + ] + + def emit_simple_content_delta( state: SimpleStreamingState, delta: str, - logprobs: list[response_text_delta_event.Logprob] | None = None, + logprobs: list[response_output_text.Logprob] | None = None, ) -> list[StreamingResponsesResponse]: state.accumulated_text += delta + if logprobs: + state.accumulated_logprobs.extend(logprobs) return [ ResponseTextDeltaEvent( type="response.output_text.delta", @@ -874,7 +900,7 @@ def emit_simple_content_delta( output_index=state.output_index, item_id=state.current_item_id, delta=delta, - logprobs=logprobs or [], + logprobs=_to_stream_logprobs(logprobs or []), ) ] @@ -886,6 +912,7 @@ def emit_simple_content_done( type="output_text", text=state.accumulated_text, annotations=[], + logprobs=state.accumulated_logprobs or None, ) events: list[StreamingResponsesResponse] = [ ResponseTextDoneEvent( @@ -915,7 +942,6 @@ def emit_simple_content_done( role="assistant", content=[part] if state.accumulated_text else [], status="completed", - summary=[], ), ), ] @@ -928,7 +954,7 @@ def emit_simple_reasoning_open( state: SimpleStreamingState, ) -> list[StreamingResponsesResponse]: state.current_state = _StateType.REASONING - state.current_item_id = random_uuid() + state.current_item_id = f"rs_{random_uuid()}" state.content_index = 0 state.accumulated_text = "" return [ @@ -1021,15 +1047,15 @@ def emit_simple_tool_call_open( name: str, index: int | None, namespace: str | None = None, + call_id: str | None = None, ) -> list[StreamingResponsesResponse]: state.current_state = _StateType.TOOL_CALL - state.current_item_id = random_uuid() - state.tool_call_id = f"call_{random_uuid()}" + state.current_item_id = f"fc_{random_uuid()}" + state.tool_call_id = call_id or make_tool_call_id() state.tool_call_name = name state.tool_call_namespace = namespace state.tool_call_index = index state.accumulated_text = "" - state.has_emitted_tool_call_delta = False return [ ResponseOutputItemAddedEvent( type="response.output_item.added", @@ -1053,7 +1079,6 @@ def emit_simple_tool_call_delta( delta: str, ) -> list[StreamingResponsesResponse]: state.accumulated_text += delta - state.has_emitted_tool_call_delta = True return [ ResponseFunctionCallArgumentsDeltaEvent( type="response.function_call_arguments.delta", @@ -1068,19 +1093,15 @@ def emit_simple_tool_call_delta( def emit_simple_tool_call_done( state: SimpleStreamingState, ) -> list[StreamingResponsesResponse]: - events: list[StreamingResponsesResponse] = [] - if state.has_emitted_tool_call_delta: - events.append( - ResponseFunctionCallArgumentsDoneEvent( - type="response.function_call_arguments.done", - sequence_number=-1, - output_index=state.output_index, - item_id=state.current_item_id, - arguments=state.accumulated_text, - name=state.tool_call_name, - ) - ) - events.append( + events: list[StreamingResponsesResponse] = [ + ResponseFunctionCallArgumentsDoneEvent( + type="response.function_call_arguments.done", + sequence_number=-1, + output_index=state.output_index, + item_id=state.current_item_id, + arguments=state.accumulated_text, + name=state.tool_call_name, + ), ResponseOutputItemDoneEvent( type="response.output_item.done", sequence_number=-1, @@ -1095,7 +1116,7 @@ def emit_simple_tool_call_done( call_id=state.tool_call_id, ), ), - ) + ] state.output_index += 1 state.tool_call_namespace = None state.current_state = _StateType.NONE @@ -1181,6 +1202,7 @@ class SimpleStreamingEventProcessor: ) -> None: self.state = state or SimpleStreamingState() self.tool_call_name_map = build_responses_tool_call_name_map(tools) + self.output_items: list[ResponseOutputItem] = [] def resolve_target_state( self, delta_message: DeltaMessage @@ -1226,7 +1248,13 @@ class SimpleStreamingEventProcessor: handlers = self._STATE_HANDLERS.get(self.state.current_state) if handlers is None: return [] - return handlers.done_fn(self.state) + events = handlers.done_fn(self.state) + self.output_items.extend( + event.item + for event in events + if isinstance(event, ResponseOutputItemDoneEvent) + ) + return events def open( self, target_state: _StateType, tool_call: Any = None @@ -1244,6 +1272,7 @@ class SimpleStreamingEventProcessor: call_name.name, tool_call.index, call_name.namespace, + tool_call.id, ) return handlers.open_fn(self.state) @@ -1251,9 +1280,7 @@ class SimpleStreamingEventProcessor: self, delta_message: DeltaMessage, output: CompletionOutput, - get_logprobs: Callable[ - [CompletionOutput], list[response_text_delta_event.Logprob] - ] + get_logprobs: Callable[[CompletionOutput], list[response_output_text.Logprob]] | None = None, ) -> list[StreamingResponsesResponse]: """Emit incremental events for the current state from the delta.""" From 09c47db1ca080793dc2351144cb39513fd7984ca Mon Sep 17 00:00:00 2001 From: xiangdong <40376367+zxd1997066@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:29:59 +0800 Subject: [PATCH 067/248] [XPU][CI]Skip test_abort_timeout_on_prefiller in nightly (#58307) Signed-off-by: zengxian Co-authored-by: Kunshang Ji --- .buildkite/intel_jobs/test-intel.yaml | 2 +- .buildkite/scripts/hardware_ci/run-intel-ci-test.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 30e14bd384..7b9bd1e873 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -106,7 +106,7 @@ steps: pytest -v -s v1/test_serial_utils.py && pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --deselect "v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]" --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py' + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]' parallelism: 2 - label: "XPU server test" key: xpu-server-test diff --git a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh index d71ee9cf62..997244bf59 100644 --- a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh @@ -44,7 +44,7 @@ case "${test_suite}" in pytest -v -s v1/test_serial_utils.py pytest -v -s v1/e2e/general/test_correctness_sliding_window.py pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray] ;; server) pip install av @@ -57,7 +57,7 @@ case "${test_suite}" in cd tests pytest -v -s quantization/test_auto_round.py - pytest -v -s quantization/test_online.py + pytest -v -s quantization/test_online.py --deselect=tests/quantization/test_online.py::test_online_quantization_loads_real_weights ;; compressed-tensors-fp8) cd tests From 765872e7ed5b6f372f0898a67e48d3365fa28139 Mon Sep 17 00:00:00 2001 From: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:24:59 -0400 Subject: [PATCH 068/248] [Core][BugFix] Tag prefix-cache extra keys by source (#51899) Signed-off-by: Nick Hill Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Tai An Co-authored-by: Nick Hill Co-authored-by: Claude Opus 5.5 --- tests/v1/core/test_kv_cache_utils.py | 80 +++++++++++++++---- tests/v1/core/test_prefix_caching.py | 20 +++-- vllm/distributed/kv_events.py | 8 +- .../kv_connector/v1/offloading/events.py | 5 +- vllm/v1/core/block_pool.py | 5 +- vllm/v1/core/kv_cache_utils.py | 51 +++++++++--- vllm/v1/simple_kv_offload/manager.py | 5 +- 7 files changed, 131 insertions(+), 43 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index c2c86053f0..478707af9d 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -54,6 +54,7 @@ from vllm.v1.core.kv_cache_utils import ( is_kv_cache_spec_uniform, make_block_hash_with_group_id, tensor_data, + to_event_extra_keys, ) from vllm.v1.hisparse.layout import ( create_hisparse_layout, @@ -822,12 +823,12 @@ def test_generate_block_hash_extra_keys(): # Test with no extra keys extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 5, 0) - assert extra_keys == (("hash1", 0),) + assert extra_keys == (("mm", "hash1", 0),) assert next_mm_idx == 1 # Test with partial overlap extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 3, 8, 0) - assert extra_keys == (("hash1", -3),) + assert extra_keys == (("mm", "hash1", -3),) assert next_mm_idx == 1 # Test with no overlap @@ -837,7 +838,7 @@ def test_generate_block_hash_extra_keys(): # Test with multiple extra keys extra_keys, next_mm_idx = generate_block_hash_extra_keys(request, 0, 15, 0) - assert extra_keys == (("hash1", 0), ("hash2", 10)) + assert extra_keys == (("mm", "hash1", 0), ("mm", "hash2", 10)) assert next_mm_idx == 2 @@ -865,9 +866,9 @@ def test_generate_block_hash_extra_keys_cache_salt(): # salt is added for the first token extra_keys, _ = generate_block_hash_extra_keys(request, 0, 1, 0) - assert extra_keys == ("salt",) + assert extra_keys == (("cache_salt", "salt"),) extra_keys, _ = generate_block_hash_extra_keys(request, 0, 10, 0) - assert extra_keys == ("salt",) + assert extra_keys == (("cache_salt", "salt"),) # no salt added for other tokens extra_keys, _ = generate_block_hash_extra_keys(request, 1, 2, 0) @@ -888,7 +889,7 @@ def test_generate_block_hash_extra_keys_cache_salt(): # Test with no extra keys extra_keys, next_mm_idx = generate_block_hash_extra_keys(request_mm, 0, 5, 0) - assert extra_keys == (("hash1", 0), "salt") + assert extra_keys == (("mm", "hash1", 0), ("cache_salt", "salt")) assert next_mm_idx == 1 @@ -906,13 +907,13 @@ def test_generate_block_hash_extra_keys_prompt_embeds(): extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0) expected_embeds = prompt_embeds[0:5] expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest() - assert extra_keys == (expected_hash,) + assert extra_keys == (("prompt_embeds", expected_hash),) # Test with prompt embeds for the second block extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0) expected_embeds = prompt_embeds[5:10] expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest() - assert extra_keys == (expected_hash,) + assert extra_keys == (("prompt_embeds", expected_hash),) def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch): @@ -976,13 +977,58 @@ def test_generate_block_hash_extra_keys_lora(): ) extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0) - assert extra_keys == ("test_lora_adapter",) + assert extra_keys == (("lora", "test_lora_adapter"),) request.lora_request = None extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0) assert extra_keys is None +@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) +def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn): + """A cache_salt equal to a LoRA name must not share that LoRA's blocks.""" + lora_request = LoRARequest( + lora_name="foo", lora_int_id=1, lora_path="/path/to/lora" + ) + lora_req = Request( + request_id="lora", + prompt_token_ids=[0, 1, 2], + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + lora_request=lora_request, + block_hasher=get_request_block_hasher(3, hash_fn), + ) + salted_req = make_request("salted", [0, 1, 2], hash_fn=hash_fn, cache_salt="foo") + + assert lora_req.block_hashes[0] != salted_req.block_hashes[0] + + +def test_to_event_extra_keys_keeps_untagged_event_format(): + """KV events keep publishing the extra-key shapes consumers already parse.""" + request = make_request( + "0", + list(range(10)), + mm_positions=[PlaceholderRange(offset=2, length=3)], + mm_hashes=["hash1"], + cache_salt="salt", + ) + request.lora_request = LoRARequest( + lora_name="adapter", lora_int_id=1, lora_path="/path/to/lora" + ) + + extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0) + + assert extra_keys == ( + ("lora", "adapter"), + ("mm", "hash1", 2), + ("cache_salt", "salt"), + ) + event_keys = ("adapter", ("hash1", 2), "salt") + assert to_event_extra_keys([extra_keys, None]) == [event_keys, None] + assert to_event_extra_keys([]) is None + assert to_event_extra_keys(None) is None + + @pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) def test_hash_block_tokens(hash_fn): parent_block_hash = BlockHash(b"123") @@ -1013,9 +1059,11 @@ def test_request_block_hasher(hash_fn): block_hashes = request.block_hashes assert len(block_hashes) == 2 assert block_hashes[0] == hash_fn( - (kv_cache_utils.NONE_HASH, (0, 1, 2), (("hash1", 0),)) + (kv_cache_utils.NONE_HASH, (0, 1, 2), (("mm", "hash1", 0),)) + ) + assert block_hashes[1] == hash_fn( + (block_hashes[0], (3, 4, 5), (("mm", "hash2", 0),)) ) - assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), (("hash2", 0),))) @pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) @@ -1043,7 +1091,7 @@ def test_request_block_hasher_incremental_append_with_multiple_mm_features(hash_ ) expected_second_hash = hash_fn( - (incremental.block_hashes[0], (4, 5, 6, 7), (("A", 0), ("B", 2))) + (incremental.block_hashes[0], (4, 5, 6, 7), (("mm", "A", 0), ("mm", "B", 2))) ) assert incremental.block_hashes[1] == expected_second_hash assert incremental.block_hashes == fresh.block_hashes @@ -3657,7 +3705,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes] ( kv_cache_utils.NONE_HASH, tuple(prompt_token_ids[:block_size]), - (block1_embeds_hash,), + (("prompt_embeds", block1_embeds_hash),), ) ) assert block_hashes[0] == expected_hash1 @@ -3669,7 +3717,7 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes] ( block_hashes[0], tuple(prompt_token_ids[block_size:num_tokens]), - (block2_embeds_hash,), + (("prompt_embeds", block2_embeds_hash),), ) ) assert block_hashes[1] == expected_hash2 @@ -3706,7 +3754,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes ( kv_cache_utils.NONE_HASH, tuple(prompt_token_ids[:block_size]), - (("hash1", 0), block1_embeds_hash), + (("mm", "hash1", 0), ("prompt_embeds", block1_embeds_hash)), ) ) assert block_hashes[0] == expected_hash1 @@ -3718,7 +3766,7 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes ( block_hashes[0], tuple(prompt_token_ids[block_size:num_tokens]), - (("hash2", 0), block2_embeds_hash), + (("mm", "hash2", 0), ("prompt_embeds", block2_embeds_hash)), ) ) assert block_hashes[1] == expected_hash2 diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 99d3235006..c46c179260 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -2901,21 +2901,21 @@ def test_mm_prefix_caching(): ( kv_cache_utils.NONE_HASH, tuple(all_token_ids[:block_size]), - (("aaa", 11),), + (("mm", "aaa", 11),), ) ) assert block_hashes[1] == sha256( ( block_hashes[0], tuple(all_token_ids[block_size : block_size * 2]), - (("aaa", -5), ("bbb", 14)), + (("mm", "aaa", -5), ("mm", "bbb", 14)), ) ) assert block_hashes[2] == sha256( ( block_hashes[1], tuple(all_token_ids[block_size * 2 : block_size * 3]), - (("bbb", -2),), + (("mm", "bbb", -2),), ) ) @@ -2938,7 +2938,7 @@ def test_mm_prefix_caching(): ( block_hashes[2], tuple(all_token_ids[3 * block_size :] + [8] * 5), - (("ccc", 0),), + (("mm", "ccc", 0),), ) ) @@ -2984,7 +2984,11 @@ def test_cache_key_salting(): block_hashes = req0.block_hashes assert len(block_hashes) == 3 assert block_hashes[0] == sha256( - (kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt1",)) + ( + kv_cache_utils.NONE_HASH, + tuple(token_ids[:block_size]), + (("cache_salt", "salt1"),), + ) ) assert block_hashes[1] == sha256( (block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None) @@ -3029,7 +3033,11 @@ def test_cache_key_salting(): block_hashes = req2.block_hashes assert len(block_hashes) == 3 assert block_hashes[0] == sha256( - (kv_cache_utils.NONE_HASH, tuple(token_ids[:block_size]), ("salt2",)) + ( + kv_cache_utils.NONE_HASH, + tuple(token_ids[:block_size]), + (("cache_salt", "salt2"),), + ) ) assert block_hashes[1] == sha256( (block_hashes[0], tuple(token_ids[block_size : block_size * 2]), None) diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index 9d7ec2f827..9eb0805b7b 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -68,9 +68,11 @@ class BlockStored(KVCacheEvent): lora_name: str | None """Extra keys used in block hash computation, one entry per block in - block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt, - prompt embedding hashes, etc. for that specific block. Exposed for external - KV cache consumers to reconstruct block hashes. + block_hashes. Each entry contains, in order, the LoRA name (if + `lora_name` is set), `(mm_identifier, offset)` pairs, the cache_salt + string and prompt embedding digests for that specific block. These are + published untagged; the block hash itself uses each value tagged with + its source, e.g. `("lora", name)` or `("mm", mm_identifier, offset)`. """ extra_keys: list[tuple[Any, ...] | None] | None = None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py index 68318a16da..05609fe86f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/events.py @@ -31,6 +31,7 @@ from vllm.v1.core.kv_cache_utils import ( BlockHash, maybe_convert_block_hash, resolve_block_hashes, + to_event_extra_keys, ) from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, @@ -393,9 +394,7 @@ class OffloadingEventsTracker: lora_id=meta.lora_id, medium=_MEDIUM_TO_EVENT_STR[event.medium], lora_name=meta.lora_name, - extra_keys=( - list(meta.extra_keys) if meta.extra_keys is not None else None - ), + extra_keys=to_event_extra_keys(meta.extra_keys), group_idx=meta.group_idx, kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind, kv_cache_spec_sliding_window=( diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index 9289e7c1f8..664804d50d 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -25,6 +25,7 @@ from vllm.v1.core.kv_cache_utils import ( make_block_hash_with_group_id, maybe_convert_block_hash, resolve_block_hashes, + to_event_extra_keys, ) from vllm.v1.request import Request @@ -367,7 +368,7 @@ class BlockPool: lora_id=request.lora_request.adapter_id if request.lora_request else None, medium=self.medium, lora_name=request.lora_request.name if request.lora_request else None, - extra_keys=extra_keys_list if extra_keys_list else None, + extra_keys=to_event_extra_keys(extra_keys_list), group_idx=kv_cache_group_id, session_id=request.session_id, ) @@ -557,7 +558,7 @@ class BlockPool: lora_name=request.lora_request.name if request.lora_request else None, - extra_keys=[extra_keys], + extra_keys=to_event_extra_keys([extra_keys]), group_idx=kv_cache_group_id, session_id=request.session_id, ) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index ddeda3ed19..7ef71dd471 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -502,8 +502,8 @@ def _gen_mm_extra_hash_keys( ) -> tuple[list[Any], int]: """Generate extra keys related to MultiModal request for block hash computation. For multi-modal inputs, the extra keys are - (mm_hash, start_offset) that indicate a mm input contained in the - block and its starting offset in the block tokens. + ("mm", mm_hash, start_offset) tuples that indicate a mm input contained + in the block and its starting offset in the block tokens. Args: request: The request object. @@ -549,7 +549,7 @@ def _gen_mm_extra_hash_keys( # relative to the start of the block so prefix-cache keys stay # distinct when the same MM item appears at different positions # within otherwise-identical placeholder blocks. - extra_keys.append((mm_feature.identifier, offset - start_token_idx)) + extra_keys.append(("mm", mm_feature.identifier, offset - start_token_idx)) if end_token_idx >= offset + length: # If this block contains the end of the current mm input, @@ -565,7 +565,7 @@ def _gen_mm_extra_hash_keys( return extra_keys, curr_mm_idx -def _gen_lora_extra_hash_keys(request: Request) -> list[str]: +def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str]]: """Generate extra keys related to LoRA for block hash computation. Args: @@ -578,12 +578,12 @@ def _gen_lora_extra_hash_keys(request: Request) -> list[str]: """ if not request.lora_request: return [] - return [request.lora_request.lora_name] + return [("lora", request.lora_request.lora_name)] def _gen_prompt_embeds_extra_hash_keys( request: Request, start_token_idx: int, end_token_idx: int -) -> list[bytes]: +) -> list[tuple[str, bytes]]: """Generate extra keys related to prompt embeds for block hash computation. Args: @@ -605,7 +605,7 @@ def _gen_prompt_embeds_extra_hash_keys( # Hash prompt embeds once per block and cache on request embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest() request._prompt_embeds_per_block_hashes[block_range] = embeds_hash - return [embeds_hash] + return [("prompt_embeds", embeds_hash)] def generate_block_hash_extra_keys( @@ -629,9 +629,11 @@ def generate_block_hash_extra_keys( mm_extra_keys, new_start_mm_idx = _gen_mm_extra_hash_keys( request, start_token_idx, end_token_idx, start_mm_idx ) - lora_extra_keys: list[str] = _gen_lora_extra_hash_keys(request) - cache_salt_keys: list[str] = ( - [request.cache_salt] if (start_token_idx == 0 and request.cache_salt) else [] + lora_extra_keys = _gen_lora_extra_hash_keys(request) + cache_salt_keys: list[tuple[str, str]] = ( + [("cache_salt", request.cache_salt)] + if (start_token_idx == 0 and request.cache_salt) + else [] ) prompt_embeds_keys = _gen_prompt_embeds_extra_hash_keys( request, start_token_idx, end_token_idx @@ -647,6 +649,35 @@ def generate_block_hash_extra_keys( return tuple(extra_keys), new_start_mm_idx +def to_event_extra_keys( + extra_keys: Iterable[tuple[Any, ...] | None] | None, +) -> list[tuple[Any, ...] | None] | None: + """Convert block-hash extra keys to the untagged per-block list published + in KV events. + + External KV event consumers parse the pre-tagging shapes: bare LoRA names + and cache salts, ``(mm_identifier, offset)`` pairs and bare prompt-embeds + digests. Events keep that format until consumers handle the tagged keys. + + Args: + extra_keys: One entry per block, each as returned by + `generate_block_hash_extra_keys`, or None. + + Returns: + One untagged entry per block, or None if there are no entries. + + """ + if not extra_keys: + return None + event_keys = [ + None + if keys is None + else tuple(key[1:] if key[0] == "mm" else key[1] for key in keys) + for keys in extra_keys + ] + return event_keys or None + + def hash_block_tokens( hash_function: Callable[[Any], bytes], parent_block_hash: BlockHash | None, diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index f70afaa343..c91537375f 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -34,6 +34,7 @@ from vllm.v1.core.kv_cache_utils import ( maybe_convert_block_hash, resolve_block_hashes, resolve_dcp_kv_block_size, + to_event_extra_keys, ) from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import ( @@ -1147,9 +1148,7 @@ class SimpleCPUOffloadScheduler: lora_id=meta.lora_id if meta else None, medium=self.kv_event_medium, lora_name=meta.lora_name if meta else None, - extra_keys=( - [extra_keys] if extra_keys is not None else None - ), + extra_keys=to_event_extra_keys(extra_keys and [extra_keys]), group_idx=group_idx, kv_cache_spec_kind=get_kv_cache_spec_kind(spec).value, kv_cache_spec_sliding_window=( From 105a4e097b905127b1d5f09a7e93bd1095e87a6c Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Tue, 29 Sep 2026 18:29:38 -0700 Subject: [PATCH 069/248] [Bugfix][Frontend] Reject LoRA adapters named after a served model (#59286) Signed-off-by: Nick Hill Co-authored-by: Claude Opus 5.5 --- .../serve/lora/test_serving_models.py | 16 ++++++++++++++++ vllm/entrypoints/openai/models/serving.py | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/tests/entrypoints/serve/lora/test_serving_models.py b/tests/entrypoints/serve/lora/test_serving_models.py index 3cbb0a480d..64be0961cb 100644 --- a/tests/entrypoints/serve/lora/test_serving_models.py +++ b/tests/entrypoints/serve/lora/test_serving_models.py @@ -99,6 +99,22 @@ async def test_load_lora_adapter_duplicate(): assert len(serving_models.lora_requests) == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize("load_inplace", [False, True]) +async def test_load_lora_adapter_base_model_name(load_inplace: bool): + """A LoRA named after a served model would shadow it for all requests.""" + serving_models = await _async_serving_models_init() + request = LoadLoRAAdapterRequest( + lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace + ) + response = await serving_models.load_lora_adapter(request) + assert isinstance(response, ErrorResponse) + assert response.error.type == "InvalidUserInput" + assert response.error.code == HTTPStatus.BAD_REQUEST + assert len(serving_models.lora_requests) == 0 + serving_models.engine_client.add_lora.assert_not_called() + + @pytest.mark.asyncio async def test_unload_lora_adapter_success(): serving_models = await _async_serving_models_init() diff --git a/vllm/entrypoints/openai/models/serving.py b/vllm/entrypoints/openai/models/serving.py index 1e13311dce..c139effa79 100644 --- a/vllm/entrypoints/openai/models/serving.py +++ b/vllm/entrypoints/openai/models/serving.py @@ -245,6 +245,14 @@ class OpenAIServingModels: status_code=HTTPStatus.BAD_REQUEST, ) + if self.is_base_model(request.lora_name): + return create_error_response( + message=f"The lora adapter '{request.lora_name}' conflicts with a " + "served base model.", + err_type="InvalidUserInput", + status_code=HTTPStatus.BAD_REQUEST, + ) + # If not loading inplace # Check if the lora adapter with the given name already exists if not request.load_inplace and request.lora_name in self.lora_requests: From 8039d3cbfd55be2aa172d544042c53b4e966369d Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 30 Sep 2026 09:34:46 +0800 Subject: [PATCH 070/248] [Rust Frontend] Use the MiMo structural-tag builder (#59148) Signed-off-by: Bugen Zhao Co-authored-by: Claude Sonnet 5.5 --- rust/Cargo.lock | 4 +-- rust/Cargo.toml | 2 +- rust/src/parser/src/tool/mimo.rs | 31 ++++++++++++++++++- .../src/unified/kimi_k3/structural_tag.rs | 2 +- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6a5d13f087..6c1a8fe727 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6605,9 +6605,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xgrammar-structural-tag" -version = "0.3.0+xgrammar.0.2.7.82505d0" +version = "0.4.1+xgrammar.0.2.8.9778737" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb" +checksum = "06637fe6ff753ef929b4cea2933b9432f78335255c6931dc748957122b9b492c" dependencies = [ "auto_impl", "serde", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 40b48f85bc..e0d3708daa 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -153,7 +153,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tracing = { path = "src/tracing" } winnow = { version = "1.0.2", features = ["simd"] } -xgrammar-structural-tag = "0.3.0" +xgrammar-structural-tag = "0.4.1" zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", diff --git a/rust/src/parser/src/tool/mimo.rs b/rust/src/parser/src/tool/mimo.rs index 2844efc1f5..f6491e9caa 100644 --- a/rust/src/parser/src/tool/mimo.rs +++ b/rust/src/parser/src/tool/mimo.rs @@ -26,7 +26,8 @@ impl ToolParser for MiMoToolParser { } fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> { - self.inner.structural_tag_builder() + // MiMo's template emits compact tags without the newlines Qwen3-Coder uses. + Some(xgrammar_structural_tag::Model::Mimo.builder()) } fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { @@ -88,4 +89,32 @@ mod tests { ); } } + + #[test] + fn structural_tag_uses_compact_mimo_tags() { + use xgrammar_structural_tag::builders::StructuralTagOptions; + use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag, + }; + + let parser = MiMoToolParser::create(&[]).unwrap(); + let builder = parser.structural_tag_builder().expect("MiMo exposes a builder"); + let tools = [ToolParam::Function(FunctionToolParam::new( + FunctionDefinition::new("get_weather"), + ))]; + let tag = build_structural_tag( + builder, + &tools, + ToolChoice::required(), + StructuralTagOptions::default().with_reasoning(false), + ) + .unwrap(); + let value = serde_json::to_value(tag).unwrap(); + assert_eq!(value["format"]["triggers"], json!([""])); + assert_eq!( + value["format"]["tags"][0]["begin"], + "" + ); + assert_eq!(value["format"]["tags"][0]["end"], ""); + } } diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs index fc30f455e9..f542f34e67 100644 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs @@ -395,7 +395,7 @@ mod tests { ) .unwrap(); - expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null,"excludes":[]},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null,"excludes":[]},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); } #[test] From 678baf53724e06cef7f07628ae8fd4cc6c96f11a Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:34:36 -0400 Subject: [PATCH 071/248] [Dependency] Upgrade FlashInfer to 0.7.0.post1 (#59323) Signed-off-by: wzhao18 --- docker/Dockerfile | 2 +- docker/versions.json | 2 +- requirements/cuda.txt | 4 ++-- requirements/rubin-prerelease.txt | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5f4c4ab35f..ba046adae1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1004,7 +1004,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \ # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) # https://docs.flashinfer.ai/installation.html # From versions.json: .flashinfer.version -ARG FLASHINFER_VERSION=0.7.0 +ARG FLASHINFER_VERSION=0.7.0.post1 RUN --mount=type=cache,target=/opt/uv/cache \ if [ "${INSTALL_RUBIN_PRERELEASE}" != "true" ]; then \ uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ diff --git a/docker/versions.json b/docker/versions.json index ec23b95186..bcec27fd55 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -71,7 +71,7 @@ "default": "true" }, "FLASHINFER_VERSION": { - "default": "0.7.0" + "default": "0.7.0.post1" }, "GDRCOPY_CUDA_VERSION": { "default": "12.8" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 2bbf1e244d..5cd1e717be 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -14,8 +14,8 @@ PyNvVideoCodec==2.0.4 # flashinfer-cubin is not on PyPI since 0.6.14; setup.py excludes it from # install_requires so the published wheel does not carry an unresolvable pin --extra-index-url https://flashinfer.ai/whl/ -flashinfer-python==0.7.0 -flashinfer-cubin==0.7.0 +flashinfer-python==0.7.0.post1 +flashinfer-cubin==0.7.0.post1 apache-tvm-ffi==0.1.11 tilelang==0.1.12 nvidia-cudnn-frontend>=1.19.1 diff --git a/requirements/rubin-prerelease.txt b/requirements/rubin-prerelease.txt index d02fb38884..e5915b6d13 100644 --- a/requirements/rubin-prerelease.txt +++ b/requirements/rubin-prerelease.txt @@ -16,11 +16,11 @@ torchaudio==2.11.0.dev20260911+cu134 # INSTALL_RUBIN_PRERELEASE=true, so they must be pinned here instead to # still get installed. nvidia-cutlass-dsl[cu13]==4.8.0.dev0 -flashinfer-python==0.7.0 +flashinfer-python==0.7.0.post1 quack-kernels==0.6.5 cuda-python==13.4.1 cuda-bindings==13.4.1 -flashinfer-jit-cache==0.7.0+cu134 -flashinfer-cubin==0.7.0 +flashinfer-jit-cache==0.7.0.post1+cu134 +flashinfer-cubin==0.7.0.post1 nixl==1.4.1 From bec710fccf5f061c2946da3afa5b9e5c0b22c431 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 29 Sep 2026 19:55:04 -0700 Subject: [PATCH 072/248] [CI] Fix mock type narrowing in LoRA serving test (#59344) Signed-off-by: khluu Co-authored-by: Codex --- tests/entrypoints/serve/lora/test_serving_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/serve/lora/test_serving_models.py b/tests/entrypoints/serve/lora/test_serving_models.py index 64be0961cb..b2aefff219 100644 --- a/tests/entrypoints/serve/lora/test_serving_models.py +++ b/tests/entrypoints/serve/lora/test_serving_models.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from http import HTTPStatus -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest @@ -112,6 +112,7 @@ async def test_load_lora_adapter_base_model_name(load_inplace: bool): assert response.error.type == "InvalidUserInput" assert response.error.code == HTTPStatus.BAD_REQUEST assert len(serving_models.lora_requests) == 0 + assert isinstance(serving_models.engine_client.add_lora, AsyncMock) serving_models.engine_client.add_lora.assert_not_called() From 2798f668608155bc8a8c74cb97e3ddd0d3053085 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Tue, 29 Sep 2026 23:01:45 -0400 Subject: [PATCH 073/248] [Bugfix][HiSparse] Resolve MTP verification rows with a union residency kernel (#59235) Signed-off-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- csrc/libtorch_stable/hisparse_kernels.cu | 617 ++++++++++++------ csrc/libtorch_stable/ops.h | 2 +- csrc/libtorch_stable/torch_bindings.cpp | 1 + docs/design/hisparse.md | 8 +- .../v1/attention/test_sparse_mla_backends.py | 336 +++++++--- tests/v1/worker/test_utils.py | 1 + .../layers/attention/sparse_mla_attention.py | 3 - .../backends/mla/flashattn_mla_sparse.py | 3 +- .../backends/mla/flashinfer_mla_sparse.py | 4 +- .../mla/flashinfer_mla_sparse_sm120.py | 3 +- .../attention/backends/mla/flashmla_sparse.py | 11 +- vllm/v1/attention/backends/mla/index_group.py | 120 +--- vllm/v1/hisparse/runtime.py | 38 +- 13 files changed, 705 insertions(+), 442 deletions(-) diff --git a/csrc/libtorch_stable/hisparse_kernels.cu b/csrc/libtorch_stable/hisparse_kernels.cu index 97ce9246cc..069eddf3b0 100644 --- a/csrc/libtorch_stable/hisparse_kernels.cu +++ b/csrc/libtorch_stable/hisparse_kernels.cu @@ -30,9 +30,7 @@ namespace { constexpr int kWarpSize = 32; -// Sentinel in the shared top-k scratch: entry already resolved (hit / -// newest / invalid), no miss handling needed. -constexpr int32_t kTokenDone = -1; +// Empty key in the shared open-addressing table of host rows. constexpr int32_t kHashEmpty = -1; bool is_pinned_cpu_tensor(const torch::stable::Tensor& tensor) { @@ -46,10 +44,15 @@ bool is_pinned_cpu_tensor(const torch::stable::Tensor& tensor) { return attributes.type == cudaMemoryTypeHost; } -__device__ __forceinline__ int32_t hash_slot(int32_t key, int32_t hash_size) { - // Knuth multiplicative hash for the open-addressing table. - return static_cast((static_cast(key) * 2654435761u) % - static_cast(hash_size)); +__device__ __forceinline__ int32_t hash_slot(int32_t key, int size) { + // Knuth multiplicative hash scaled onto [0, size) by a multiply-high, so + // the table can take any size without a division. + return static_cast( + __umulhi(static_cast(key) * 2654435761u, size)); +} + +__device__ __forceinline__ int next_slot(int h, int size) { + return h + 1 == size ? 0 : h + 1; } // Copy one row of `row_bytes` bytes with a single warp. MLA rows take the @@ -176,16 +179,117 @@ __device__ __forceinline__ void store_hot_index( } } -// One block per batch row. +// A union entry's value holds its first (row, top-k) position until phase 2 +// or 3 resolves it to a hot slot tagged with how it was resolved. +constexpr int32_t kEntryHit = 1 << 30; +constexpr int32_t kEntryMiss = 1 << 29; +constexpr int32_t kEntryInvalid = 1 << 28; +constexpr int32_t kEntrySlotMask = 0xFFFF; +constexpr int kResidencyCounters = 5; +// Per-request table entries per referenced top-k position (load <= 1/4 +// until the hash_size cap). +constexpr int kHashLoadInverse = 4; + +// Translate a request-relative top-k position to its resident GPU row (-1 when +// the page is not resident) and returns its host row (-1 when not host-backed). +__device__ __forceinline__ int32_t translate_topk_entry( + const int32_t token_index, const int32_t request_row, + const int32_t* __restrict__ source_block_table, + const int32_t* __restrict__ resident_block_table, const int64_t host_rows, + const int64_t source_bt_stride, const int32_t source_num_reqs, + const int32_t source_num_blocks, const int32_t source_block_size, + const int64_t resident_bt_stride, const int32_t resident_num_reqs, + const int32_t resident_num_blocks, const int32_t resident_block_size, + const int32_t resident_null_block, int32_t& resident_row) { + int32_t g = token_index; + resident_row = -1; + if (source_block_table != nullptr) { + const int32_t source_block = + token_index >= 0 ? token_index / source_block_size : -1; + if (request_row >= 0 && request_row < source_num_reqs && + source_block >= 0 && source_block < source_num_blocks) { + const int32_t physical_block = + source_block_table[static_cast(request_row) * + source_bt_stride + + source_block]; + g = physical_block > 0 ? physical_block * source_block_size + + token_index % source_block_size + : -1; + } else { + g = -1; + } + if (resident_block_table != nullptr) { + const int32_t resident_block = + token_index >= 0 ? token_index / resident_block_size : -1; + if (request_row >= 0 && request_row < resident_num_reqs && + resident_block >= 0 && resident_block < resident_num_blocks) { + const int32_t physical_block = + resident_block_table[static_cast(request_row) * + resident_bt_stride + + resident_block]; + if (physical_block != resident_null_block && physical_block >= 0) { + resident_row = physical_block * resident_block_size + + token_index % resident_block_size; + } + } + } + } + return g >= host_rows ? -1 : g; +} + +// Write the hot index of one reference to a resolved union entry and tally it. +__device__ __forceinline__ void write_entry_index( + const int32_t entry, const int row, const int i, const int32_t top_k, + int32_t* __restrict__ hot_indices, int32_t* __restrict__ attention_indices, + const int64_t attention_row_stride, int32_t* __restrict__ miss_mask, + const int32_t* __restrict__ hot_block_table, const int request_row, + const int64_t hot_table_stride, const int32_t hot_block_size, + const int64_t attention_block_stride, const bool first_reference, int& hits, + int& misses) { + int32_t* row_out = hot_indices + static_cast(row) * top_k; + int32_t* row_attention = + attention_indices != nullptr + ? attention_indices + static_cast(row) * attention_row_stride + : nullptr; + if (entry & kEntryInvalid) { + store_hot_index(row_out, row_attention, i, -1, hot_block_size, + attention_block_stride); + return; + } + store_hot_index(row_out, row_attention, i, + static_cast(get_physical_hot_row( + hot_block_table, request_row, hot_table_stride, + hot_block_size, entry & kEntrySlotMask)), + hot_block_size, attention_block_stride); + // Only an entry's first reference loads it; later references reuse the + // slot like a hit. + if ((entry & kEntryMiss) && first_reference) { + if (miss_mask != nullptr) { + miss_mask[static_cast(row) * top_k + i] = 1; + } + ++misses; + } else { + ++hits; + } +} + +// One block per request, launched per batch row. A request's rows (several +// with speculative decoding) are contiguous and share its hot-buffer state, so +// the block of its first row resolves the union of their host misses once and +// maps every row onto it; the other blocks exit. The union is bounded by the +// caller's max_union_rows (every decode query's top-k), which +// device_buffer_size covers; a larger union traps. Misses take eviction slots +// in (row, top-k) order of first reference, so the plan is deterministic. // // Shared memory layout (int32 region followed by int16 region): -// s_topk[top_k] top-k global ids; reused as miss scratch -// s_chunk_off[nbc + 1] prefix sums for hit (then miss) compaction +// s_hash_keys[hash_size] open addressing: union of the rows' host ids +// s_hash_vals[hash_size] first position, then slot | kEntry* tag +// s_chunk_off[nbc + 1] prefix sums for hit, then miss, compaction // s_evict_off[nbc + 1] prefix sums for evictable compaction -// s_hash_keys[hash_size] open addressing: global id -> top-k index -// s_counters[3] hits, phase-1 resolved, valid count +// s_counters[5] distinct hits, union size, per-row valid count, +// misses, per-entry hits +// s_done[nbc] positions phase 3 resolved, one word per chunk // s_lru_out[hot_size] int16, compacted slots: [hits fwd | evict bwd] -// s_hash_vals[hash_size] int16 hash values (top-k index) // Valid global ids must be unique within each row. __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( const int32_t* __restrict__ hot_block_table, // [max_rows, hot_blocks] @@ -218,11 +322,25 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( const int64_t attention_row_stride, const int64_t valid_count_stride) { const int NUM_WARPS = blockDim.x / kWarpSize; const int num_buffer_chunks = (hot_size + kWarpSize - 1) / kWarpSize; - const int num_token_chunks = (top_k + kWarpSize - 1) / kWarpSize; - const int batch_row = blockIdx.x; + // The first row of each contiguous run of a request's rows leads it. FULL + // CUDA-graph padding rows map to request 0 after the real rows, forming a + // second run, but carry no valid top-k: that run returns before phase 2 + // touches the request's shared state. + const int first_row = blockIdx.x; const int request_row = - request_ids != nullptr ? request_ids[batch_row] : batch_row; + request_ids != nullptr ? request_ids[first_row] : first_row; + if (request_ids != nullptr && first_row > 0 && + request_ids[first_row - 1] == request_row) { + return; + } + int end_row = first_row + 1; + if (request_ids != nullptr) { + while (end_row < static_cast(gridDim.x) && + request_ids[end_row] == request_row) { + ++end_row; + } + } const int state_row = request_state_indices != nullptr && request_row >= 0 && request_row < request_state_count @@ -230,23 +348,24 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( : (request_state_indices == nullptr ? request_row : -1); // V2 publishes -1 for CUDA-graph padding rows. if (state_row < 0) { - for (int i = threadIdx.x; i < top_k; i += blockDim.x) { - const int64_t index = static_cast(batch_row) * top_k + i; - hot_indices[index] = -1; - if (attention_indices != nullptr) { - attention_indices[static_cast(batch_row) * - attention_row_stride + - i] = -1; + for (int row = first_row; row < end_row; ++row) { + for (int i = threadIdx.x; i < top_k; i += blockDim.x) { + const int64_t index = static_cast(row) * top_k + i; + hot_indices[index] = -1; + if (attention_indices != nullptr) { + attention_indices[static_cast(row) * attention_row_stride + + i] = -1; + } + if (resolved_global_indices != nullptr) { + resolved_global_indices[index] = -1; + } } - if (resolved_global_indices != nullptr) { - resolved_global_indices[index] = -1; + if (valid_counts != nullptr && threadIdx.x == 0) { + valid_counts[static_cast(row) * valid_count_stride] = 0; + } + if (swap_counts != nullptr && threadIdx.x == 0) { + swap_counts[row] = 0; } - } - if (valid_counts != nullptr && threadIdx.x == 0) { - valid_counts[static_cast(batch_row) * valid_count_stride] = 0; - } - if (swap_counts != nullptr && threadIdx.x == 0) { - swap_counts[batch_row] = 0; } return; } @@ -255,136 +374,130 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( const int lane_id = tid % kWarpSize; const unsigned int lanes_before = ((unsigned int)1 << lane_id) - 1; - const int32_t* row_topk = - global_indices + static_cast(batch_row) * input_row_stride; - int32_t* row_out = hot_indices + static_cast(batch_row) * top_k; - int32_t* row_attention = - attention_indices != nullptr - ? attention_indices + - static_cast(batch_row) * attention_row_stride - : nullptr; - int32_t* row_miss = (miss_mask != nullptr) - ? miss_mask + static_cast(batch_row) * top_k - : nullptr; int32_t* row_dgi = device_global_indices + static_cast(state_row) * region_stride; int16_t* row_lru = lru_slots + static_cast(state_row) * hot_size; + // A union within max_union_rows (hash_size - 1) and within the rows' top-k + // leaves an empty slot, so every probe sequence ends. Use a slice sized to + // this request's rows so single-row requests initialize a small table. + const int table_size = + min(hash_size, kHashLoadInverse * (end_row - first_row) * top_k); + extern __shared__ char smem_raw[]; - int32_t* s_topk = reinterpret_cast(smem_raw); - int32_t* s_chunk_off = s_topk + top_k; + int32_t* s_hash_keys = reinterpret_cast(smem_raw); + int32_t* s_hash_vals = s_hash_keys + hash_size; + int32_t* s_chunk_off = s_hash_vals + hash_size; int32_t* s_evict_off = s_chunk_off + (num_buffer_chunks + 1); - int32_t* s_hash_keys = s_evict_off + (num_buffer_chunks + 1); - int32_t* s_counters = s_hash_keys + hash_size; - int16_t* s_lru_out = reinterpret_cast(s_counters + 3); - int16_t* s_hash_vals = s_lru_out + hot_size; + int32_t* s_counters = s_evict_off + (num_buffer_chunks + 1); + unsigned int* s_done = + reinterpret_cast(s_counters + kResidencyCounters); + int16_t* s_lru_out = reinterpret_cast(s_done + num_buffer_chunks); - if (tid < 3) { - s_counters[tid] = 0; - } - __syncthreads(); - - // Phase 1: translate request-relative positions and resolve resident rows. - for (int i = tid; i < top_k; i += blockDim.x) { - const int32_t token_index = row_topk[i]; - int32_t g = token_index; - int32_t resident_row = -1; - if (source_block_table != nullptr) { - const int32_t request_id = request_ids[batch_row]; - const int32_t source_block = - token_index >= 0 ? token_index / source_block_size : -1; - if (request_id >= 0 && request_id < source_num_reqs && - source_block >= 0 && source_block < source_num_blocks) { - const int32_t physical_block = - source_block_table[static_cast(request_id) * - source_bt_stride + - source_block]; - g = physical_block > 0 ? physical_block * source_block_size + - token_index % source_block_size - : -1; - } else { - g = -1; - } - if (resident_block_table != nullptr) { - const int32_t resident_block = - token_index >= 0 ? token_index / resident_block_size : -1; - if (request_id >= 0 && request_id < resident_num_reqs && - resident_block >= 0 && resident_block < resident_num_blocks) { - const int32_t physical_block = - resident_block_table[static_cast(request_id) * - resident_bt_stride + - resident_block]; - if (physical_block != resident_null_block && physical_block >= 0) { - resident_row = physical_block * resident_block_size + - token_index % resident_block_size; - } - } - } - } - if (g >= host_rows) { - g = -1; - } - if (resolved_global_indices != nullptr) { - resolved_global_indices[static_cast(batch_row) * top_k + i] = g; - } - if (resident_row >= 0 || g >= 0) atomicAdd(&s_counters[2], 1); - if (row_miss != nullptr) row_miss[i] = 0; - if (resident_row >= 0) { - store_hot_index(row_out, row_attention, i, resident_row, hot_block_size, - attention_block_stride); - s_topk[i] = kTokenDone; - atomicAdd(&s_counters[1], 1); - } else if (g < 0) { - store_hot_index(row_out, row_attention, i, -1, hot_block_size, - attention_block_stride); - s_topk[i] = kTokenDone; - atomicAdd(&s_counters[1], 1); - } else { - s_topk[i] = g; - } - } - __syncthreads(); - if (valid_counts != nullptr && tid == 0) { - valid_counts[static_cast(batch_row) * valid_count_stride] = - s_counters[2]; - } - // Fully resident rows need only request-relative page translation. Avoid - // scanning or rewriting the hot LRU when no selected row can consult it. - if (s_counters[1] == top_k) { - if (tid == 0 && swap_counts != nullptr) { - swap_counts[batch_row] = 0; - } - return; - } - - for (int i = tid; i < hash_size; i += blockDim.x) { + for (int i = tid; i < table_size; i += blockDim.x) { s_hash_keys[i] = kHashEmpty; + s_hash_vals[i] = INT32_MAX; } for (int i = tid; i < num_buffer_chunks + 1; i += blockDim.x) { s_chunk_off[i] = 0; s_evict_off[i] = 0; } - __syncthreads(); - - for (int i = tid; i < top_k; i += blockDim.x) { - const int32_t g = s_topk[i]; - if (g == kTokenDone) { - continue; - } - int slot = hash_slot(g, hash_size); - while (true) { - const int32_t old = atomicCAS(&s_hash_keys[slot], kHashEmpty, g); - if (old == kHashEmpty || old == g) { - s_hash_vals[slot] = static_cast(i); - break; - } - slot = (slot + 1) % hash_size; - } + if (tid < kResidencyCounters) { + s_counters[tid] = 0; } __syncthreads(); + // Phase 1: translate every row's request-relative positions, resolve + // resident rows directly, and collect the union of host rows. + for (int row = first_row; row < end_row; ++row) { + const int32_t* row_topk = + global_indices + static_cast(row) * input_row_stride; + int32_t* row_out = hot_indices + static_cast(row) * top_k; + int32_t* row_attention = + attention_indices != nullptr + ? attention_indices + + static_cast(row) * attention_row_stride + : nullptr; + int32_t* row_miss = miss_mask != nullptr + ? miss_mask + static_cast(row) * top_k + : nullptr; + for (int i = tid; i < top_k; i += blockDim.x) { + int32_t resident_row; + const int32_t g = translate_topk_entry( + row_topk[i], request_row, source_block_table, resident_block_table, + host_rows, source_bt_stride, source_num_reqs, source_num_blocks, + source_block_size, resident_bt_stride, resident_num_reqs, + resident_num_blocks, resident_block_size, resident_null_block, + resident_row); + if (resolved_global_indices != nullptr) { + resolved_global_indices[static_cast(row) * top_k + i] = g; + } + if (resident_row >= 0 || g >= 0) atomicAdd(&s_counters[2], 1); + if (row_miss != nullptr) row_miss[i] = 0; + if (resident_row >= 0) { + store_hot_index(row_out, row_attention, i, resident_row, hot_block_size, + attention_block_stride); + } else if (g < 0) { + store_hot_index(row_out, row_attention, i, -1, hot_block_size, + attention_block_stride); + } else { + int h = hash_slot(g, table_size); + bool inserted = false; + for (int probe = 0; probe < table_size; ++probe) { + const int32_t old = atomicCAS(&s_hash_keys[h], kHashEmpty, g); + if (old == kHashEmpty) { + atomicAdd(&s_counters[1], 1); + inserted = true; + break; + } + if (old == g) { + inserted = true; + break; + } + h = next_slot(h, table_size); + } + if (inserted) { + atomicMin(&s_hash_vals[h], (row - first_row) * top_k + i); + } else { + // A full table: push the union count past the cap so the check + // below traps. + atomicAdd(&s_counters[1], hash_size); + } + } + } + __syncthreads(); + if (tid == 0) { + if (valid_counts != nullptr) { + valid_counts[static_cast(row) * valid_count_stride] = + s_counters[2]; + } + s_counters[2] = 0; + } + __syncthreads(); + } + // The table and eviction slots are sized for max_union_rows (hash_size - 1); + // a larger union would never reach an empty probe slot or run past s_lru_out. + if (s_counters[1] >= hash_size) { + if (tid == 0) { + printf( + "HiSparse residency: request %d resolves %d host rows, above the " + "configured bound of %d\n", + request_row, s_counters[1], hash_size - 1); + } + __trap(); + } + // Fully resident rows need only request-relative page translation. Avoid + // scanning or rewriting the hot LRU when no selected row can consult it. + if (s_counters[1] == 0) { + for (int row = first_row + tid; row < end_row; row += blockDim.x) { + if (swap_counts != nullptr) swap_counts[row] = 0; + } + return; + } + // Phase 2: walk hot slots in LRU order, classify hit / evictable, and - // compact them (hits forward, evictables backward) into s_lru_out. + // compact them (hits forward, evictables backward) into s_lru_out. A hit + // records its slot on the union entry. const int iters_buffer = (num_buffer_chunks + NUM_WARPS - 1) / NUM_WARPS; int total_hit_count = 0; int total_evict_count = 0; @@ -397,36 +510,28 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( const int16_t slot = has_valid_pos ? row_lru[pos] : int16_t(-1); // Corruption tripwire: lru/dgi are long-lived device state; if some // external writer (e.g. stray RDMA into reused VRAM) corrupts a slot - // out of [0, region_stride), treat it as no-hit so it degrades to a re-miss - // instead of an unbounded dgi read (phases 3/5 bound the write side). + // out of [0, region_stride), treat it as no-hit so it degrades to a + // re-miss instead of an unbounded dgi read (phase 3 bounds the write + // side). const int32_t cached_g = (slot >= 0 && slot < region_stride) ? row_dgi[slot] : -1; - int found_topk_idx = -1; + bool is_hit = false; if (cached_g >= 0) { - int h = hash_slot(cached_g, hash_size); + int h = hash_slot(cached_g, table_size); while (true) { const int32_t k = s_hash_keys[h]; if (k == cached_g) { - found_topk_idx = static_cast(s_hash_vals[h]); + s_hash_vals[h] = kEntryHit | slot; + is_hit = true; break; } if (k == kHashEmpty) break; - h = (h + 1) % hash_size; + h = next_slot(h, table_size); } } - const bool is_hit = found_topk_idx >= 0; const bool is_evictable = has_valid_pos && !is_hit; - if (is_hit) { - s_topk[found_topk_idx] = kTokenDone; - store_hot_index(row_out, row_attention, found_topk_idx, - static_cast(get_physical_hot_row( - hot_block_table, request_row, hot_table_stride, - hot_block_size, slot)), - hot_block_size, attention_block_stride); - } - int local_hit_off = 0; int local_evict_off = 0; if (has_valid_chunk) { @@ -465,39 +570,59 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( } __syncthreads(); - // Reset prefix sums for the miss compaction (token chunks <= buffer - // chunks because hot_size >= top_k). - for (int i = tid; i < num_token_chunks + 1; i += blockDim.x) { + for (int i = tid; i < num_buffer_chunks + 1; i += blockDim.x) { s_chunk_off[i] = 0; } __syncthreads(); - // Phase 3: compact misses, assign them eviction slots (oldest first) and - // record the new ownership in device_global_indices. - const int iters_token = (num_token_chunks + NUM_WARPS - 1) / NUM_WARPS; + // Phase 3: walk the rows' positions in (row, top-k) order; the first + // reference to a union entry no slot holds is a miss. Assign misses + // eviction slots (oldest first) and record the new ownership in + // device_global_indices. Swaps fill the rows' swap segments in order; the + // union never exceeds their combined capacity, and position chunks never + // exceed buffer chunks because hot_size covers every row's top-k. + const int num_positions = (end_row - first_row) * top_k; + const int num_position_chunks = (num_positions + kWarpSize - 1) / kWarpSize; + const int iters_position = (num_position_chunks + NUM_WARPS - 1) / NUM_WARPS; int miss_running_total = 0; - for (int iter = 0; iter < iters_token; iter++) { + int entry_hits = 0; + int entry_misses = 0; + for (int iter = 0; iter < iters_position; iter++) { const int chunk_idx = warp_id + iter * NUM_WARPS; - const bool has_valid_chunk = chunk_idx < num_token_chunks; + const bool has_valid_chunk = chunk_idx < num_position_chunks; - const int i = chunk_idx * kWarpSize + lane_id; - const bool has_valid_token = has_valid_chunk && (i < top_k); - - int32_t g = 0; + const int position = chunk_idx * kWarpSize + lane_id; + int32_t g = -1; + int h = -1; + int32_t entry = 0; bool is_miss = false; - if (has_valid_token) { - is_miss = s_topk[i] != kTokenDone; - if (is_miss) { - g = s_topk[i]; + if (has_valid_chunk && position < num_positions) { + const int row = first_row + position / top_k; + int32_t resident_row; + g = translate_topk_entry( + global_indices[static_cast(row) * input_row_stride + + position % top_k], + request_row, source_block_table, resident_block_table, host_rows, + source_bt_stride, source_num_reqs, source_num_blocks, + source_block_size, resident_bt_stride, resident_num_reqs, + resident_num_blocks, resident_block_size, resident_null_block, + resident_row); + if (resident_row < 0 && g >= 0) { + h = hash_slot(g, table_size); + while (s_hash_keys[h] != g) { + h = next_slot(h, table_size); + } + entry = s_hash_vals[h]; + is_miss = entry == position; } } int local_miss_off = 0; if (has_valid_chunk) { - const unsigned int miss_mask = __ballot_sync(0xFFFFFFFF, is_miss); - local_miss_off = __popc(miss_mask & lanes_before); + const unsigned int miss_mask_bits = __ballot_sync(0xFFFFFFFF, is_miss); + local_miss_off = __popc(miss_mask_bits & lanes_before); if (lane_id == 0) { - s_chunk_off[chunk_idx + 1] = __popc(miss_mask); + s_chunk_off[chunk_idx + 1] = __popc(miss_mask_bits); } } __syncthreads(); @@ -505,60 +630,112 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel( if (warp_id == 0) { miss_running_total = warp_inclusive_scan(s_chunk_off, lane_id, chunk_idx + 1, - num_token_chunks + 1, miss_running_total); + num_position_chunks + 1, miss_running_total); + if (tid == 0) { + s_counters[3] = miss_running_total; + } } __syncthreads(); if (is_miss) { const int m = s_chunk_off[chunk_idx] + local_miss_off; const int16_t evict_slot = s_lru_out[hot_size - 1 - m]; + const int64_t compact_index = static_cast(first_row) * top_k + m; if (evict_slot < 0 || evict_slot >= region_stride) { // Corruption tripwire: an out-of-range slot (corrupted lru state, // see phase 2) must not become a hot-cache/dgi write. Resolve the - // entry as invalid (-1, masked by attention, not in the miss set); - // it re-misses on a later step. Phase 5 re-checks the same - // s_lru_out value, so its copy is skipped consistently. - store_hot_index(row_out, row_attention, i, -1, hot_block_size, - attention_block_stride); + // entry as invalid (-1, masked by attention); it re-misses on a later + // step, and its copy is skipped. + s_hash_vals[h] = kEntryInvalid; if (swap_host_physical_rows != nullptr) { - const int64_t compact_index = - static_cast(batch_row) * top_k + m; swap_host_physical_rows[compact_index] = g; swap_device_physical_rows[compact_index] = -1; } } else { - // Reuse s_topk as compacted miss scratch: m < i always holds (done - // entries are skipped), so writes never overrun pending reads. - s_topk[m] = g; - const int32_t physical_row = static_cast( - get_physical_hot_row(hot_block_table, request_row, hot_table_stride, - hot_block_size, evict_slot)); - store_hot_index(row_out, row_attention, i, physical_row, hot_block_size, - attention_block_stride); + s_hash_vals[h] = kEntryMiss | evict_slot; if (swap_host_physical_rows != nullptr) { - const int64_t compact_index = - static_cast(batch_row) * top_k + m; swap_host_physical_rows[compact_index] = g; - swap_device_physical_rows[compact_index] = physical_row; + swap_device_physical_rows[compact_index] = + static_cast(get_physical_hot_row( + hot_block_table, request_row, hot_table_stride, + hot_block_size, evict_slot)); } - if (row_miss != nullptr) row_miss[i] = 1; row_dgi[evict_slot] = g; } + entry = s_hash_vals[h]; + } + + // Hits, first references and references to entries resolved in an + // earlier chunk write their hot index now; later references to an entry + // resolved in this same chunk pass (only across rows) are left to phase 4. + const bool resolved = + h >= 0 && (entry & (kEntryHit | kEntryMiss | kEntryInvalid)); + if (resolved) { + write_entry_index( + entry, first_row + position / top_k, position % top_k, top_k, + hot_indices, attention_indices, attention_row_stride, miss_mask, + hot_block_table, request_row, hot_table_stride, hot_block_size, + attention_block_stride, is_miss, entry_hits, entry_misses); + } + if (has_valid_chunk) { + const unsigned int done_bits = __ballot_sync(0xFFFFFFFF, resolved); + if (lane_id == 0) { + s_done[chunk_idx] = done_bits; + } } } __syncthreads(); const int total_hits = s_counters[0]; - const int total_misses = top_k - total_hits - s_counters[1]; - if (swap_counts != nullptr && tid == 0) { - swap_counts[batch_row] = total_misses; - } - if (stats != nullptr && tid == 0) { - atomicAdd(&stats[0], static_cast(total_hits)); - atomicAdd(&stats[1], static_cast(total_misses)); + const int total_misses = s_counters[3]; + if (swap_counts != nullptr) { + for (int row = first_row + tid; row < end_row; row += blockDim.x) { + const int remaining = total_misses - (row - first_row) * top_k; + swap_counts[row] = max(0, min(top_k, remaining)); + } } - // Phase 4: write back the LRU order: stale evictables at the front, + // Phase 4: references phase 3 left unresolved (later references across rows + // to entries loaded in the same chunk pass) map onto the union's slots. + // Single-row requests resolve everything in phase 3. + if (end_row - first_row > 1) { + for (int position = tid; position < num_positions; position += blockDim.x) { + if (s_done[position / kWarpSize] & (1u << (position % kWarpSize))) { + continue; + } + const int row = first_row + position / top_k; + int32_t resident_row; + const int32_t g = translate_topk_entry( + global_indices[static_cast(row) * input_row_stride + + position % top_k], + request_row, source_block_table, resident_block_table, host_rows, + source_bt_stride, source_num_reqs, source_num_blocks, + source_block_size, resident_bt_stride, resident_num_reqs, + resident_num_blocks, resident_block_size, resident_null_block, + resident_row); + if (resident_row >= 0 || g < 0) continue; + int h = hash_slot(g, table_size); + while (s_hash_keys[h] != g) { + h = next_slot(h, table_size); + } + write_entry_index( + s_hash_vals[h], row, position % top_k, top_k, hot_indices, + attention_indices, attention_row_stride, miss_mask, hot_block_table, + request_row, hot_table_stride, hot_block_size, attention_block_stride, + false, entry_hits, entry_misses); + } + } + if (stats != nullptr) { + atomicAdd(&s_counters[4], entry_hits); + atomicAdd(&s_counters[2], entry_misses); + __syncthreads(); + if (tid == 0) { + atomicAdd(&stats[0], static_cast(s_counters[4])); + atomicAdd(&stats[1], static_cast(s_counters[2])); + } + } + + // Phase 5: write back the LRU order: stale evictables at the front, // freshly loaded misses next, then hits at MRU. const int total_evictable = hot_size - total_hits; const int remaining_evictable = total_evictable - total_misses; @@ -783,7 +960,7 @@ void hisparse_resolve_residency( torch::stable::Tensor& device_global_indices, torch::stable::Tensor& lru_slots, std::optional const& request_state_indices, - int64_t region_stride, + int64_t region_stride, int64_t max_union_rows, std::optional const& miss_mask, std::optional const& stats, std::optional const& attention_indices, @@ -1034,14 +1211,32 @@ void hisparse_resolve_residency( } constexpr int kBlockSize = 1024; - const int hash_size = 2 * top_k; + STD_TORCH_CHECK(max_union_rows > 0 && max_union_rows <= hot_size, + "max_union_rows must be in [1, hot_size], got ", + max_union_rows); + // One spare entry above the union bound keeps every probe sequence finite. + const int hash_size = static_cast(max_union_rows) + 1; const int num_buffer_chunks = (hot_size + kWarpSize - 1) / kWarpSize; const size_t smem_bytes = - sizeof(int32_t) * (top_k + 2 * (num_buffer_chunks + 1) + hash_size + 3) + - sizeof(int16_t) * (hot_size + hash_size); + sizeof(int32_t) * (2 * hash_size + 2 * (num_buffer_chunks + 1) + + kResidencyCounters + num_buffer_chunks) + + sizeof(int16_t) * hot_size; const torch::stable::accelerator::DeviceGuard device_guard( hot_cache.get_device_index()); + int max_smem_bytes = 0; + STD_TORCH_CHECK(cudaDeviceGetAttribute( + &max_smem_bytes, cudaDevAttrMaxSharedMemoryPerBlockOptin, + hot_cache.get_device_index()) == cudaSuccess, + "failed to query shared memory per block"); + STD_TORCH_CHECK(smem_bytes <= static_cast(max_smem_bytes), + "HiSparse device_buffer_size=", hot_size, " needs ", + smem_bytes, + " bytes of shared memory to resolve residency, " + "but the device allows ", + max_smem_bytes, + "; lower device_buffer_size or the number of speculative " + "tokens."); const cudaStream_t stream = get_current_cuda_stream(); const int64_t attention_row_stride = attention_indices.has_value() ? attention_indices.value().stride(0) : 0; diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index fba4220d73..6d45f99881 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -680,7 +680,7 @@ void hisparse_resolve_residency( torch::stable::Tensor& device_global_indices, torch::stable::Tensor& lru_slots, std::optional const& request_state_indices, - int64_t region_stride, + int64_t region_stride, int64_t max_union_rows, std::optional const& miss_mask, std::optional const& stats, std::optional const& attention_indices, diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 3833c95fde..98e89c39d3 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -995,6 +995,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) { " Tensor! lru_slots," " Tensor? request_state_indices," " int region_stride," + " int max_union_rows," " Tensor(a!)? miss_mask=None," " Tensor(b!)? stats=None," " Tensor(c!)? attention_indices=None," diff --git a/docs/design/hisparse.md b/docs/design/hisparse.md index 53a3093717..2c72ddcbcc 100644 --- a/docs/design/hisparse.md +++ b/docs/design/hisparse.md @@ -130,10 +130,10 @@ path. The resolver consumes the existing graph-stable request mapping from attention metadata; neither the worker nor individual cache handles keep a duplicate mapping. -Speculative decoding resolves and consumes each verification step in order. -Each step receives distinct replayable plan rows while sharing the request's -hot-cache state, so a later step cannot reuse a hot row before an earlier step -has consumed it. +Speculative decoding resolves all verification rows of a request in one pass: +one block resolves the union of the rows' top-k against the request's hot-cache +state, so rows that select the same host row share its hot row and no row +evicts a hot row another row of the step still reads. ## P/D import target diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 1d7fc389ef..1ea1415029 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -68,10 +68,10 @@ from vllm.utils.torch_utils import current_stream from vllm.v1.attention.backends.mla import index_group as index_group_module from vllm.v1.attention.backends.mla.flashattn_mla_sparse import ( FlashAttnMLASparseImpl, + FlashAttnMLASparseMetadataBuilder, ) from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import ( FlashInferMLASparseImpl, - FlashInferMLASparseMetadataBuilder, FlashInferMLASparseTRTLLMBackend, ) from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import ( @@ -1880,11 +1880,13 @@ def _make_hisparse_runtime( block_size: int = 64, max_swap_rows: int | None = None, index_group: hisparse_runtime.HiSparseIndexGroup | None = None, + max_union_rows: int | None = None, ) -> HiSparseRuntime: runtime = HiSparseRuntime( config=ResolvedHiSparseConfig( top_k=top_k, device_buffer_size=device_buffer_size, + max_union_rows=max_union_rows or device_buffer_size, ), max_num_reqs=max_num_reqs, row_width=row_width, @@ -2015,6 +2017,167 @@ def test_hisparse_maps_speculative_rows_through_request_state(): assert physical.tolist() == [[block_size]] * 4 +@requires_hisparse_ops +@pytest.mark.parametrize( + "num_rows,row_stride,top_k", + [(8, 128, 128), (4, 32, 128), (8, 2048, 2048)], + ids=["disjoint", "overlap", "long_mtp"], +) +def test_hisparse_speculative_rows_resolve_host_misses_consistently( + num_rows, row_stride, top_k +): + """Verification rows of one request must agree on its hot-buffer slots. + + Each row resolved residency in its own thread block against the request's + shared slot ownership and LRU, so rows missing on different host rows could + claim the same free slot. One row's hot index then pointed at a slot holding + another row's KV. Overlapping rows must share one slot per host row, and + long_mtp's rows span several resolution passes. + """ + device = torch.device(DEVICE_TYPE) + block_size, row_width = 64, 64 + num_blocks = 2 * num_rows * top_k // block_size + + kv_pool = torch.randn( + (num_blocks + 1, block_size, row_width), dtype=torch.float32 + ).pin_memory() + flat_pool = kv_pool.reshape(-1, row_width) + runtime = _make_hisparse_runtime( + top_k=top_k, + device_buffer_size=num_rows * top_k, + max_num_reqs=1, + row_width=row_width, + block_size=block_size, + max_swap_rows=num_rows, + ) + runtime.bind_source_cache(kv_pool) + cache = HiSparseCacheHandle(runtime) + cache.all_context_pages_resident = False + + block_table = torch.arange( + 1, num_blocks + 1, dtype=torch.int32, device=device + ).view(1, num_blocks) + req_ids = torch.zeros(num_rows, dtype=torch.int32, device=device) + rows = torch.arange(num_rows, dtype=torch.int32, device=device)[:, None] + base = torch.arange(top_k, dtype=torch.int32, device=device)[None, :] + for step in range(4): + runtime.begin_forward() + # Rows start row_stride apart (disjoint at row_stride == top_k) and + # shift by half a top-k per step: all misses first, then a mix. + topk = (rows * row_stride + base + step * (top_k // 2)) % ( + num_blocks * block_size + ) + hot_indices = cache.swap_in( + req_id_per_token=req_ids, + block_table=block_table, + logical_topk_indices=topk.clone(), + block_size=block_size, + ) + torch.accelerator.synchronize() + + global_ref = _triton_convert_reference_impl( + req_ids, block_table, topk, block_size, top_k + ) + gathered = runtime.hot.attention_cache.reshape(-1, row_width)[ + hot_indices.to(torch.long) + ].cpu() + torch.testing.assert_close(gathered, flat_pool[global_ref.cpu().to(torch.long)]) + + +@requires_hisparse_ops +def test_hisparse_speculative_rows_count_a_shared_load_once(): + """A host row two verification rows select loads once; the reuse is a hit.""" + device = torch.device(DEVICE_TYPE) + block_size, row_width, top_k = 64, 64, 128 + num_blocks = top_k // block_size + kv_pool = torch.randn( + (num_blocks + 1, block_size, row_width), dtype=torch.float32 + ).pin_memory() + runtime = _make_hisparse_runtime( + top_k=top_k, + device_buffer_size=2 * top_k, + max_num_reqs=1, + row_width=row_width, + block_size=block_size, + max_swap_rows=2, + ) + runtime.bind_source_cache(kv_pool) + cache = HiSparseCacheHandle(runtime) + cache.all_context_pages_resident = False + block_table = torch.arange( + 1, num_blocks + 1, dtype=torch.int32, device=device + ).view(1, num_blocks) + topk = torch.arange(top_k, dtype=torch.int32, device=device).repeat(2, 1) + + runtime.begin_forward() + cache.swap_in( + req_id_per_token=torch.zeros(2, dtype=torch.int32, device=device), + block_table=block_table, + logical_topk_indices=topk, + block_size=block_size, + ) + torch.accelerator.synchronize() + + assert runtime.index_group.swap_stats.cpu().tolist() == [top_k, top_k] + + +@requires_hisparse_ops +def test_hisparse_union_table_scales_with_union_bound(): + """Shared memory follows the per-request union bound, not the hot buffer. + + Sizing the table from device_buffer_size needs ~340 KB here and fails. + """ + device = torch.device(DEVICE_TYPE) + block_size, row_width, top_k = 64, 64, 128 + num_blocks = top_k // block_size + kv_pool = torch.randn( + (num_blocks + 1, block_size, row_width), dtype=torch.float32 + ).pin_memory() + runtime = _make_hisparse_runtime( + top_k=top_k, + device_buffer_size=32768, + max_union_rows=top_k, + max_num_reqs=1, + row_width=row_width, + block_size=block_size, + max_swap_rows=1, + ) + runtime.bind_source_cache(kv_pool) + cache = HiSparseCacheHandle(runtime) + cache.all_context_pages_resident = False + req_ids = torch.zeros(1, dtype=torch.int32, device=device) + block_table = torch.arange( + 1, num_blocks + 1, dtype=torch.int32, device=device + ).view(1, num_blocks) + topk = torch.arange(top_k, dtype=torch.int32, device=device).view(1, top_k) + + runtime.begin_forward() + hot_indices = cache.swap_in( + req_id_per_token=req_ids, + block_table=block_table, + logical_topk_indices=topk, + block_size=block_size, + ) + torch.accelerator.synchronize() + + global_ref = _triton_convert_reference_impl( + req_ids, block_table, topk, block_size, top_k + ) + gathered = runtime.hot.attention_cache.reshape(-1, row_width)[ + hot_indices.to(torch.long) + ].cpu() + torch.testing.assert_close( + gathered, kv_pool.reshape(-1, row_width)[global_ref.cpu().to(torch.long)] + ) + + +@requires_hisparse_ops +def test_hisparse_rejects_union_bound_beyond_shared_memory(): + """An oversized union bound fails at construction, not on the first forward.""" + with pytest.raises(ValueError, match="shared memory"): + _make_hisparse_runtime(top_k=2048, device_buffer_size=32768) + + @requires_hisparse_ops def test_hisparse_resident_rows_bypass_hot_lru(): device = torch.device(DEVICE_TYPE) @@ -3001,8 +3164,8 @@ def test_hisparse_gather_prefill_cache_prefers_resident_rows(): torch.testing.assert_close(staged_flat[i], expected) -def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypatch): - """FP8 decode must use active dimensions when common metadata is padded.""" +def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention(): + """Verification rows resolve in one swap_in; FP8 attention sees them batched.""" device = torch.device("cpu") num_decodes = 2 query_len = 3 @@ -3013,48 +3176,31 @@ def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypa .view(-1, 1) .expand(-1, 4) ) - steps: list[int] = [] kernel_shapes: list[torch.Size] = [] def swap_in(req_ids, *, logical_topk_indices, **kwargs): # noqa: ARG001 - steps.append(int(logical_topk_indices[0, 0])) - output = kwargs["attention_indices_out"] - if output is not None: - output.copy_(logical_topk_indices + 10) - return topk[:num_decodes] + return logical_topk_indices + 10 def run_kernel(self, *, q, **kwargs): # noqa: ARG001 kernel_shapes.append(q.shape) return q[..., :1], None - runtime = SimpleNamespace( - hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)) - ) - leader_cache = SimpleNamespace( - runtime=runtime, - source_block_table=torch.empty( - num_decodes, 1, dtype=torch.int32, device=device + cache = SimpleNamespace( + runtime=SimpleNamespace( + hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)) ), - swap_in=MagicMock(side_effect=swap_in), - ) - follower_cache = SimpleNamespace( - runtime=runtime, source_block_table=torch.empty( num_decodes, 1, dtype=torch.int32, device=device ), swap_in=MagicMock(side_effect=swap_in), ) index_group = object.__new__(HiSparseMLAIndexGroup) - index_group.caches = [leader_cache, follower_cache] + index_group.caches = [cache] index_group.physical_topk_indices = torch.empty( (num_tokens + 1, 4), dtype=torch.int32, device=device ) - index_group.valid_topk_counts = torch.empty( - num_tokens + 1, dtype=torch.int32, device=device - ) - index_group.request_ids = torch.arange( - num_decodes, dtype=torch.int32, device=device - ) + index_group.row_indices = torch.arange(num_tokens, dtype=torch.int32, device=device) + index_group.request_ids = torch.empty_like(index_group.row_indices) impl = SimpleNamespace( kv_lora_rank=1, index_group=index_group, @@ -3062,84 +3208,83 @@ def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypa ) impl._fp8_flash_mla_kernel = MethodType(run_kernel, impl) metadata = SimpleNamespace( - num_decodes=num_tokens, - num_decode_tokens=num_tokens, - decode_max_query_len=1, query_start_loc=torch.arange( - 0, - num_tokens + 1, - query_len, - dtype=torch.int32, - device=device, + 0, num_tokens + 1, query_len, dtype=torch.int32, device=device ), - block_table=torch.empty(num_decodes, 1, dtype=torch.int32, device=device), + req_id_per_token=torch.arange( + num_decodes, dtype=torch.int32, device=device + ).repeat_interleave(query_len), block_size=64, ) output = FlashMLASparseImpl._host_backed_fp8_decode( - impl, - q, - topk, - metadata, - SimpleNamespace(), - num_decodes, - query_len, + impl, q, topk, metadata, SimpleNamespace(), num_decodes ) - assert steps == [0, 1, 2] + cache.swap_in.assert_called_once() + torch.testing.assert_close( + cache.swap_in.call_args.args[0], metadata.req_id_per_token + ) + torch.testing.assert_close( + cache.swap_in.call_args.kwargs["logical_topk_indices"], topk + ) assert kernel_shapes == [(num_decodes, query_len, 2, 4)] assert output.shape == (num_tokens, 2, 1) - torch.testing.assert_close( - index_group.physical_topk_indices[:num_tokens], topk + 10 + + +def test_hisparse_keeps_speculative_rows_as_decodes(): + """HiSparse limits decodes to verification requests, not short extends. + + FlashAttn sparse MLA used to classify MTP verification as prefill under + HiSparse, whose residency-dependent route a FULL CUDA graph freezes at + capture. + """ + builder = object.__new__(FlashAttnMLASparseMetadataBuilder) + builder.vllm_config = SimpleNamespace( + attention_config=SimpleNamespace(hisparse_config=object()), + speculative_config=SimpleNamespace( + num_speculative_tokens=3, parallel_drafting=False + ), + parallel_config=SimpleNamespace(decode_context_parallel_size=1), + use_v2_model_runner=True, ) - follower_indices = index_group.convert_decode_logical_to_physical_topk( - 1, - topk, - metadata, - return_valid_counts=False, - num_decodes=num_decodes, - decode_query_len=query_len, - ) - assert steps == [0, 1, 2, 0, 1, 2] - torch.testing.assert_close(follower_indices, topk + 10) + builder._init_reorder_batch_threshold(256, supports_spec_as_decode=True) + + assert builder.reorder_batch_threshold == 4 -def test_hisparse_varlen_decode_uses_canonical_request_rows(): - """Graph metadata's query upper bound must preserve packed token rows.""" - num_decodes = 2 - topk = torch.arange(18, dtype=torch.int32).view(9, 2) - - def swap_in(_request_ids, *, logical_topk_indices, **_kwargs): - return logical_topk_indices + 10 +def test_hisparse_decode_skips_padding_rows(): + """Padding rows are masked, and the ids stay in the group's persistent buffer. + The resolver reads them on the HiSparse copy stream, where a compute-stream + temporary can already be recycled (CUDA graphs do), scrambling rows' requests. + """ + topk = torch.arange(20, dtype=torch.int32).view(10, 2) cache = SimpleNamespace( - source_block_table=torch.zeros((num_decodes, 1), dtype=torch.int32), - swap_in=MagicMock(side_effect=swap_in), + source_block_table=torch.zeros((2, 1), dtype=torch.int32), + swap_in=MagicMock(return_value=topk), ) index_group = object.__new__(HiSparseMLAIndexGroup) index_group.caches = [cache] - index_group.physical_topk_indices = torch.empty((10, 2), dtype=torch.int32) - index_group.valid_topk_counts = torch.empty(10, dtype=torch.int32) - index_group.request_ids = torch.arange(num_decodes, dtype=torch.int32) + index_group.physical_topk_indices = torch.empty((11, 2), dtype=torch.int32) + index_group.row_indices = torch.arange(10, dtype=torch.int32) + index_group.request_ids = torch.empty_like(index_group.row_indices) metadata = SimpleNamespace( - num_decodes=num_decodes, - num_decode_tokens=topk.shape[0], - decode_max_query_len=15, query_start_loc=torch.tensor([0, 8, 9], dtype=torch.int32), + req_id_per_token=torch.tensor([0] * 8 + [1, 0], dtype=torch.int32), block_size=64, ) - physical_topk = index_group.convert_decode_logical_to_physical_topk( - 0, - topk, - metadata, - return_valid_counts=False, + index_group.convert_logical_to_physical_topk( + 0, topk, metadata, block_stride_rows=None, return_valid_counts=False ) - torch.testing.assert_close(physical_topk, topk + 10) - request_rows = cache.swap_in.call_args_list[0].args[0] - torch.testing.assert_close(request_rows, index_group.request_ids) + request_ids = cache.swap_in.call_args.args[0] + assert request_ids.data_ptr() == index_group.request_ids.data_ptr() + torch.testing.assert_close( + request_ids, torch.tensor([0] * 8 + [1, -1], dtype=torch.int32) + ) def test_flashinfer_hisparse_decode_runs_batched_attention(): @@ -3168,7 +3313,7 @@ def test_flashinfer_hisparse_decode_runs_batched_attention(): ) index_group = object.__new__(HiSparseMLAIndexGroup) index_group.caches = [cache_handle] - index_group.convert_decode_logical_to_physical_topk = MethodType( + index_group.convert_logical_to_physical_topk = MethodType( convert_decode, index_group ) impl = object.__new__(FlashInferMLASparseImpl) @@ -3200,7 +3345,7 @@ def test_flashattn_hisparse_decode_uses_index_group(): physical = topk.clone() counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE) index_group = object.__new__(HiSparseMLAIndexGroup) - index_group.convert_decode_logical_to_physical_topk = MagicMock( + index_group.convert_logical_to_physical_topk = MagicMock( return_value=(physical, counts) ) index_group.physical_kv_cache = MagicMock( @@ -3223,7 +3368,7 @@ def test_flashattn_hisparse_decode_uses_index_group(): assert output.shape == (num_tokens, 2, 1) assert lse is None - index_group.convert_decode_logical_to_physical_topk.assert_called_once() + index_group.convert_logical_to_physical_topk.assert_called_once() impl._run_mqa_kernel.assert_called_once() @@ -3233,9 +3378,7 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group(): topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE) physical = topk.clone() index_group = object.__new__(HiSparseMLAIndexGroup) - index_group.convert_decode_logical_to_physical_topk = MagicMock( - return_value=physical - ) + index_group.convert_logical_to_physical_topk = MagicMock(return_value=physical) index_group.physical_kv_cache = MagicMock( return_value=torch.empty(1, device=DEVICE_TYPE) ) @@ -3259,7 +3402,7 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group(): assert output.shape == (num_tokens, 2, 1) assert lse is None - index_group.convert_decode_logical_to_physical_topk.assert_called_once() + index_group.convert_logical_to_physical_topk.assert_called_once() impl._run_mqa_kernel.assert_called_once() @@ -3291,27 +3434,6 @@ def test_hisparse_resident_prefill_uses_attention_block_stride(): assert index_group._convert_once.call_args.kwargs["block_stride_rows"] == 832 -def test_hisparse_shared_sparse_builder_routes_multi_token_chunks_to_prefill(): - """Default sparse backends must route multi-token chunks to prefill.""" - builder = object.__new__(FlashInferMLASparseMetadataBuilder) - builder.vllm_config = SimpleNamespace( - attention_config=SimpleNamespace(hisparse_config=object()), - speculative_config=SimpleNamespace( - num_speculative_tokens=4, - parallel_drafting=False, - ), - parallel_config=SimpleNamespace(decode_context_parallel_size=1), - ) - - builder._init_reorder_batch_threshold( - 1024, - supports_spec_as_decode=True, - supports_dcp_with_varlen=True, - ) - - assert builder.reorder_batch_threshold == 1 - - def test_hisparse_flashmla_reorders_full_speculative_window_as_decode(): builder = object.__new__(FlashMLASparseMetadataBuilder) builder.vllm_config = SimpleNamespace( diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 40ac8b56ee..1833bca8ab 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -1156,6 +1156,7 @@ def test_hisparse_cache_handles_join_index_groups_during_construction(monkeypatc resolved = hisparse_runtime_module.ResolvedHiSparseConfig( top_k=4, device_buffer_size=8, + max_union_rows=8, ) monkeypatch.setattr(hisparse_runtime_module, "_has_hisparse_ops", lambda: True) monkeypatch.setattr( diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index ee7409b190..3788359342 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -165,7 +165,6 @@ T = TypeVar("T", bound=SparseMLACommonMetadata) class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): metadata_cls: type[T] require_uniform_decodes: ClassVar[bool] = False - hisparse_supports_multi_token_decode: ClassVar[bool] = False def __init__( self, @@ -259,8 +258,6 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]): ) -> None: if self.vllm_config.attention_config.hisparse_config is not None: reorder_batch_threshold = 1 - if not self.hisparse_supports_multi_token_decode: - supports_spec_as_decode = False super()._init_reorder_batch_threshold( reorder_batch_threshold, supports_spec_as_decode, diff --git a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py index 0c3f0f4288..57f4462759 100644 --- a/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashattn_mla_sparse.py @@ -207,10 +207,11 @@ class FlashAttnMLASparseImpl(SparseMLACommonImpl[FlashAttnMLASparseMetadata]): outputs = [] if num_decode_tokens: physical_topk, valid_counts = ( - index_group.convert_decode_logical_to_physical_topk( + index_group.convert_logical_to_physical_topk( self.index_group_index, topk_indices[:num_decode_tokens], attn_metadata, + block_stride_rows=None, return_valid_counts=True, ) ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py index 5af50ef1bd..05af2de797 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse.py @@ -273,7 +273,6 @@ class FlashInferMLASparseTRTLLMMetadataBuilder(FlashInferMLASparseMetadataBuilde """Metadata builder for the SM100 TRT-LLM sparse MLA kernel.""" _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS - hisparse_supports_multi_token_decode: ClassVar[bool] = True def __init__( self, @@ -502,10 +501,11 @@ class FlashInferMLASparseImpl(SparseMLACommonImpl[FlashInferMLASparseMetadata]): decode_lse: torch.Tensor | None = None if num_decode_tokens > 0: physical_topk, valid_counts = ( - index_group.convert_decode_logical_to_physical_topk( + index_group.convert_logical_to_physical_topk( self.index_group_index, topk_indices[:num_decode_tokens], attn_metadata, + block_stride_rows=None, return_valid_counts=True, ) ) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py index e59c9ce46d..2c918c0e4a 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py @@ -125,10 +125,11 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada if num_decode_tokens: topk_indices_physical = cast( torch.Tensor, - index_group.convert_decode_logical_to_physical_topk( + index_group.convert_logical_to_physical_topk( self.index_group_index, topk_indices[:num_decode_tokens], attn_metadata, + block_stride_rows=None, return_valid_counts=False, ), ) diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 014358f530..8371fb8b85 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -317,7 +317,6 @@ class FlashMLASparseMetadataBuilder( ): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH require_uniform_decodes: ClassVar[bool] = True - hisparse_supports_multi_token_decode: ClassVar[bool] = True metadata_cls = FlashMLASparseMetadata def __init__( @@ -818,10 +817,11 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): num_decode_tokens = attn_metadata.num_decode_tokens if num_decode_tokens > 0: decode_topk, decode_lengths = ( - index_group.convert_decode_logical_to_physical_topk( + index_group.convert_logical_to_physical_topk( self.index_group_index, topk_indices[:num_decode_tokens], attn_metadata, + block_stride_rows=None, return_valid_counts=True, ) ) @@ -1027,7 +1027,6 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): attn_metadata, fp8_metadata.decode.kernel_metadata, num_decodes, - fp8_metadata.decode.decode_query_len, ) # Reshape q: (num_decode_tokens, num_heads, head_dim) # -> (num_decodes, seq_len, num_heads, head_dim) @@ -1267,16 +1266,14 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): attn_metadata: FlashMLASparseMetadata, kernel_metadata: FlashMLASparseMetadata.FP8KernelMetadata, num_decodes: int, - decode_query_len: int, ) -> torch.Tensor: assert isinstance(self.index_group, HiSparseMLAIndexGroup) - physical_topk = self.index_group.convert_decode_logical_to_physical_topk( + physical_topk = self.index_group.convert_logical_to_physical_topk( self.index_group_index, topk_indices, attn_metadata, + block_stride_rows=None, return_valid_counts=False, - num_decodes=num_decodes, - decode_query_len=decode_query_len, ) assert isinstance(physical_topk, torch.Tensor) q = reshape_query_for_spec_decode(q, num_decodes) diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index a3c6df6fe6..21d578efd0 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -39,6 +39,7 @@ class SparseMLAIndexGroup: logical_topk_indices: torch.Tensor physical_topk_indices: torch.Tensor valid_topk_counts: torch.Tensor + row_indices: torch.Tensor request_ids: torch.Tensor side_stream: torch.Stream logical_topk_ready: torch.Event @@ -230,12 +231,10 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup): *, block_stride_rows: int | None, return_valid_counts: bool, - req_id_per_token: torch.Tensor | None = None, ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: cache = self.cache(layer_index) num_tokens = logical_topk_indices.shape[0] - if req_id_per_token is None: - req_id_per_token = attn_metadata.req_id_per_token[:num_tokens] + req_id_per_token = attn_metadata.req_id_per_token[:num_tokens] if num_tokens > self.physical_topk_indices.shape[0]: # Prefill-sized batches do not fit the decode residency workspace. # Non-resident prefills are staged before reaching this path. @@ -253,113 +252,21 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup): ) source_block_table = cache.source_block_table assert source_block_table is not None + # CUDA-graph padding rows past the batch's tokens map to request 0; + # mark them -1 so residency resolution skips them. + request_ids = self.request_ids[:num_tokens] + request_ids.copy_(req_id_per_token) + request_ids.masked_fill_( + self.row_indices[:num_tokens] >= attn_metadata.query_start_loc[-1], -1 + ) return cache.swap_in( - req_id_per_token, + request_ids, block_table=source_block_table, logical_topk_indices=logical_topk_indices, block_size=attn_metadata.block_size, return_valid_counts=return_valid_counts, ) - def convert_decode_logical_to_physical_topk( - self, - layer_index: int, - logical_topk_indices: torch.Tensor, - attn_metadata: Any, - *, - return_valid_counts: bool, - num_decodes: int | None = None, - decode_query_len: int | None = None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - num_tokens = logical_topk_indices.shape[0] - if num_decodes is None: - num_decodes = attn_metadata.num_decodes - if decode_query_len is None: - decode_query_len = attn_metadata.decode_max_query_len - if decode_query_len == 1: - return self.convert_logical_to_physical_topk( - layer_index, - logical_topk_indices, - attn_metadata, - block_stride_rows=None, - return_valid_counts=return_valid_counts, - req_id_per_token=self.request_ids[:num_decodes], - ) - - if num_tokens != num_decodes * decode_query_len: - query_start_loc = attn_metadata.query_start_loc[: num_decodes + 1] - physical_topk_indices = self.physical_topk_indices[: num_tokens + 1] - valid_topk_counts = self.valid_topk_counts[: num_tokens + 1] - request_ids = self.request_ids[:num_decodes] - cache = self.cache(layer_index) - source_block_table = cache.source_block_table - assert source_block_table is not None - for step in range(decode_query_len): - token_indices = query_start_loc[:-1] + step - active = (token_indices < query_start_loc[1:]) & ( - token_indices < num_tokens - ) - output_indices = torch.where( - active, token_indices, torch.full_like(token_indices, num_tokens) - ).long() - token_indices = token_indices.clamp(0, num_tokens - 1).long() - step_topk = logical_topk_indices.index_select( - 0, token_indices - ).masked_fill(~active.unsqueeze(1), -1) - step_result = cache.swap_in( - request_ids, - block_table=source_block_table, - logical_topk_indices=step_topk, - block_size=attn_metadata.block_size, - return_valid_counts=return_valid_counts, - ) - if layer_index == 0: - if return_valid_counts: - step_indices, step_counts = step_result - valid_topk_counts.index_copy_(0, output_indices, step_counts) - else: - step_indices = step_result - physical_topk_indices.index_copy_(0, output_indices, step_indices) - physical_topk_indices = physical_topk_indices[:num_tokens] - if return_valid_counts: - return physical_topk_indices, valid_topk_counts[:num_tokens] - return physical_topk_indices - - assert num_tokens == num_decodes * decode_query_len - logical_topk_by_request = logical_topk_indices.view( - num_decodes, decode_query_len, -1 - ) - physical_topk_by_request = self.physical_topk_indices[:num_tokens].view( - num_decodes, decode_query_len, -1 - ) - valid_topk_by_request = self.valid_topk_counts[:num_tokens].view( - num_decodes, decode_query_len - ) - request_ids = self.request_ids[:num_decodes] - for step in range(decode_query_len): - cache = self.cache(layer_index) - source_block_table = cache.source_block_table - assert source_block_table is not None - cache.swap_in( - request_ids, - block_table=source_block_table, - logical_topk_indices=logical_topk_by_request[:, step], - block_size=attn_metadata.block_size, - return_valid_counts=return_valid_counts, - attention_indices_out=( - physical_topk_by_request[:, step] if layer_index == 0 else None - ), - valid_counts_out=( - valid_topk_by_request[:, step] - if layer_index == 0 and return_valid_counts - else None - ), - ) - physical_topk_indices = physical_topk_by_request.view(num_tokens, -1) - if return_valid_counts: - return physical_topk_indices, valid_topk_by_request.view(num_tokens) - return physical_topk_indices - def stage_prefill_rows( self, layer_index: int, kv_cache: torch.Tensor, attn_metadata: Any ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: @@ -477,7 +384,12 @@ class SparseMLAIndexGroupBuilder: dtype=torch.int32, device=self.logical_topk_indices.device, ), - request_ids=torch.arange( + row_indices=torch.arange( + workspace_rows, + dtype=torch.int32, + device=self.logical_topk_indices.device, + ), + request_ids=torch.empty( workspace_rows, dtype=torch.int32, device=self.logical_topk_indices.device, diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index 4614e1aa32..2d9c6cc83f 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -19,7 +19,8 @@ from vllm.distributed import get_tp_group from vllm.forward_context import get_forward_context, in_piecewise_cudagraph from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.utils.math_utils import round_up +from vllm.utils.math_utils import cdiv, round_up +from vllm.utils.mem_utils import get_max_shared_memory_bytes from vllm.utils.torch_utils import current_stream from vllm.v1.attention.backend import max_decode_query_len from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion @@ -96,6 +97,9 @@ class PagedCacheView: class ResolvedHiSparseConfig: top_k: int device_buffer_size: int + # Most host rows one request's decode rows can select in a step: every + # verification row's top-k. + max_union_rows: int eager_host_mirror: bool = True @classmethod @@ -151,10 +155,38 @@ class ResolvedHiSparseConfig: return cls( top_k=model_top_k, device_buffer_size=device_buffer_size, + max_union_rows=min_device_buffer_size, eager_host_mirror=config.eager_host_mirror, ) +def _check_residency_shared_memory( + device: torch.device, config: ResolvedHiSparseConfig +) -> None: + """Reject a config whose residency resolver exceeds per-block shared memory.""" + # Mirrors hisparse_resolve_residency's layout: union hash keys and values + # (plus an empty slot), per-chunk offsets, five counters, done bits, and + # the int16 compacted LRU. + hot_size = config.device_buffer_size + num_chunks = cdiv(hot_size, 32) + required = 4 * (2 * (config.max_union_rows + 1) + 3 * num_chunks + 7) + 2 * hot_size + device_index = ( + device.index + if device.index is not None + else torch.accelerator.current_device_index() + ) + available = get_max_shared_memory_bytes(device_index) + if required > available: + raise ValueError( + "HiSparse residency resolution needs " + f"{required} B of shared memory per block for " + f"device_buffer_size={config.device_buffer_size} and " + f"{config.max_union_rows} rows per request " + "(decode query length x index_topk), but the device allows " + f"{available} B. Lower device_buffer_size or num_speculative_tokens." + ) + + def check_hisparse_host_memory(pool_bytes: int) -> None: """Reject a physical host-pool allocation that cannot fit in RAM.""" mem = psutil.virtual_memory() @@ -677,6 +709,9 @@ class HiSparseRuntime: # Logical slots per request. Physical rows come from its ephemeral HMA # block table and need not be contiguous in the shared slab. self.region_stride = config.device_buffer_size + self.max_union_rows = config.max_union_rows + if self.device.type == "cuda": + _check_residency_shared_memory(self.device, config) row_bytes = row_width * kv_dtype.itemsize if row_bytes % 16 != 0: @@ -919,6 +954,7 @@ class HiSparseRuntime: group.lru_slots, request_state_indices, self.region_stride, + self.max_union_rows, None, group.swap_stats, physical_topk_indices, From a8e069f81b4842723a04a84fad2cd5f757820af3 Mon Sep 17 00:00:00 2001 From: xiaozhoupy Date: Wed, 30 Sep 2026 11:04:10 +0800 Subject: [PATCH 074/248] [Bugfix][MiniMax M3] Share target embeddings with MTP under PP (#58648) Signed-off-by: Peiyuan Zhou Co-authored-by: Codex Co-authored-by: Woosuk Kwon Co-authored-by: Jee Jee Li --- .../test_spec_decode_embed_sharing_pp.py | 188 ++++++++++++++++++ vllm/model_executor/models/utils.py | 15 +- vllm/models/minimax_m3/amd/model.py | 5 +- vllm/models/minimax_m3/amd/mtp.py | 11 + vllm/models/minimax_m3/nvidia/model.py | 5 +- vllm/models/minimax_m3/nvidia/mtp.py | 11 + 6 files changed, 229 insertions(+), 6 deletions(-) diff --git a/tests/v1/worker/test_spec_decode_embed_sharing_pp.py b/tests/v1/worker/test_spec_decode_embed_sharing_pp.py index 1a1d96a6b4..36b1268ca3 100644 --- a/tests/v1/worker/test_spec_decode_embed_sharing_pp.py +++ b/tests/v1/worker/test_spec_decode_embed_sharing_pp.py @@ -1,15 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib from types import SimpleNamespace import pytest import torch import torch.nn as nn +from vllm.model_executor.model_loader.mtp_validation import ( + disable_mtp_completeness_check, +) from vllm.model_executor.models.utils import ( PPMissingLayer, spec_decode_needs_target_embed, ) +from vllm.platforms import current_platform from vllm.v1.worker.gpu.spec_decode.eagle import utils as eagle_utils VOCAB, HIDDEN = 32, 8 @@ -108,3 +113,186 @@ def test_target_embedding_provisioning( speculative_config = None if method is None else SimpleNamespace(method=method) config = SimpleNamespace(speculative_config=speculative_config) assert spec_decode_needs_target_embed(config) is expected + + +@pytest.mark.parametrize( + "pp_size,is_last_rank,expected", + [(2, True, True), (2, False, False), (1, True, False)], +) +def test_mtp_target_embedding_requires_model_opt_in( + monkeypatch, pp_size, is_last_rank, expected +): + monkeypatch.setattr( + "vllm.distributed.parallel_state.get_pp_group", + _fake_pp(pp_size, is_last_rank), + ) + config = SimpleNamespace(speculative_config=SimpleNamespace(method="mtp")) + assert spec_decode_needs_target_embed(config, include_mtp=True) is expected + assert not spec_decode_needs_target_embed(config) + + +@pytest.fixture +def m3_modules(): + if not current_platform.is_cuda_alike(): + pytest.skip("M3 model imports require CUDA or ROCm dependencies") + backend = "amd" if current_platform.is_rocm() else "nvidia" + root = f"vllm.models.minimax_m3.{backend}" + return importlib.import_module(f"{root}.mtp"), importlib.import_module( + f"{root}.model" + ) + + +@pytest.fixture +def m3_config(): + hf_config = SimpleNamespace( + vocab_size=VOCAB, + hidden_size=HIDDEN, + num_local_experts=0, + num_hidden_layers=0, + rms_norm_eps=1e-5, + ) + return SimpleNamespace( + use_v2_model_runner=True, + load_config=SimpleNamespace(load_format="auto"), + model_config=SimpleNamespace(hf_text_config=hf_config), + speculative_config=SimpleNamespace( + method="mtp", draft_model_config=SimpleNamespace(hf_config=hf_config) + ), + quant_config=None, + cache_config=None, + ) + + +@pytest.fixture +def make_m3_draft(monkeypatch, m3_modules, m3_config): + mtp, _ = m3_modules + inner = _inner(_embed()) + inner.layers = nn.ModuleDict({"0": nn.Linear(1, 1, bias=False)}) + inner.num_mtp_layers = 1 + monkeypatch.setattr(mtp, "MiniMaxM3MultiTokenPredictor", lambda **_: inner) + monkeypatch.setattr(mtp, "ParallelLMHead", lambda *_, **kw: _embed()) + monkeypatch.setattr(mtp, "LogitsProcessor", lambda *_: None) + monkeypatch.setattr( + mtp, "fused_moe_make_expert_params_mapping", lambda *_, **kw: [] + ) + monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(2)) + monkeypatch.setattr(eagle_utils, "get_pp_group", _fake_pp(2)) + return lambda: mtp.MiniMaxM3MTP(vllm_config=m3_config) + + +@pytest.mark.parametrize( + "key", + [None, "model.embed_tokens.weight", "language_model.model.embed_tokens.weight"], +) +def test_m3_checkpoint_embedding_ownership(make_m3_draft, key): + """Actual M3 loading determines sharing and survives an unrelated reload.""" + draft = make_m3_draft() + assert draft.has_own_embed_tokens + weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))] + if key is not None: + weights.append((key, torch.ones(VOCAB, HIDDEN))) + loaded = draft.load_weights(weights) + assert ("model.embed_tokens.weight" in loaded) is (key is not None) + with disable_mtp_completeness_check(): + draft.load_weights([("lm_head.weight", torch.zeros(VOCAB, HIDDEN))]) + assert draft.has_own_embed_tokens is (key is not None) + target = _inner(_embed(fill=2.0)) + original = draft.model.embed_tokens + eagle_utils.maybe_share_target_embed(draft, draft.model, target) + if key is None: + assert draft.model.embed_tokens is target.embed_tokens + else: + assert draft.model.embed_tokens is original + torch.testing.assert_close(original.weight, torch.ones(VOCAB, HIDDEN)) + + +@pytest.mark.parametrize( + "load_format", ["dummy", "sharded_state", "runai_streamer_sharded", "tensorizer"] +) +def test_m3_direct_loader_embedding_ownership(make_m3_draft, m3_config, load_format): + """Directly loaded embeddings survive transfers and share only when equal.""" + m3_config.load_config.load_format = load_format + draft = make_m3_draft() + draft.load_state_dict( + {key: torch.ones_like(value) for key, value in draft.state_dict().items()} + ) + with disable_mtp_completeness_check(): + draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))]) + target = _inner(_embed(fill=2.0)) + original = draft.model.embed_tokens + eagle_utils.maybe_share_target_embed(draft, draft.model, target) + assert draft.model.embed_tokens is ( + target.embed_tokens if load_format == "dummy" else original + ) + if load_format != "dummy": + equal_target = _inner(_embed(fill=1.0)) + eagle_utils.maybe_share_target_embed(draft, draft.model, equal_target) + assert draft.model.embed_tokens is equal_target.embed_tokens + + +@pytest.mark.parametrize("use_v2,pp_size", [(True, 1), (False, 1), (False, 2)]) +def test_m3_preserves_legacy_embedding_sharing( + monkeypatch, m3_modules, m3_config, make_m3_draft, use_v2, pp_size +): + mtp, _ = m3_modules + m3_config.use_v2_model_runner = use_v2 + monkeypatch.setattr(mtp, "get_pp_group", _fake_pp(pp_size)) + draft = make_m3_draft() + draft.load_weights( + [ + ("model.mtp.layers.0.weight", torch.ones(1, 1)), + ("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN)), + ] + ) + assert not hasattr(draft, "has_own_embed_tokens") + + +@pytest.mark.parametrize( + "rank,use_v2,method,expected", + [ + (0, True, "mtp", True), + (1, True, "mtp", False), + (2, True, "mtp", True), + (2, False, "mtp", False), + (2, True, None, False), + ], +) +def test_m3_target_allocates_embedding_on_required_stage( + monkeypatch, m3_modules, m3_config, rank, use_v2, method, expected +): + _, model = m3_modules + pp = lambda: SimpleNamespace( + world_size=3, is_first_rank=rank == 0, is_last_rank=rank == 2 + ) + monkeypatch.setattr(model, "get_pp_group", pp) + monkeypatch.setattr("vllm.distributed.parallel_state.get_pp_group", pp) + monkeypatch.setattr(model, "VocabParallelEmbedding", lambda *_, **kw: _embed()) + monkeypatch.setattr(model, "make_layers", lambda *_, **kw: (0, 0, nn.ModuleList())) + monkeypatch.setattr(model, "MiniMAXGemmaRMSNorm", lambda *_, **kw: nn.Identity()) + m3_config.use_v2_model_runner = use_v2 + if method is None: + m3_config.speculative_config = None + target = model.MiniMaxM3Model(vllm_config=m3_config) + assert isinstance(target.embed_tokens, nn.Embedding) is expected + + +@pytest.mark.parametrize("initial_embed", [False, True]) +def test_m3_weight_transfer_updates_embedding_ownership(make_m3_draft, initial_embed): + """Layer transfers retain ownership; newly loaded embeddings become owned.""" + draft = make_m3_draft() + weights = [("model.mtp.layers.0.weight", torch.ones(1, 1))] + if initial_embed: + weights.append(("model.embed_tokens.weight", torch.ones(VOCAB, HIDDEN))) + draft.load_weights(weights) + with disable_mtp_completeness_check(): + draft.load_weights([("model.mtp.layers.0.weight", torch.zeros(1, 1))]) + assert draft.has_own_embed_tokens is initial_embed + draft.load_weights( + [("model.embed_tokens.weight", torch.full((VOCAB, HIDDEN), 3.0))] + ) + assert draft.has_own_embed_tokens + target = _inner(_embed(fill=2.0)) + original = draft.model.embed_tokens + eagle_utils.maybe_share_target_embed(draft, draft.model, target) + assert draft.model.embed_tokens is original + torch.testing.assert_close(original.weight, torch.full((VOCAB, HIDDEN), 3.0)) diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index e762ad1c47..c6498319cd 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -844,17 +844,24 @@ class PPMissingLayer(torch.nn.Identity): return args[0] if args else next(iter(kwargs.values())) -def spec_decode_needs_target_embed(vllm_config: VllmConfig) -> bool: - """Whether the last PP rank needs the target input embedding.""" +def spec_decode_needs_target_embed( + vllm_config: VllmConfig, *, include_mtp: bool = False +) -> bool: + """Whether the last PP rank needs the target input embedding. + + MTP targets opt in when their drafter supports sharing this embedding. + """ from vllm.distributed.parallel_state import get_pp_group speculative_config = vllm_config.speculative_config - if speculative_config is None or speculative_config.method not in ( + if speculative_config is None: + return False + if speculative_config.method not in ( "eagle", "eagle3", "dflash", "dspark", - ): + ) and not (include_mtp and speculative_config.method == "mtp"): return False pp = get_pp_group() return pp.world_size > 1 and pp.is_last_rank diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 40daa4a6ee..9105f97af8 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -85,6 +85,7 @@ from vllm.model_executor.models.utils import ( make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, + spec_decode_needs_target_embed, ) from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model from vllm.models.minimax_m3.amd.indexer_aiter import ( @@ -1370,7 +1371,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): self.vocab_size = config.vocab_size - if get_pp_group().is_first_rank: + if get_pp_group().is_first_rank or spec_decode_needs_target_embed( + vllm_config, include_mtp=vllm_config.use_v2_model_runner + ): self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, diff --git a/vllm/models/minimax_m3/amd/mtp.py b/vllm/models/minimax_m3/amd/mtp.py index cfb26d7948..2f7dd8a831 100644 --- a/vllm/models/minimax_m3/amd/mtp.py +++ b/vllm/models/minimax_m3/amd/mtp.py @@ -24,6 +24,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig +from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -164,6 +165,10 @@ class MiniMaxM3MTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1: + # Preserve embeddings from loaders that bypass load_weights. + self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy" + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config self.quant_config = vllm_config.quant_config @@ -333,4 +338,10 @@ class MiniMaxM3MTP(nn.Module): f"Failed to load MTP layer {layer_idx} weights from checkpoint." ) + if hasattr(self, "has_own_embed_tokens"): + if "model.embed_tokens.weight" in loaded_params: + self.has_own_embed_tokens = True + elif is_mtp_completeness_check_enabled(): + # A complete checkpoint without embeddings borrows the target's. + self.has_own_embed_tokens = False return loaded_params diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index c16918bd37..d1c0a01c40 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -68,6 +68,7 @@ from vllm.model_executor.models.utils import ( make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, + spec_decode_needs_target_embed, ) from vllm.model_executor.models.vision import run_dp_sharded_mrope_vision_model from vllm.models.minimax_m3.common.encoder_cudagraph import ( @@ -803,7 +804,9 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): self.vocab_size = config.vocab_size - if get_pp_group().is_first_rank: + if get_pp_group().is_first_rank or spec_decode_needs_target_embed( + vllm_config, include_mtp=vllm_config.use_v2_model_runner + ): self.embed_tokens = VocabParallelEmbedding( config.vocab_size, config.hidden_size, diff --git a/vllm/models/minimax_m3/nvidia/mtp.py b/vllm/models/minimax_m3/nvidia/mtp.py index 832c872a10..9c10a52b7c 100644 --- a/vllm/models/minimax_m3/nvidia/mtp.py +++ b/vllm/models/minimax_m3/nvidia/mtp.py @@ -8,6 +8,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig +from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -146,6 +147,10 @@ class MiniMaxM3MTP(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + if vllm_config.use_v2_model_runner and get_pp_group().world_size > 1: + # Preserve embeddings from loaders that bypass load_weights. + self.has_own_embed_tokens = vllm_config.load_config.load_format != "dummy" + assert vllm_config.speculative_config is not None self.config = vllm_config.speculative_config.draft_model_config.hf_config self.quant_config = vllm_config.quant_config @@ -315,4 +320,10 @@ class MiniMaxM3MTP(nn.Module): f"Failed to load MTP layer {layer_idx} weights from checkpoint." ) + if hasattr(self, "has_own_embed_tokens"): + if "model.embed_tokens.weight" in loaded_params: + self.has_own_embed_tokens = True + elif is_mtp_completeness_check_enabled(): + # A complete checkpoint without embeddings borrows the target's. + self.has_own_embed_tokens = False return loaded_params From 42d9f8ccdbbb7e24c173272809c66ad627a0de9d Mon Sep 17 00:00:00 2001 From: shiyuan680 <72335504+shiyuan680@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:13:43 +0800 Subject: [PATCH 075/248] support rl feature : weight checker (#51350) Signed-off-by: shiyuan680 Signed-off-by: Ronald1995 Signed-off-by: aoshen02 Co-authored-by: shiyuan680 Co-authored-by: Ronald1995 Co-authored-by: DeepSeek Harness Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 Co-authored-by: aoshen02 --- docs/training/rlhf.md | 2 +- docs/training/weight_checker.md | 60 ++++++++ rust/src/server/src/routes.rs | 2 + rust/src/server/src/routes/tests.rs | 76 ++++++++++ rust/src/server/src/routes/weight_checker.rs | 113 +++++++++++++++ .../state_transitions/test_weight_checker.py | 133 ++++++++++++++++++ .../model_loader/test_weight_checksum.py | 93 ++++++++++++ tests/v1/engine/test_engine_core_client.py | 18 +++ vllm/engine/protocol.py | 4 + vllm/entrypoints/serve/dev/rlhf/api_router.py | 39 ++++- .../model_loader/weight_checksum.py | 43 ++++++ vllm/v1/engine/async_llm.py | 3 + vllm/v1/engine/core.py | 3 + vllm/v1/engine/core_client.py | 4 + vllm/v1/worker/gpu_worker.py | 20 +++ 15 files changed, 611 insertions(+), 2 deletions(-) create mode 100644 docs/training/weight_checker.md create mode 100644 rust/src/server/src/routes/weight_checker.rs create mode 100644 tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py create mode 100644 tests/model_executor/model_loader/test_weight_checksum.py create mode 100644 vllm/model_executor/model_loader/weight_checksum.py diff --git a/docs/training/rlhf.md b/docs/training/rlhf.md index 3eddd4fbec..2aae267e1d 100644 --- a/docs/training/rlhf.md +++ b/docs/training/rlhf.md @@ -16,7 +16,7 @@ The following open-source RL libraries use vLLM for fast rollouts (sorted alphab - [Unsloth](https://github.com/unslothai/unsloth) - [verl](https://github.com/volcengine/verl) -For weight synchronization between training and inference, see the [Weight Transfer](weight_transfer/README.md) documentation, which covers the pluggable backend system with [NCCL](weight_transfer/nccl.md) (multi-GPU) and [IPC](weight_transfer/ipc.md) (same-GPU) engines. +For weight synchronization between training and inference, see the [Weight Transfer](weight_transfer/README.md) documentation, which covers the pluggable backend system with [NCCL](weight_transfer/nccl.md) (multi-GPU) and [IPC](weight_transfer/ipc.md) (same-GPU) engines. To verify that an update landed on every rank, see the [Weight Checker](weight_checker.md). For pipelining generation and training to improve GPU utilization and throughput, see the [Async Reinforcement Learning](async_rl.md) guide, which covers the pause/resume API for safely updating weights mid-flight. diff --git a/docs/training/weight_checker.md b/docs/training/weight_checker.md new file mode 100644 index 0000000000..9936001f09 --- /dev/null +++ b/docs/training/weight_checker.md @@ -0,0 +1,60 @@ +# Weight Checker + +The Weight Checker verifies that an RL weight update landed on every rank. It +hashes each model parameter with SHA-256, so a caller can save the original +digests, reset the weights, transfer them back, and confirm they match. + +Enable it with `VLLM_SERVER_DEV_MODE=1`. All operations use +`POST /weight_checker`: + +| Request | Response | +| --- | --- | +| `{"action": "checksum"}` | `{"checksums": {key: sha256_hex}}` | +| `{"action": "reset"}` | `{"status": "reset"}` | +| `{"action": "compare", "baseline": {key: sha256_hex}}` | `{"match": bool, "mismatches": [key]}` | + +Keys have the form `dp{dp}:pp{pp}:pcp{pcp}:tp{tp}:{tensor_name}`, so each +shard is checked separately. All actions cover the target model's parameters, +which is what weight loading writes; `reset` zeroes them. The endpoint keeps no +state, so the caller holds the baseline. Invalid requests return HTTP 400. + +## RL weight-update check + +```bash +URL=http://localhost:8000 + +# 1. Stop serving, then save the original digests. +curl -X POST "$URL/pause?mode=abort" +curl -X POST $URL/weight_checker -H 'Content-Type: application/json' \ + -d '{"action":"checksum"}' > baseline.json + +# 2. Zero the weights. +curl -X POST $URL/weight_checker -H 'Content-Type: application/json' \ + -d '{"action":"reset"}' + +# 3. Transfer the original weights with start/update/finish_weight_update. + +# 4. Compare against the baseline; expect {"match": true, "mismatches": []}. +curl -X POST $URL/weight_checker -H 'Content-Type: application/json' \ + -d "{\"action\":\"compare\",\"baseline\":$(jq .checksums baseline.json)}" + +curl -X POST $URL/resume +``` + +## Limitations + +- Stay paused from `checksum` to `compare`: the weights are invalid between + `reset` and the transfer, and EPLB moves experts while serving. +- A request covers the engines managed by the API server it reaches. With + several API servers (for example `--data-parallel-external-lb`), send it to + each one. +- Buffers, draft models and LoRA adapters are not checked. +- CPU backends that repack linear weights hide them from the checker. +- compressed-tensors online transforms are not supported: their Hadamard + weights are normalized once per storage, so restored copies differ. +- MoE weights padded beyond the checkpoint shape (DeepEP hidden size, MXFP4) + keep uninitialized padding, so `compare` reports them as mismatches. +- The engine must be awake: sleep levels 1 and 2 release the weight memory. +- `--offload-backend prefetch` is not supported: parameters point at staging + buffers, not the offloaded weights. +- Hashing copies every weight to CPU, so keep it off latency-sensitive paths. diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 4611d0c6d6..28f3557324 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -18,6 +18,7 @@ mod server_info; mod sleep; mod tokenize; mod version; +mod weight_checker; mod weight_transfer; mod world_size; @@ -142,6 +143,7 @@ fn build_router_with_options( post(weight_transfer::update_weight_version), ) .route("/weight_info", get(weight_transfer::weight_info)) + .route("/weight_checker", post(weight_checker::weight_checker)) .route("/abort_requests", post(abort_requests::abort_requests)) .route("/sleep", post(sleep::sleep)) .route( diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 5c1fff2cae..b016810b59 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -5813,6 +5813,82 @@ async fn collective_rpc_route_sends_expected_utility_call_and_returns_results() ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_checker_route_merges_worker_checksums_and_compares() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let workers = Value::Array(vec![ + Value::Map(vec![(Value::from("tp0:a"), Value::from("1"))]), + Value::Map(vec![(Value::from("tp1:a"), Value::from("2"))]), + ]); + for (method, result) in [ + ("compute_weight_checksums", workers.clone()), + ("compute_weight_checksums", workers), + ("reset_weights", Value::Array(vec![Value::Nil, Value::Nil])), + ] { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let array = payload.as_array().expect("utility payload array"); + assert_eq!(array[2], Value::from("collective_rpc")); + assert_eq!( + array[3].as_array().expect("rpc args")[0], + Value::from(method) + ); + let call_id = array[1].as_u64().expect("call id"); + let envelope = UtilityResultEnvelope::without_type_info(result); + send_outputs(push, utility_outputs(call_id, envelope)).await; + } + }) + }) + .await; + + let (status, body) = + post_json(&mut app, "/weight_checker", json!({"action": "checksum"})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, json!({"checksums": {"tp0:a": "1", "tp1:a": "2"}})); + + let baseline = json!({"tp0:a": "1", "tp1:a": "9", "tp2:a": "3"}); + let (status, body) = post_json( + &mut app, + "/weight_checker", + json!({"action": "compare", "baseline": baseline}), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!( + body, + json!({"match": false, "mismatches": ["tp1:a", "tp2:a"]}) + ); + + let (status, body) = post_json(&mut app, "/weight_checker", json!({"action": "reset"})).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body, json!({"status": "reset"})); + engine_task.abort_and_join().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() { + let (mut app, engine_task) = test_admin_app_with_engine_script(|dealer, _push| { + boxed_test_future(async move { + let message = recv_engine_message(dealer).await; + panic!("invalid request reached engine: {message:?}"); + }) + }) + .await; + for body in [ + json!({}), + json!({"action": "frobnicate"}), + json!({"action": "compare"}), + json!({"action": "compare", "baseline": {"k": 1}}), + ] { + let (status, _) = post_json(&mut app, "/weight_checker", body.clone()).await; + assert_eq!(status, StatusCode::BAD_REQUEST, "{body}"); + } + engine_task.abort_and_join().await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn sleep_route_uses_python_compatible_default_query_values() { diff --git a/rust/src/server/src/routes/weight_checker.rs b/rust/src/server/src/routes/weight_checker.rs new file mode 100644 index 0000000000..af8baa06ed --- /dev/null +++ b/rust/src/server/src/routes/weight_checker.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use axum::Json; +use axum::extract::State; +use axum::extract::rejection::JsonRejection; +use serde::{Deserialize, Serialize}; +use serde_json::Value as JsonValue; + +use crate::error::{ApiError, invalid_request}; +use crate::state::AppState; +use crate::utils::utility_call_error; + +#[derive(Debug, Deserialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum WeightCheckerAction { + Checksum, + Reset, + Compare, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct WeightCheckerRequest { + action: WeightCheckerAction, + #[serde(default)] + baseline: Option>, +} + +#[derive(Serialize)] +#[serde(untagged)] +pub(crate) enum WeightCheckerResponse { + Checksums { + checksums: BTreeMap, + }, + Reset { + status: &'static str, + }, + Compare { + #[serde(rename = "match")] + matched: bool, + mismatches: Vec, + }, +} + +/// Checksum, reset, or compare model weights against a baseline. +pub async fn weight_checker( + State(state): State>, + body: Result, JsonRejection>, +) -> Result, ApiError> { + let Json(body) = body?; + let client = state.engine_core_client(); + let no_args = || { + ( + Vec::::new(), + BTreeMap::::new(), + ) + }; + + if body.action == WeightCheckerAction::Reset { + let (args, kwargs) = no_args(); + client + .collective_rpc("reset_weights", None, args, kwargs) + .await + .map_err(|error| utility_call_error("reset_weights", error))?; + return Ok(Json(WeightCheckerResponse::Reset { status: "reset" })); + } + if body.action == WeightCheckerAction::Compare && body.baseline.is_none() { + return Err(invalid_request!( + param = Some("baseline"), + "action='compare' requires a 'baseline' object" + )); + } + + let (args, kwargs) = no_args(); + let workers = client + .collective_rpc("compute_weight_checksums", None, args, kwargs) + .await + .map_err(|error| utility_call_error("compute_weight_checksums", error))?; + let mut checksums = BTreeMap::new(); + for worker in workers { + let worker: BTreeMap = rmpv::ext::from_value(worker).map_err(|error| { + ApiError::server_error(format!("invalid weight checksums: {error}")) + })?; + for (key, digest) in worker { + if checksums.contains_key(&key) { + return Err(ApiError::server_error(format!( + "Duplicate weight checksum keys: {key}" + ))); + } + checksums.insert(key, digest); + } + } + + let Some(baseline) = body.baseline.filter(|_| body.action == WeightCheckerAction::Compare) + else { + return Ok(Json(WeightCheckerResponse::Checksums { checksums })); + }; + let mismatches: Vec = checksums + .keys() + .chain(baseline.keys()) + .collect::>() + .into_iter() + .filter(|key| checksums.get(*key) != baseline.get(*key)) + .cloned() + .collect(); + Ok(Json(WeightCheckerResponse::Compare { + matched: mismatches.is_empty(), + mismatches, + })) +} diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py new file mode 100644 index 0000000000..262b022ef5 --- /dev/null +++ b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os +from unittest.mock import patch + +import pytest +import regex as re +import requests +import torch +from transformers import AutoModelForCausalLM + +from tests.entrypoints.serve.dev.rlhf.conftest import ( + MODEL_NAME, + gen, + health, + ok, + server, +) +from tests.utils import multi_gpu_test +from vllm.distributed.weight_transfer import ( + HTTPVLLMWeightSyncClient, + ModuleSource, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo + + +def weight_checker(url: str, action: str, baseline=None) -> requests.Response: + body = {"action": action} + if baseline is not None: + body["baseline"] = baseline + return requests.post(f"{url}/weight_checker", json=body, timeout=300) + + +def checksums(url: str) -> dict[str, str]: + response = weight_checker(url, "checksum") + response.raise_for_status() + return response.json()["checksums"] + + +def ranks(keys) -> set[str]: + return {key.rsplit(":", 1)[0] for key in keys} + + +def reset_reload_and_compare(url: str) -> None: + baseline = checksums(url) + try: + weight_checker(url, "reset").raise_for_status() + comparison = weight_checker(url, "compare", baseline).json() + assert ranks(comparison["mismatches"]) == ranks(baseline) + finally: + requests.post( + f"{url}/collective_rpc", json={"method": "reload_weights"}, timeout=300 + ).raise_for_status() + assert weight_checker(url, "compare", baseline).json() == { + "match": True, + "mismatches": [], + } + + +@pytest.fixture(scope="class") +def server_url(): + with server() as url: + yield url + + +class TestSingleGPU: + @pytest.mark.parametrize( + "payload", + [{}, {"action": "frobnicate"}, {"action": "compare"}, [], None], + ) + def test_invalid_request_returns_400(self, server_url, payload): + response = requests.post( + f"{server_url}/weight_checker", json=payload, timeout=10 + ) + assert response.status_code == 400, response.text + assert health(server_url) == 200 + + def test_checksum_is_stable_and_compare_reports_differences(self, server_url): + baseline = checksums(server_url) + assert all( + re.fullmatch(r"[0-9a-f]{64}", digest) for digest in baseline.values() + ) + assert checksums(server_url) == baseline + + changed, missing, *_ = baseline + altered = {**baseline, changed: "0" * 64} + del altered[missing] + assert weight_checker(server_url, "compare", altered).json() == { + "match": False, + "mismatches": sorted([changed, missing]), + } + + def test_reset_changes_weights_and_reload_restores_them(self, server_url): + reset_reload_and_compare(server_url) + assert ok(gen(server_url)) + + +def test_ipc_weight_transfer_restores_reset_weights(): + """The documented RL flow: reset, then transfer the checkpoint back.""" + args = ["--weight-transfer-config", '{"backend": "ipc"}'] + with ( + patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}), + server(extra_args=args, port=8772) as url, + ): + baseline = checksums(url) + weight_checker(url, "reset").raise_for_status() + trainer_model = AutoModelForCausalLM.from_pretrained( + MODEL_NAME, torch_dtype=torch.bfloat16 + ).cuda() + WeightTransferTrainerFactory.trainer_init( + IPCTrainerInitInfo(rank=0), + client=HTTPVLLMWeightSyncClient(url), + source=ModuleSource(trainer_model), + ).send_weights() + assert weight_checker(url, "compare", baseline).json() == { + "match": True, + "mismatches": [], + } + + +@multi_gpu_test(num_gpus=2) +@pytest.mark.parametrize( + "args, expected", + [ + (["--data-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp1:pp0:pcp0:tp0"}), + (["--tensor-parallel-size", "2"], {"dp0:pp0:pcp0:tp0", "dp0:pp0:pcp0:tp1"}), + ], +) +def test_checksum_and_reset_cover_every_worker(args, expected): + with server(extra_args=args, port=8771, timeout=600) as url: + assert ranks(checksums(url)) == expected + reset_reload_and_compare(url) diff --git a/tests/model_executor/model_loader/test_weight_checksum.py b/tests/model_executor/model_loader/test_weight_checksum.py new file mode 100644 index 0000000000..e03d3b8814 --- /dev/null +++ b/tests/model_executor/model_loader/test_weight_checksum.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch +import torch.nn as nn +from torch.testing._internal.two_tensor import TwoTensor + +from vllm.config import ParallelConfig +from vllm.model_executor import parameter +from vllm.model_executor.model_loader.weight_checksum import ( + compute_tensor_digests, + zero_weights, +) +from vllm.v1.worker import gpu_worker +from vllm.v1.worker.gpu_worker import Worker + +pytestmark = pytest.mark.cpu_test + + +class _Model(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(2, 2) + nn.init.ones_(self.linear.weight) + nn.init.ones_(self.linear.bias) + # A strided view, which cannot be reinterpreted as bytes in place. + self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2]) + self.scalar = nn.Parameter(torch.tensor(2.0)) + # Loading never restores buffers, so reset must leave them alone. + self.register_buffer("k_scale", torch.tensor(2.0)) + + +def test_digests_cover_every_parameter(): + assert set(compute_tensor_digests(_Model())) == { + "linear.weight", + "linear.bias", + "strided", + "scalar", + } + + +def test_zero_weights_changes_every_parameter_and_no_buffer(): + model = _Model() + before = compute_tensor_digests(model) + zero_weights(model) + after = compute_tensor_digests(model) + assert all(before[name] != after[name] for name in before) + assert model.k_scale.item() == 2.0 + + +def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch): + monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1) + model = nn.Module() + model.transform = parameter.SharedWeightParameter(weight_loader=None) + model.transform.add_partition(0, object(), 2, 2) + model.transform.partitions[0].data.fill_(3.0) + + before = compute_tensor_digests(model) + zero_weights(model) + assert set(before) == {"transform.0"} + assert compute_tensor_digests(model) != before + + +def test_tensor_subclass_inner_tensors_are_hashed_and_reset(): + model = nn.Module() + model.weight = nn.Parameter( + TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False + ) + + before = compute_tensor_digests(model) + zero_weights(model) + assert set(before) == {"weight.a", "weight.b"} + assert all( + before[name] != digest for name, digest in compute_tensor_digests(model).items() + ) + + +def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch): + for name in ("get_tp_group", "get_pp_group", "get_pcp_group"): + monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0)) + prefixes = [] + for dp_rank in (0, 1): + worker = object.__new__(Worker) + worker.parallel_config = ParallelConfig( + data_parallel_size=2, data_parallel_rank=dp_rank + ) + # Dense DP resets data_parallel_rank to 0 in each engine. + worker.parallel_config.reconfigure_for_independent_dp_rank() + prefixes.append(worker._weight_checksum_prefix()) + assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"] diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index add424d32a..9bcc9d91eb 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -222,6 +222,24 @@ def _make_dplb_client(num_engines: int = 3, client_count: int = 1) -> DPLBAsyncM return client +def test_dplb_weight_checksums_gather_every_engine(): + """call_utility_async on a DPLB client would keep only the first engine.""" + client = _make_dplb_client(num_engines=2) + + async def call_utility(method, *args, engine): + assert (method, args) == ("compute_weight_checksums", ()) + return [{f"dp{engine[0]}:tp{tp}:w": "x"} for tp in (0, 1)] + + client._call_utility_async = call_utility + workers = asyncio.run(client.compute_weight_checksums_async()) + assert [key for worker in workers for key in worker] == [ + "dp0:tp0:w", + "dp0:tp1:w", + "dp1:tp0:w", + "dp1:tp1:w", + ] + + def test_dplb_late_interaction_sticky_routing(): client = _make_dplb_client() diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 74849a85ce..71a61b1648 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -268,6 +268,10 @@ class EngineClient(ABC): """Perform a collective RPC call to the given path.""" raise NotImplementedError + async def compute_weight_checksums(self) -> list[dict[str, str]]: + """Return rank-qualified weight digests from every worker of every engine.""" + raise NotImplementedError + async def handle_fault( self, fault_tolerance_request: FaultToleranceRequest ) -> FaultToleranceResult: diff --git a/vllm/entrypoints/serve/dev/rlhf/api_router.py b/vllm/entrypoints/serve/dev/rlhf/api_router.py index 7af2ead22c..9c464e6a11 100644 --- a/vllm/entrypoints/serve/dev/rlhf/api_router.py +++ b/vllm/entrypoints/serve/dev/rlhf/api_router.py @@ -3,7 +3,7 @@ import json from http import HTTPStatus -from typing import Annotated, Any +from typing import Annotated, Any, Literal from fastapi import APIRouter, Body, FastAPI, HTTPException, Query, Request from fastapi.responses import JSONResponse @@ -216,6 +216,43 @@ async def weight_info(raw_request: Request): return JSONResponse(content={"weight_version": weight_version}) +@router.post("/weight_checker") +async def weight_checker( + raw_request: Request, + action: Annotated[Literal["checksum", "reset", "compare"], Body(embed=True)], + baseline: Annotated[dict[str, str] | None, Body(embed=True)] = None, +) -> JSONResponse: + """Checksum, reset, or compare model weights against a baseline.""" + client = engine_client(raw_request) + if action == "reset": + await client.collective_rpc("reset_weights") + return JSONResponse(content={"status": "reset"}) + if action == "compare" and baseline is None: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST.value, + detail="action='compare' requires a 'baseline' object", + ) + + checksums: dict[str, str] = {} + for worker_checksums in await client.compute_weight_checksums(): + if duplicates := checksums.keys() & worker_checksums.keys(): + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + detail=f"Duplicate weight checksum keys: {sorted(duplicates)}", + ) + checksums.update(worker_checksums) + if action == "checksum": + return JSONResponse(content={"checksums": checksums}) + + assert baseline is not None + mismatches = sorted( + key + for key in checksums.keys() | baseline.keys() + if checksums.get(key) != baseline.get(key) + ) + return JSONResponse(content={"match": not mismatches, "mismatches": mismatches}) + + @router.get("/get_world_size") async def get_world_size( raw_request: Request, diff --git a/vllm/model_executor/model_loader/weight_checksum.py b/vllm/model_executor/model_loader/weight_checksum.py new file mode 100644 index 0000000000..40e62b0ed6 --- /dev/null +++ b/vllm/model_executor/model_loader/weight_checksum.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-parameter weight digests and reset, for verifying weight updates.""" + +import hashlib +from collections.abc import Iterator + +import torch +import torch.nn as nn +from torch.utils._python_dispatch import is_traceable_wrapper_subclass + +from vllm.model_executor.parameter import SharedWeightParameter + + +def _tensors(name: str, tensor: torch.Tensor) -> Iterator[tuple[str, torch.Tensor]]: + # Both keep their bytes in inner tensors (e.g. TorchAO's qdata and scale). + if isinstance(tensor, SharedWeightParameter): + for index, partition in tensor.partitions.items(): + yield from _tensors(f"{name}.{index}", partition) + elif is_traceable_wrapper_subclass(tensor): + for attr in tensor.__tensor_flatten__()[0]: + yield from _tensors(f"{name}.{attr}", getattr(tensor, attr)) + else: + yield name, tensor + + +def _weights(model: nn.Module) -> Iterator[tuple[str, torch.Tensor]]: + for name, param in model.named_parameters(): + yield from _tensors(name, param) + + +def compute_tensor_digests(model: nn.Module) -> dict[str, str]: + return { + name: hashlib.sha256( + weight.detach().cpu().contiguous().view(-1).view(torch.uint8).numpy() + ).hexdigest() + for name, weight in _weights(model) + } + + +def zero_weights(model: nn.Module) -> None: + for _, weight in _weights(model): + weight.data.zero_() diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 34f65f3ea2..ad9420247f 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1125,6 +1125,9 @@ class AsyncLLM(EngineClient): async def is_sleeping(self) -> bool: return await self.engine_core.is_sleeping_async() + async def compute_weight_checksums(self) -> list[dict[str, str]]: + return await self.engine_core.compute_weight_checksums_async() + async def add_lora(self, lora_request: LoRARequest) -> bool: """Load a new LoRA adapter into the engine for future requests.""" return await self.engine_core.add_lora_async(lora_request) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 5c09a0059e..cc0a8beddd 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1005,6 +1005,9 @@ class EngineCore: def execute_dummy_batch(self): self.model_executor.execute_dummy_batch() + def compute_weight_checksums(self) -> list[dict[str, str]]: + return self.collective_rpc("compute_weight_checksums") + def add_lora(self, lora_request: LoRARequest) -> bool: return self.model_executor.add_lora(lora_request) diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index b03c40f440..9372c8a172 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1362,6 +1362,10 @@ class AsyncMPClient(MPClient): "collective_rpc", method, timeout, args, kwargs ) + async def compute_weight_checksums_async(self) -> list[dict[str, str]]: + per_engine = await self.call_utility_all_async("compute_weight_checksums") + return [worker for workers in per_engine for worker in workers] + async def handle_fault( self, ft_request: FaultToleranceRequest ) -> FaultToleranceResult: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index c6da90b077..8b7a0260a7 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -58,6 +58,10 @@ from vllm.distributed.weight_transfer import ( from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.model_executor.model_loader import get_model_loader +from vllm.model_executor.model_loader.weight_checksum import ( + compute_tensor_digests, + zero_weights, +) from vllm.model_executor.warmup.kernel_warmup import kernel_warmup from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory from vllm.platforms import current_platform @@ -342,6 +346,22 @@ class Worker(WorkerBase): def checkpoint_restore(self) -> None: checkpoint_restore_distributed_state() + def _weight_checksum_prefix(self) -> str: + # data_parallel_rank is 0 in dense workers; only the index is unique. + dp = self.parallel_config.data_parallel_index + pp = get_pp_group().rank_in_group + pcp = get_pcp_group().rank_in_group + tp = get_tp_group().rank_in_group + return f"dp{dp}:pp{pp}:pcp{pcp}:tp{tp}:" + + def compute_weight_checksums(self) -> dict[str, str]: + prefix = self._weight_checksum_prefix() + digests = compute_tensor_digests(self.model_runner.get_model()) + return {prefix + name: digest for name, digest in digests.items()} + + def reset_weights(self) -> None: + zero_weights(self.model_runner.get_model()) + def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager: if ( current_platform.is_cuda_alike() From 1b77cc3d67eddd98c7ecbad7024213a3eeaf7e93 Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Tue, 29 Sep 2026 20:16:41 -0700 Subject: [PATCH 076/248] [Benchmark] Add sweep warmup and failure recovery (#57305) Signed-off-by: louie-tsai --- docs/benchmarking/sweeps.md | 6 + tests/benchmarks/sweep/test_param_sweep.py | 126 +++++++++++++++ vllm/benchmarks/sweep/serve.py | 171 +++++++++++++++++---- vllm/benchmarks/sweep/serve_workload.py | 30 +++- 4 files changed, 299 insertions(+), 34 deletions(-) diff --git a/docs/benchmarking/sweeps.md b/docs/benchmarking/sweeps.md index 41a799cf21..8975f3a2c8 100644 --- a/docs/benchmarking/sweeps.md +++ b/docs/benchmarking/sweeps.md @@ -88,6 +88,12 @@ vllm bench sweep serve \ By default, each parameter combination is benchmarked 3 times to make the results more reliable. You can adjust the number of runs by setting `--num-runs`. +You can optionally run an unmeasured warmup before each parameter combination by setting `--warmup-num-prompts`. The warmup result is saved as `warmup.json` for inspection, but is excluded from the measured summary. + +You can use `--continue-on-error` to record a failed warmup, benchmark run, or server configuration and continue processing the remaining sweep combinations. Failure details are saved in `.failure.json` files. + +Both options are also supported by `vllm bench sweep serve_workload`. + !!! important If both `--serve-params` and `--bench-params` are passed, the script will iterate over the Cartesian product between them. You can use `--dry-run` to preview the commands to be run. diff --git a/tests/benchmarks/sweep/test_param_sweep.py b/tests/benchmarks/sweep/test_param_sweep.py index 467797d991..483eb13378 100644 --- a/tests/benchmarks/sweep/test_param_sweep.py +++ b/tests/benchmarks/sweep/test_param_sweep.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from vllm.benchmarks.sweep import serve as sweep_serve from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem @@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization: # The prefix (compilation_config) gets converted to hyphens, # but the suffix (some_nested_param) is preserved assert any("compilation-config.some_nested_param" in arg for arg in cmd) + + +def test_run_comb_excludes_warmup_from_measured_results( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + calls: list[tuple[int, int, str]] = [] + + def fake_run_benchmark( + server, + bench_cmd, + *, + serve_overrides, + bench_overrides, + run_number, + output_path, + dry_run, + ): + calls.append( + (run_number, int(bench_overrides["num_prompts"]), output_path.name) + ) + return {"run_number": run_number} + + monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark) + base_path = tmp_path / "combination" + base_path.mkdir() + + measured = sweep_serve.run_comb( + None, + [], + serve_comb=ParameterSweepItem(), + bench_comb=ParameterSweepItem({"num_prompts": 320}), + link_vars=[], + base_path=base_path, + num_runs=2, + warmup_num_prompts=32, + dry_run=False, + ) + + assert calls == [ + (-1, 32, "warmup.json"), + (0, 320, "run=0.json"), + (1, 320, "run=1.json"), + ] + assert measured == [{"run_number": 0}, {"run_number": 1}] + + +def test_run_comb_continue_on_error_keeps_later_runs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + calls: list[int] = [] + + def fake_run_benchmark( + server, + bench_cmd, + *, + serve_overrides, + bench_overrides, + run_number, + output_path, + dry_run, + ): + calls.append(run_number) + if run_number == 0: + raise RuntimeError("synthetic run failure") + return {"run_number": run_number} + + monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark) + base_path = tmp_path / "combination" + base_path.mkdir() + + measured = sweep_serve.run_comb( + None, + [], + serve_comb=ParameterSweepItem(), + bench_comb=ParameterSweepItem({"num_prompts": 100}), + link_vars=[], + base_path=base_path, + num_runs=2, + warmup_num_prompts=0, + dry_run=False, + continue_on_error=True, + ) + + assert calls == [0, 1] + assert measured == [{"run_number": 1}] + assert (base_path / "run=0.failure.json").exists() + assert (base_path / "summary.json").exists() + + +def test_run_comb_warmup_default_is_backward_compatible( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + """Older callers can omit warmup_num_prompts.""" + calls: list[int] = [] + + def fake_run_benchmark( + server, + bench_cmd, + *, + serve_overrides, + bench_overrides, + run_number, + output_path, + dry_run, + ): + calls.append(run_number) + return {"run_number": run_number} + + monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark) + base_path = tmp_path / "combination" + base_path.mkdir() + + measured = sweep_serve.run_comb( + None, + [], + serve_comb=ParameterSweepItem(), + bench_comb=ParameterSweepItem({"num_prompts": 10}), + link_vars=[], + base_path=base_path, + num_runs=1, + dry_run=False, + ) + + assert calls == [0] + assert measured == [{"run_number": 0}] diff --git a/vllm/benchmarks/sweep/serve.py b/vllm/benchmarks/sweep/serve.py index 369eded5f6..9f16134720 100644 --- a/vllm/benchmarks/sweep/serve.py +++ b/vllm/benchmarks/sweep/serve.py @@ -221,29 +221,80 @@ def run_comb( base_path: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int = 0, + continue_on_error: bool = False, ): if not _comb_is_valid(serve_comb, bench_comb, link_vars): return None comb_data = list[dict[str, object]]() + def run_benchmark_with_error_handling( + *, + bench_overrides: ParameterSweepItem, + run_number: int, + output_path: Path, + ): + try: + return run_benchmark( + server, + bench_cmd, + serve_overrides=serve_comb, + bench_overrides=bench_overrides, + run_number=run_number, + output_path=output_path, + dry_run=dry_run, + ) + except Exception as exc: + if not continue_on_error: + raise + failure_path = output_path.with_name(output_path.stem + ".failure.json") + failure_path.parent.mkdir(parents=True, exist_ok=True) + failure_path.write_text( + json.dumps( + { + "run_number": run_number, + "error_type": type(exc).__name__, + "error": str(exc), + "serve_overrides": dict(serve_comb), + "bench_overrides": dict(bench_overrides), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}") + print(f"Failure details: {failure_path}") + if server is not None: + with contextlib.suppress(Exception): + server.after_bench() + return None + + if warmup_num_prompts > 0: + warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts} + run_benchmark_with_error_handling( + bench_overrides=warmup_overrides, + run_number=-1, + output_path=base_path / "warmup.json", + ) + for run_number in range(num_runs): - run_data = run_benchmark( - server, - bench_cmd, - serve_overrides=serve_comb, + run_data = run_benchmark_with_error_handling( bench_overrides=bench_comb, run_number=run_number, output_path=_get_comb_run_path(base_path, run_number), - dry_run=dry_run, ) - if run_data is not None: comb_data.append(run_data) if dry_run: return None + if not comb_data: + print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable") + return [] + with _get_comb_run_path(base_path, run_number=None).open("w") as f: json.dump(comb_data, f, indent=4) @@ -262,36 +313,69 @@ def run_combs( link_vars: list[tuple[str, str]], experiment_dir: Path, num_runs: int, + warmup_num_prompts: int, dry_run: bool, + continue_on_error: bool = False, ): all_data = list[dict[str, object]]() for serve_comb in serve_params: - with server_ctx( - serve_cmd, - after_bench_cmd, - show_stdout=show_stdout, - serve_comb=serve_comb, - bench_params=bench_params, - experiment_dir=experiment_dir, - dry_run=dry_run, - server_ready_timeout=server_ready_timeout, - ) as server: - for bench_comb in bench_params: - base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb) + try: + with server_ctx( + serve_cmd, + after_bench_cmd, + show_stdout=show_stdout, + serve_comb=serve_comb, + bench_params=bench_params, + experiment_dir=experiment_dir, + dry_run=dry_run, + server_ready_timeout=server_ready_timeout, + ) as server: + for bench_comb in bench_params: + base_path = _get_comb_base_path( + experiment_dir, serve_comb, bench_comb + ) - comb_data = run_comb( - server, - bench_cmd, - serve_comb=serve_comb, - bench_comb=bench_comb, - link_vars=link_vars, - base_path=base_path, - num_runs=num_runs, - dry_run=dry_run, + comb_data = run_comb( + server, + bench_cmd, + serve_comb=serve_comb, + bench_comb=bench_comb, + link_vars=link_vars, + base_path=base_path, + num_runs=num_runs, + warmup_num_prompts=warmup_num_prompts, + dry_run=dry_run, + continue_on_error=continue_on_error, + ) + + if comb_data is not None: + all_data.extend(comb_data) + except Exception as exc: + if not continue_on_error: + raise + failure_name = sanitize_filename( + f"SERVE-{serve_comb.name or 'default'}-failure.json" + ) + failure_path = experiment_dir / failure_name + failure_path.parent.mkdir(parents=True, exist_ok=True) + failure_path.write_text( + json.dumps( + { + "error_type": type(exc).__name__, + "error": str(exc), + "serve_overrides": dict(serve_comb), + }, + indent=2, ) - - if comb_data is not None: - all_data.extend(comb_data) + + "\n", + encoding="utf-8", + ) + print( + "[FAILED SERVER COMBINATION] " + f"{serve_comb.name or '(default)'}: " + f"{type(exc).__name__}: {exc}" + ) + print(f"Failure details: {failure_path}") if dry_run: return None @@ -315,6 +399,8 @@ class SweepServeArgs: output_dir: Path experiment_name: str num_runs: int + warmup_num_prompts: int + continue_on_error: bool dry_run: bool resume: bool @@ -351,6 +437,9 @@ class SweepServeArgs: num_runs = args.num_runs if num_runs < 1: raise ValueError("`num_runs` should be at least 1.") + warmup_num_prompts = args.warmup_num_prompts + if warmup_num_prompts < 0: + raise ValueError("`warmup_num_prompts` should be at least 0.") return cls( serve_cmd=serve_cmd, @@ -363,6 +452,8 @@ class SweepServeArgs: output_dir=Path(args.output_dir), experiment_name=experiment_name, num_runs=num_runs, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=args.continue_on_error, dry_run=args.dry_run, resume=args.resume, server_ready_timeout=args.server_ready_timeout, @@ -453,6 +544,24 @@ class SweepServeArgs: default=3, help="Number of runs per parameter combination.", ) + parser.add_argument( + "--warmup-num-prompts", + type=int, + default=0, + help=( + "Number of prompts in one unmeasured warmup run before each " + "parameter combination. The warmup is saved as warmup.json but " + "excluded from summary statistics." + ), + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + help=( + "Record and skip a failed warmup or benchmark run instead of " + "terminating the entire sweep." + ), + ) parser.add_argument( "--dry-run", action="store_true", @@ -523,7 +632,9 @@ def run_main(args: SweepServeArgs): bench_params=args.bench_params, experiment_dir=experiment_dir, num_runs=args.num_runs, + warmup_num_prompts=args.warmup_num_prompts, dry_run=args.dry_run, + continue_on_error=args.continue_on_error, ) diff --git a/vllm/benchmarks/sweep/serve_workload.py b/vllm/benchmarks/sweep/serve_workload.py index d762ca2c64..a488837e2e 100644 --- a/vllm/benchmarks/sweep/serve_workload.py +++ b/vllm/benchmarks/sweep/serve_workload.py @@ -63,6 +63,8 @@ def run_comb_workload( experiment_dir: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int, + continue_on_error: bool, workload_var: WorkloadVariable, workload_value: int, ) -> list[dict[str, object]] | None: @@ -82,6 +84,8 @@ def run_comb_workload( ), num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, ) @@ -97,6 +101,8 @@ def explore_comb_workloads( experiment_dir: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int, + continue_on_error: bool, ): print("[WL START]") print(f"Serve parameters: {serve_comb.as_text() or '(None)'}") @@ -129,6 +135,8 @@ def explore_comb_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, workload_var=workload_var, workload_value=1, ) @@ -141,16 +149,22 @@ def explore_comb_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, workload_var=workload_var, workload_value=dataset_size, ) - if serial_workload_data is None or batch_workload_data is None: + if not serial_workload_data or not batch_workload_data: if dry_run: print("Omitting intermediate Workload iterations.") - print("[WL END]") - - return + else: + print( + "Unable to establish both Workload Explorer endpoints; " + "skipping intermediate workload levels." + ) + print("[WL END]") + return [] serial_workload_value = math.ceil( _estimate_workload_avg(serial_workload_data, workload_var) @@ -181,6 +195,8 @@ def explore_comb_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, workload_var=workload_var, workload_value=inter_workload_value, ) @@ -207,6 +223,8 @@ def explore_combs_workloads( experiment_dir: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int, + continue_on_error: bool, ): if any(bench_comb.has_param(workload_var) for bench_comb in bench_params): raise ValueError( @@ -238,6 +256,8 @@ def explore_combs_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, ) if comb_data is not None: @@ -315,6 +335,8 @@ def run_main(args: SweepServeWorkloadArgs): experiment_dir=experiment_dir, num_runs=args.num_runs, dry_run=args.dry_run, + warmup_num_prompts=args.warmup_num_prompts, + continue_on_error=args.continue_on_error, ) From e006d761a526d0a69555a59f7c10c0c889babd98 Mon Sep 17 00:00:00 2001 From: Mia Ojeda Date: Tue, 29 Sep 2026 21:28:50 -0600 Subject: [PATCH 077/248] [Bugfix][CPU][DiffusionGemma] Support narrower canvas w/ sync scheduling (#59107) Signed-off-by: ojeda-e --- .../features/structured_diffusion/README.md | 7 +-- tests/test_sampling_params.py | 9 +-- tests/v1/core/test_scheduler.py | 57 ++++++++++++++++--- vllm/config/vllm.py | 8 ++- vllm/utils/diffusion.py | 10 ---- vllm/v1/core/sched/diffusion_scheduler.py | 27 ++++++--- vllm/v1/engine/input_processor.py | 1 - 7 files changed, 78 insertions(+), 41 deletions(-) diff --git a/examples/features/structured_diffusion/README.md b/examples/features/structured_diffusion/README.md index 37e58dbcdd..88b496bde8 100644 --- a/examples/features/structured_diffusion/README.md +++ b/examples/features/structured_diffusion/README.md @@ -36,10 +36,9 @@ layer when available, otherwise Triton. FlashInfer cannot serve this model (a batch mixes causal prefill with bidirectional denoising), and `--attention-backend FLASHINFER` is rejected. -Per-request canvas widths smaller than the served canvas require async -scheduling. The diffusion async scheduler is selected automatically; no -`--scheduler-cls` argument is needed. Synchronous execution supports full-width -canvases only. +Per-request canvas widths may be smaller than the served canvas with either +synchronous or asynchronous scheduling. Omit `diffusion_canvas_length` to use +the served canvas width. Question types: `noul` (yes/no), `choice` with `options`, `score` with ordered `levels`. Each label must be a single token in the answer template, diff --git a/tests/test_sampling_params.py b/tests/test_sampling_params.py index 98ef59a089..a6e057ad7a 100644 --- a/tests/test_sampling_params.py +++ b/tests/test_sampling_params.py @@ -71,7 +71,6 @@ def _verify_diffusion(params: SamplingParams, canvas_length: int | None = None): params, canvas_length=canvas_length, vocab_size=model_config.get_vocab_size(), - async_scheduling=True, ) @@ -100,7 +99,7 @@ def test_diffusion_extra_args_are_validated_without_a_served_canvas(): {"diffusion_canvas_length": 8}, ], ) -def test_narrow_diffusion_canvas_requires_async_scheduling( +def test_diffusion_canvas_width_is_accepted_with_either_scheduler( async_scheduling, extra_args ): processor = SimpleNamespace( @@ -115,11 +114,7 @@ def test_narrow_diffusion_canvas_requires_async_scheduling( validate_logits_processors_params=lambda params: None, ) params = SamplingParams(extra_args=extra_args) - if not async_scheduling and extra_args.get("diffusion_canvas_length") == 4: - with pytest.raises(VLLMValidationError, match="requires --async-scheduling"): - InputProcessor._validate_params(processor, params, ("generate",)) - else: - InputProcessor._validate_params(processor, params, ("generate",)) + InputProcessor._validate_params(processor, params, ("generate",)) @pytest.mark.parametrize( diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index d439a197e9..89a73b675c 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -42,6 +42,7 @@ from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash from vllm.v1.core.sched.diffusion_scheduler import ( DiffusionAsyncScheduler, + DiffusionScheduler, diffusion_canvas_width, ) from vllm.v1.core.sched.interface import PauseState @@ -7109,14 +7110,15 @@ def diffusion_model_runner(monkeypatch): monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") -def _diffusion_scheduler(**kwargs) -> DiffusionAsyncScheduler: +def _diffusion_scheduler(async_scheduling: bool = True, **kwargs) -> DiffusionScheduler: + scheduler_cls = DiffusionAsyncScheduler if async_scheduling else DiffusionScheduler scheduler = create_scheduler( - async_scheduling=True, + async_scheduling=async_scheduling, diffusion_canvas_length=8, - scheduler_cls=DiffusionAsyncScheduler, + scheduler_cls=scheduler_cls, **kwargs, ) - assert isinstance(scheduler, DiffusionAsyncScheduler) + assert isinstance(scheduler, scheduler_cls) return scheduler @@ -7127,7 +7129,7 @@ def test_diffusion_scheduler_is_selected_by_default(async_scheduling): async_scheduling=async_scheduling, diffusion_canvas_length=8 ).vllm_config.scheduler_config assert config.get_scheduler_cls() is ( - DiffusionAsyncScheduler if config.async_scheduling else Scheduler + DiffusionAsyncScheduler if config.async_scheduling else DiffusionScheduler ) @@ -7149,10 +7151,13 @@ def test_diffusion_scheduler_narrows_the_canvas_per_request(): @pytest.mark.parametrize("structured", [False, True]) +@pytest.mark.parametrize("async_scheduling", [False, True]) @pytest.mark.usefixtures("diffusion_model_runner") -def test_diffusion_scheduler_trims_full_width_worker_drafts(structured): +def test_diffusion_scheduler_trims_full_width_worker_drafts( + structured, async_scheduling +): """Padded worker drafts must be narrowed before scheduling or grammar validation.""" - scheduler = _diffusion_scheduler() + scheduler = _diffusion_scheduler(async_scheduling=async_scheduling) wide = _diffusion_request("wide", {}) narrow = _diffusion_request("narrow", {"diffusion_canvas_length": 4}) for request in (wide, narrow): @@ -7167,8 +7172,12 @@ def test_diffusion_scheduler_trims_full_width_worker_drafts(structured): ) tokens = list(range(8)) if structured else [-1] * 8 drafts = DraftTokenIds(["wide", "narrow"], [tokens.copy(), tokens.copy()]) - output = scheduler.schedule() - scheduler.update_draft_token_ids_in_output(drafts, output) + if async_scheduling: + output = scheduler.schedule() + scheduler.update_draft_token_ids_in_output(drafts, output) + else: + scheduler.update_draft_token_ids(drafts) + output = scheduler.schedule() assert output.scheduled_spec_decode_tokens == { "wide": tokens, @@ -7180,6 +7189,36 @@ def test_diffusion_scheduler_trims_full_width_worker_drafts(structured): assert narrow.structured_output_request.grammar.seen == [tokens[:4]] +@pytest.mark.parametrize( + "sampled, expected_status", + [ + pytest.param([], RequestStatus.RUNNING, id="denoising"), + pytest.param([1] * 4, RequestStatus.FINISHED_LENGTH_CAPPED, id="final"), + ], +) +@pytest.mark.usefixtures("diffusion_model_runner") +def test_sync_diffusion_step_commits_only_emitted_tokens(sampled, expected_status): + """An empty denoising step reuses the canvas; a completed read emits it.""" + scheduler = _diffusion_scheduler(async_scheduling=False) + request = _diffusion_request( + "read", {"diffusion_canvas_length": 4, "diffusion_read_only": True} + ) + request.max_tokens = request.sampling_params.max_tokens = 4 + scheduler.add_request(request) + _model_output(scheduler, scheduler.schedule(), [[]]) + + scheduler.update_draft_token_ids( + DraftTokenIds([request.request_id], [list(range(8))]) + ) + output = scheduler.schedule() + assert output.num_scheduled_tokens == {request.request_id: 4} + _model_output(scheduler, output, [sampled]) + + assert request.num_computed_tokens == request.num_prompt_tokens + len(sampled) + assert list(request.output_token_ids) == sampled + assert request.status == expected_status + + @pytest.mark.usefixtures("diffusion_model_runner") def test_diffusion_scheduler_defers_a_read_with_every_step_in_flight(): scheduler = _diffusion_scheduler() diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 28764e1fcc..a01023e34b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2020,11 +2020,15 @@ class VllmConfig: # After the platform hook, which has the last word on async scheduling. if ( self.diffusion_config is not None - and self.scheduler_config.async_scheduling and self.scheduler_config.scheduler_cls is None ): + scheduler_name = ( + "DiffusionAsyncScheduler" + if self.scheduler_config.async_scheduling + else "DiffusionScheduler" + ) self.scheduler_config.scheduler_cls = ( - "vllm.v1.core.sched.diffusion_scheduler.DiffusionAsyncScheduler" + f"vllm.v1.core.sched.diffusion_scheduler.{scheduler_name}" ) self._normalize_piecewise_cudagraph_mode( diff --git a/vllm/utils/diffusion.py b/vllm/utils/diffusion.py index 4980f940d8..f3018c18e7 100644 --- a/vllm/utils/diffusion.py +++ b/vllm/utils/diffusion.py @@ -15,7 +15,6 @@ def validate_diffusion_sampling_params( *, canvas_length: int | None, vocab_size: int, - async_scheduling: bool | None, ) -> None: """Validate request options and normalize one-canvas reads at admission.""" extra = params.extra_args @@ -105,12 +104,3 @@ def validate_diffusion_sampling_params( bool(params.logprob_token_ids), "diffusion_constrained needs logprob_token_ids: they are the allowed set.", ) - - check( - bool(async_scheduling) - or width is None - or canvas_length is None - or width == canvas_length, - "A diffusion_canvas_length smaller than the served canvas " - "requires --async-scheduling.", - ) diff --git a/vllm/v1/core/sched/diffusion_scheduler.py b/vllm/v1/core/sched/diffusion_scheduler.py index 96462aa6ab..9803307037 100644 --- a/vllm/v1/core/sched/diffusion_scheduler.py +++ b/vllm/v1/core/sched/diffusion_scheduler.py @@ -1,15 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Async scheduling for diffusion requests. - -These rules live here so Scheduler.schedule() and AsyncScheduler stay -unchanged. VllmConfig selects this class for a diffusion model under async -scheduling. A sync scheduler creates no output placeholders, which both rules -read. -""" +"""Canvas-width handling and async read deferral for diffusion requests.""" from vllm.v1.core.sched.async_scheduler import AsyncScheduler from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.core.sched.scheduler import Scheduler +from vllm.v1.outputs import DraftTokenIds from vllm.v1.request import Request @@ -38,7 +34,22 @@ def _read_in_flight(request: Request, width: int) -> bool: return request.num_output_placeholders >= steps * width -class DiffusionAsyncScheduler(AsyncScheduler): +class DiffusionScheduler(Scheduler): + def update_draft_token_ids(self, draft_token_ids: DraftTokenIds) -> None: + for i, (req_id, token_ids) in enumerate( + zip(draft_token_ids.req_ids, draft_token_ids.draft_token_ids) + ): + request = self.requests.get(req_id) + if request is None: + continue + width = diffusion_canvas_width(request, self.num_spec_tokens) + # Remove padded positions before the base scheduler validates grammar. + if len(token_ids) > width: + draft_token_ids.draft_token_ids[i] = token_ids[:width] + super().update_draft_token_ids(draft_token_ids) + + +class DiffusionAsyncScheduler(DiffusionScheduler, AsyncScheduler): def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: for request in self.running: width = diffusion_canvas_width(request, self.num_spec_tokens) diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 33d413ba22..3113b90f3e 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -158,7 +158,6 @@ class InputProcessor: else None ), vocab_size=self.model_config.get_vocab_size(), - async_scheduling=self.vllm_config.scheduler_config.async_scheduling, ) if self.model_config.return_sampling_mask: From 90e13fc757cffc11b55689d9cc33d844d9606516 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 30 Sep 2026 00:13:29 -0400 Subject: [PATCH 078/248] [Bugfix][HiSparse] Never allocate GPU pages without host backing (#59036) Signed-off-by: Lucas Wilkinson Signed-off-by: Matthew Bonanni Co-authored-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- tests/test_config.py | 13 ++ tests/v1/core/test_kv_cache_utils.py | 44 +++++++ tests/v1/core/test_prefix_caching.py | 124 ++++++++++++------- tests/v1/e2e/general/test_hisparse.py | 78 ++++++++++++ vllm/config/vllm.py | 8 ++ vllm/v1/core/single_type_kv_cache_manager.py | 53 +++----- vllm/v1/hisparse/coordinator.py | 9 +- vllm/v1/hisparse/layout.py | 8 ++ 8 files changed, 252 insertions(+), 85 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 39798acb05..30161e89fb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -420,6 +420,19 @@ def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch): ) +def test_hisparse_rejects_disabled_full_isl_reservation(monkeypatch): + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + with pytest.raises(ValueError, match="requires --scheduler-reserve-full-isl"): + VllmConfig( + attention_config=AttentionConfig(hisparse_config=HiSparseConfig()), + scheduler_config=SchedulerConfig( + max_model_len=2048, + is_encoder_decoder=False, + scheduler_reserve_full_isl=False, + ), + ) + + def test_hisparse_rejects_non_cuda(monkeypatch): monkeypatch.setattr(current_platform, "is_cuda", lambda: False) with pytest.raises(ValueError, match="requires NVIDIA CUDA"): diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 478707af9d..fb82d28c36 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -205,6 +205,50 @@ def test_hisparse_hma_uses_resolved_gpu_block_size( assert scheduler_block_size == hash_block_size == gpu_block_size +@pytest.mark.parametrize("extra_blocks,ok", [(1, False), (2, True)]) +def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok): + """Every page needs a host block, so one max_model_len request must fit.""" + monkeypatch.setattr( + hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True + ) + specs = { + "model.layers.0.self_attn": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + is_index_group_leader=True, + ), + "model.layers.0.self_attn.indexer": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + cache_role=SparseCacheRole.INDEXER, + ), + } + group_spec = UniformTypeKVCacheSpecs.from_specs(specs) + assert group_spec is not None + config = SimpleNamespace( + attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), + model_config=SimpleNamespace( + hf_config=SimpleNamespace(index_topk=128), max_model_len=64 * 4 + ), + parallel_config=SimpleNamespace(tensor_parallel_size=1, world_size=1), + cache_config=SimpleNamespace(num_gpu_blocks_override=7), + ) + host_page = specs["model.layers.0.self_attn"].page_size_bytes + host_budget = (4 + extra_blocks) * host_page + group = KVCacheGroupSpec(list(specs), group_spec) + + if ok: + layout = create_hisparse_layout(config, [group], host_budget=host_budget) + assert layout.host_num_blocks == 4 + extra_blocks + else: + with pytest.raises(ValueError, match="increase host_pool_gib"): + create_hisparse_layout(config, [group], host_budget=host_budget) + + def test_hisparse_rejects_deepseek_v4(): full_specs = { "model.layers.0.attn": MLAAttentionSpec( diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index c46c179260..c8d42665f0 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -782,14 +782,15 @@ def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages(): assert required == 4 -@pytest.mark.parametrize( - "full_sequence_must_fit,host_num_blocks,admitted", - [(True, 2, False), (True, 3, True), (False, 2, False), (False, 3, True)], -) -def test_hisparse_async_admission_requires_only_import_destinations( - full_sequence_must_fit, host_num_blocks, admitted, tmp_path +@pytest.mark.parametrize("host_num_blocks,admitted", [(5, False), (6, True)]) +def test_hisparse_async_admission_requires_full_prompt_host_pages( + host_num_blocks, admitted, tmp_path ): - """Imports need host destinations, but do not reserve future prefill pages.""" + """An async load is admitted only if its whole prompt's host pages fit. + + The in-flight prefill's host page is not free, so the waiting request's + four prompt pages need six host blocks including the null block. + """ from .utils import create_scheduler, mock_kv (tmp_path / "config.json").write_text( @@ -804,7 +805,7 @@ def test_hisparse_async_admission_requires_only_import_destinations( manager = make_hisparse_kv_cache_manager(32, host_num_blocks) scheduler.kv_cache_manager = manager scheduler.kv_cache_config = manager.kv_cache_config - scheduler.scheduler_reserve_full_isl = full_sequence_must_fit + assert scheduler.scheduler_reserve_full_isl inflight = make_request( "inflight", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 ) @@ -822,6 +823,52 @@ def test_hisparse_async_admission_requires_only_import_destinations( assert bool(manager.get_blocks(request.request_id).blocks[0]) == admitted +def test_hisparse_admitted_async_loads_can_finish_with_nothing_running(tmp_path): + """Admitted async loads must not strand each other on host capacity. + + Waiting requests keep their blocks and are never preempted, so two imports + that each hold a host page while together needing more than the pool has + left would never be scheduled again. HiSparse relies on full-ISL admission: + a load is admitted only if its whole prompt's host pages fit, so the newest + admitted load can always finish and free host blocks for the others. + """ + from .utils import create_scheduler, mock_kv + + (tmp_path / "config.json").write_text( + '{"architectures": ["OPTForCausalLM"], "model_type": "opt"}' + ) + scheduler = create_scheduler( + model=str(tmp_path), + skip_tokenizer_init=True, + max_model_len=128, + use_kv_connector=mock_kv(matched_tokens=HISPARSE_BLOCK_SIZE, is_async=True), + ) + # One null block plus three usable host blocks. + manager = make_hisparse_kv_cache_manager(32, 4) + scheduler.kv_cache_manager = manager + scheduler.kv_cache_config = manager.kv_cache_config + assert scheduler.scheduler_reserve_full_isl + first = make_request( + "first", list(range(3 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 + ) + second = make_request( + "second", + list(range(1000, 1000 + 3 * HISPARSE_BLOCK_SIZE)), + HISPARSE_BLOCK_SIZE, + sha256, + ) + for request in (first, second): + scheduler.add_request(request) + scheduler.schedule() + admitted = {r.request_id for r in scheduler._inflight_prefills} + + scheduler.finished_recving_kv_req_ids.update(admitted) + num_scheduled_tokens = 0 + for _ in range(4): + num_scheduled_tokens += scheduler.schedule().total_num_scheduled_tokens + assert num_scheduled_tokens > 0 + + @pytest.mark.parametrize( "evictable,local_tokens,free_blocks,admitted", [ @@ -845,58 +892,47 @@ def test_hisparse_import_capacity_includes_hits_and_cow( assert host_pool.get_num_free_blocks() == free_blocks assert ( coordinator.host_manager.get_num_blocks_to_allocate( - "waiting", 64, [hit], 32, local_tokens, 64 + "waiting", 32, [hit], 32, local_tokens, 32 ) <= host_pool.get_num_free_blocks() ) == admitted @pytest.mark.parametrize("enable_caching", [False, True]) -def test_hisparse_host_exhaustion_keeps_gpu_pages_readable(enable_caching): - """Pages without a host destination must survive GPU cache reclamation.""" - manager = make_hisparse_kv_cache_manager(32, 2, enable_caching=enable_caching) +def test_hisparse_host_exhaustion_defers_allocation(enable_caching): + """Pages are only allocated with host backing, so every one can be reclaimed.""" + manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching) + donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256) + assert manager.allocate_slots(donor, 16) is not None request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, 64) is not None + free_gpu = manager.block_pool.get_num_free_blocks() + assert manager.allocate_slots(request, 64) is None + assert manager.block_pool.get_num_free_blocks() == free_gpu + + # The donor's in-flight write-back holds its host block until acked. + manager.free(donor) coordinator = get_hisparse_coordinator(manager) + assert coordinator.has_pending_reclamation() + assert manager.allocate_slots(request, 64) is None + _publish_hisparse_pages(manager) + assert not coordinator.has_pending_reclamation() + assert manager.allocate_slots(request, 64) is not None host = coordinator.host_manager assert host is not None - assert [b.is_null for b in host.req_to_blocks[request.request_id]] == [ - False, - True, - True, - True, - ] + assert not any(b.is_null for b in host.req_to_blocks[request.request_id]) _publish_hisparse_pages(manager) for hot in coordinator.hot_managers: hot.require_hot(request.request_id) hot.allocate_new_blocks(request.request_id, 64, 64) coordinator.update_residency(request.request_id) + assert coordinator.request_states[request.request_id].unpinned_pages == {0, 1} for resident in coordinator.resident_managers: - for block in resident.req_to_blocks[request.request_id][1:]: - assert not block.is_null and block.ref_cnt > 0 - assert coordinator.request_states[request.request_id].valid_pages == {0} - pool = coordinator.get_host_block_pool() - assert pool is not None - manager.free(request) - assert pool.get_num_free_blocks() == 1 - - -def test_hisparse_host_cow_takes_priority_over_new_pages(): - """The last host block must hold the private tail, not a fresh page.""" - manager = make_hisparse_kv_cache_manager(32, 3) - coordinator = get_hisparse_coordinator(manager) - host = coordinator.host_manager - assert host is not None - source = host.block_pool.get_new_blocks(1)[0] - host.req_to_blocks["cow"] = [source] - host._partial_hit_reqs["cow"] = (0, source) - - host.allocate_new_blocks("cow", 32, 32) - - private, missing = host.req_to_blocks["cow"] - assert not private.is_null and private is not source - assert missing.is_null - assert host.take_host_cow_copies() == [(source, private)] + assert [b.ref_cnt for b in resident.req_to_blocks[request.request_id]] == [ + 0, + 0, + 1, + 1, + ] @pytest.mark.parametrize("failed", [False, True]) diff --git a/tests/v1/e2e/general/test_hisparse.py b/tests/v1/e2e/general/test_hisparse.py index dd4c588657..21f28d0b6b 100644 --- a/tests/v1/e2e/general/test_hisparse.py +++ b/tests/v1/e2e/general/test_hisparse.py @@ -223,3 +223,81 @@ def test_hisparse_spill_and_prefix_restore( assert actual == expected if with_offloading: assert _offload_load_bytes() > load_bytes + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA" +) +@fork_new_process_for_each_test +def test_hisparse_host_exhaustion_defers_requests( + monkeypatch: pytest.MonkeyPatch, + vllm_runner: type[VllmRunner], +): + """A full host pool defers requests instead of leaving pages GPU-only.""" + capability = current_platform.get_device_capability() + if capability is None or capability.major < 9: + pytest.skip("Sparse MLA requires Hopper or newer") + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip") + prompts = [ + [1000 + request_idx * 64 + i % 64 for i in range(257)] + for request_idx in range(3) + ] + with vllm_runner( + MODEL, + load_format="dummy", + hf_overrides=_shrink_config, + attention_config=AttentionConfig( + hisparse_config=HiSparseConfig(device_buffer_size=512) + ), + kv_transfer_config=KVTransferConfig( + kv_connector="HiSparseConnector", + kv_role="kv_both", + kv_connector_extra_config={"host_pool_gib": 1}, + ), + block_size=64, + max_model_len=320, + max_num_batched_tokens=512, + max_num_seqs=3, + num_gpu_blocks_override=128, + enable_prefix_caching=False, + enable_chunked_prefill=True, + enforce_eager=True, + ) as runner: + expected = [runner.generate_greedy([p], max_tokens=16)[0] for p in prompts] + engine = runner.llm.llm_engine.engine_core.engine_core + coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager) + host = coordinator.host_manager + assert host is not None + # Leave room for one full request plus part of another. + spare_blocks = 8 + num_pressure = host.block_pool.get_num_free_blocks() - spare_blocks + host.allocate_new_blocks( + "host-pressure", + num_pressure * host.block_size, + num_pressure * host.block_size, + ) + original_plan = coordinator.plan_prefix_materialization + original_count = host.get_num_blocks_to_allocate + checked_steps = 0 + refusals = 0 + + def plan(request_id, num_computed_tokens): + nonlocal checked_steps + assert not any(b.is_null for b in host.req_to_blocks[request_id]) + checked_steps += 1 + original_plan(request_id, num_computed_tokens) + + def count(*args, **kwargs): + nonlocal refusals + required = original_count(*args, **kwargs) + refusals += int(required > 0) + return required + + monkeypatch.setattr(coordinator, "plan_prefix_materialization", plan) + monkeypatch.setattr(host, "get_num_blocks_to_allocate", count) + actual = runner.generate_greedy(prompts, max_tokens=16) + + assert checked_steps > 0 + assert refusals > 0 + assert actual == expected diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index a01023e34b..824c49efa0 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1822,6 +1822,14 @@ class VllmConfig: raise ValueError( "HiSparse does not support decode context parallelism." ) + if not self.scheduler_config.scheduler_reserve_full_isl: + # Without it, async loads admitted against free host blocks can + # each wait on host pages the others hold, and waiting requests + # are never preempted to free them. + raise ValueError( + "HiSparse requires --scheduler-reserve-full-isl; remove " + "--no-scheduler-reserve-full-isl." + ) if self.model_config is not None and not hasattr( self.model_config.hf_config, "index_topk" ): diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 707786f17f..31e610b578 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2223,8 +2223,8 @@ class SinkFullAttentionManager(FullAttentionManager): class HiSparseSourceManager(FullAttentionManager): """Host-tier manager with a private pool; publishes hashes once durable. - Host capacity is best effort: a page that cannot get a host block keeps a - null host entry, so its GPU copy is never written back and stays pinned. + Every page gets a host block: allocation fails when the host pool cannot + back it, so each GPU-resident page can be written back and later reclaimed. """ coordinator: "HiSparseCoordinator | None" = None @@ -2273,44 +2273,21 @@ class HiSparseSourceManager(FullAttentionManager): num_tokens_main_model: int, apply_admission_cap: bool = False, ) -> int: - if ( - total_computed_tokens > num_local_computed_tokens - or self._has_partial_local_hit( - new_computed_blocks, num_local_computed_tokens - ) - ): - # External loads need real destinations; future GPU-computed pages - # remain best effort. Use the same admission sentinel as Mamba. - required = super().get_num_blocks_to_allocate( - request_id, - total_computed_tokens, - new_computed_blocks, - total_computed_tokens, - num_local_computed_tokens, - total_computed_tokens, - ) - if required > self.block_pool.get_num_free_blocks(): - assert self.coordinator is not None - assert self.coordinator.gpu_pool is not None - return self.coordinator.gpu_pool.num_gpu_blocks + 1 - return 0 - - def allocate_new_blocks( - self, request_id: str, num_tokens: int, num_tokens_main_model: int - ) -> list[KVCacheBlock]: - req_blocks = self.req_to_blocks[request_id] - num_required = cdiv(num_tokens, self.block_size) - num_free = self.block_pool.get_num_free_blocks() - if request_id in self._partial_hit_reqs: - num_free -= 1 - fit_blocks = min(num_required, len(req_blocks) + max(0, num_free)) - new_blocks = super().allocate_new_blocks( + required = super().get_num_blocks_to_allocate( request_id, - fit_blocks * self.block_size, - min(num_tokens_main_model, fit_blocks * self.block_size), + num_tokens, + new_computed_blocks, + total_computed_tokens, + num_local_computed_tokens, + num_tokens_main_model, ) - req_blocks.extend([self._null_block] * (num_required - len(req_blocks))) - return new_blocks + # Host blocks come from a private pool, so they never count against + # the GPU pool. Use the same admission sentinel as Mamba instead. + if required > self.block_pool.get_num_free_blocks(): + assert self.coordinator is not None + assert self.coordinator.gpu_pool is not None + return self.coordinator.gpu_pool.num_gpu_blocks + 1 + return 0 def allocate_external_computed_blocks( self, diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index 056e19479a..0ca0d40c88 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -660,10 +660,13 @@ class HiSparseCoordinator: return command def has_pending_reclamation(self) -> bool: - """Whether an in-flight spill will free GPU blocks on completion.""" + """Whether an in-flight spill will free GPU or host blocks on completion.""" return any( - pending.page[0] in self.request_states - and self._can_read_from_host(pending.page[0]) + pending.host_block.ref_cnt == 1 + or ( + pending.page[0] in self.request_states + and self._can_read_from_host(pending.page[0]) + ) for pending in self.pending_spills.values() ) diff --git a/vllm/v1/hisparse/layout.py b/vllm/v1/hisparse/layout.py index b3b45d3334..e177768fc9 100644 --- a/vllm/v1/hisparse/layout.py +++ b/vllm/v1/hisparse/layout.py @@ -235,6 +235,14 @@ def create_hisparse_layout( host_num_blocks = host_budget // host_block_stride if host_num_blocks <= 0: raise ValueError("HiSparse has no allocatable host blocks.") + # Every computed page needs a host block, so one request at max_model_len + # must fit alongside the pool's null block and a copy-on-write tail. + min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2 + if host_num_blocks < min_host_blocks: + raise ValueError( + f"HiSparse host pool has {host_num_blocks} blocks but max_model_len " + f"needs {min_host_blocks}; increase host_pool_gib." + ) return HiSparseLayout( source_group=source_group, From 834ad456d314f801696b259e426c6946e50ee42d Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 30 Sep 2026 00:39:03 -0400 Subject: [PATCH 079/248] [Refactor] Remove dead tests code (#58916) Signed-off-by: yewentao256 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/test-amd.yaml | 2 +- .buildkite/test_areas/models_distributed.yaml | 6 +- tests/compile/passes/test_fusion.py | 4 - tests/compile/passes/test_fusion_attn.py | 1 - tests/conftest.py | 5 - tests/entrypoints/conftest.py | 120 ----------- .../launchers/test_dp_supervisor.py | 1 - .../responses/test_serving_responses.py | 8 - .../tool_parsers/test_openai_tool_parser.py | 3 - tests/kernels/attention/test_attention.py | 1 - tests/kernels/moe/utils.py | 92 -------- tests/lora/conftest.py | 18 -- .../multimodal/generation/test_phi4siglip.py | 198 ------------------ .../test_kimi_k2_reasoning_parser.py | 15 -- tests/tracing/conftest.py | 6 - tests/v1/engine/conftest.py | 4 - tests/v1/engine/test_llm_engine.py | 7 - 17 files changed, 3 insertions(+), 488 deletions(-) delete mode 100644 tests/models/multimodal/generation/test_phi4siglip.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 2e02d1b404..61f33d74ee 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -586,7 +586,7 @@ steps: - tests/utils.py - tests/models/test_vision.py commands: - - case "$$BUILDKITE_PARALLEL_JOB" in 0) TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' && pytest models/language -v -s -m 'distributed(num_gpus=2)' && VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' && pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; esac + - case "$$BUILDKITE_PARALLEL_JOB" in 0) TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' && HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' ;; 1) pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' && pytest models/language -v -s -m 'distributed(num_gpus=2)' && VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' && pytest -v -s models/test_vision.py -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; esac #---------------------------------------------------------- mi300 · compile ----------------------------------------------------------# diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index a054c00db8..9ab75cfc6e 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -26,8 +26,7 @@ steps: pytest models/language -v -s -m 'distributed(num_gpus=2)' ;; 2) - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; @@ -72,8 +71,7 @@ steps: VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'distributed(num_gpus=2)' ;; 2) - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py && - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' && + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' ;; *) exit 2 ;; diff --git a/tests/compile/passes/test_fusion.py b/tests/compile/passes/test_fusion.py index bb15022459..934522a1b9 100644 --- a/tests/compile/passes/test_fusion.py +++ b/tests/compile/passes/test_fusion.py @@ -60,10 +60,6 @@ from vllm.utils.deep_gemm import ( is_deep_gemm_supported, ) -FP8_DTYPE = current_platform.fp8_dtype() - -RMS_ADD_OP = torch.ops._C.fused_add_rms_norm.default - # Kernel and group_shape combinations: (kernel, group_shape) # CUDA kernels CUDA_KERNEL_GROUPSHAPE_COMBINATIONS = [ diff --git a/tests/compile/passes/test_fusion_attn.py b/tests/compile/passes/test_fusion_attn.py index 759970f14d..6c3ca6b832 100644 --- a/tests/compile/passes/test_fusion_attn.py +++ b/tests/compile/passes/test_fusion_attn.py @@ -248,7 +248,6 @@ class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel): PATTERN_TEST_MODELS_FP8: list[tuple[str, type]] = [] PATTERN_TEST_MODELS_FP4: list[tuple[str, type]] = [] HEADS: list[tuple[int, int]] = [] -SPLIT_ATTENTION: list[bool] = [] BACKENDS_FP8: list[AttentionBackendEnum] = [] BACKENDS_FP4: list[AttentionBackendEnum] = [] diff --git a/tests/conftest.py b/tests/conftest.py index 37bfee1f8b..c1380f61cd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -317,11 +317,6 @@ def example_system_message() -> str: return f.read() -@pytest.fixture -def example_long_prompts() -> list[str]: - return [prompt for filename in _LONG_PROMPTS for prompt in _read_prompts(filename)] - - @pytest.fixture(scope="session") def image_assets() -> ImageTestAssets: return IMAGE_ASSETS diff --git a/tests/entrypoints/conftest.py b/tests/entrypoints/conftest.py index 27383f87ea..097d56cc9d 100644 --- a/tests/entrypoints/conftest.py +++ b/tests/entrypoints/conftest.py @@ -14,16 +14,6 @@ def sample_prompts(): ] -@pytest.fixture -def sample_token_ids(): - return [ - [0], - [0, 1], - [0, 2, 1], - [0, 3, 1, 2], - ] - - @pytest.fixture def sample_regex(): return ( @@ -61,104 +51,6 @@ def sample_json_schema(): } -@pytest.fixture -def sample_complex_json_schema(): - return { - "type": "object", - "properties": { - "score": { - "type": "integer", - "minimum": 0, - "maximum": 100, # Numeric range - }, - "grade": { - "type": "string", - "pattern": "^[A-D]$", # Regex pattern - }, - "email": { - "type": "string", - "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", - }, - "tags": { - "type": "array", - "items": { - "type": "string", - # Combining length and pattern restrictions - "pattern": "^[a-z]{1,10}$", - }, - }, - }, - "required": ["score", "grade", "email", "tags"], - } - - -@pytest.fixture -def sample_definition_json_schema(): - return { - "$defs": { - "Step": { - "properties": { - "explanation": {"title": "Explanation", "type": "string"}, - "output": {"title": "Output", "type": "string"}, - }, - "required": ["explanation", "output"], - "title": "Step", - "type": "object", - } - }, - "properties": { - "steps": { - "items": {"$ref": "#/$defs/Step"}, - "title": "Steps", - "type": "array", - }, - "final_answer": {"title": "Final Answer", "type": "string"}, - }, - "required": ["steps", "final_answer"], - "title": "MathReasoning", - "type": "object", - } - - -@pytest.fixture -def sample_enum_json_schema(): - return { - "type": "object", - "properties": { - "status": { - "type": "string", - "enum": ["active", "inactive", "pending"], # Literal values using enum - }, - "priority": { - "type": "string", - "enum": ["low", "medium", "high", "critical"], - }, - "category": { - "type": "object", - "properties": { - "type": { - "type": "string", - "enum": ["bug", "feature", "improvement"], - }, - "severity": { - "type": "integer", - "enum": [1, 2, 3, 4, 5], # Enum can also contain numbers - }, - }, - "required": ["type", "severity"], - }, - "flags": { - "type": "array", - "items": { - "type": "string", - "enum": ["urgent", "blocked", "needs_review", "approved"], - }, - }, - }, - "required": ["status", "priority", "category", "flags"], - } - - @pytest.fixture def sample_structured_outputs_choices(): return [ @@ -175,18 +67,6 @@ def sample_structured_outputs_choices(): ] -@pytest.fixture -def sample_sql_statements(): - return """ -start: select_statement -select_statement: "SELECT" column "from" table "where" condition -column: "col_1" | "col_2" -table: "table_1" | "table_2" -condition: column "=" number -number: "1" | "2" -""" - - @pytest.fixture(scope="session") def qwen3_lora_files(): """Download Qwen3 LoRA files once per test session.""" diff --git a/tests/entrypoints/launchers/test_dp_supervisor.py b/tests/entrypoints/launchers/test_dp_supervisor.py index f96b020d10..c10f7ee698 100644 --- a/tests/entrypoints/launchers/test_dp_supervisor.py +++ b/tests/entrypoints/launchers/test_dp_supervisor.py @@ -47,7 +47,6 @@ _SUPERVISOR_PORT = 19256 _CHILD_PORT_BASE = 18000 _N_CHILDREN = 2 _PROBE_INTERVAL = 1.0 -_POLL_INTERVAL = 1.0 # --------------------------------------------------------------------------- diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index 33e5eaa323..d07a89ec9c 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -127,14 +127,6 @@ def test_serialize_message_pydantic_model_returns_dict() -> None: assert serialized["message"] == "hello" -@pytest.fixture -def mock_serving_responses(): - """Create a mock OpenAIServingResponses instance.""" - serving_responses = MagicMock(spec=OpenAIServingResponses) - serving_responses.tool_server = MagicMock(spec=ToolServer) - return serving_responses - - @pytest.fixture def mock_context(): """Create a mock conversation context.""" diff --git a/tests/entrypoints/tool_parsers/test_openai_tool_parser.py b/tests/entrypoints/tool_parsers/test_openai_tool_parser.py index de402b435c..f840404111 100644 --- a/tests/entrypoints/tool_parsers/test_openai_tool_parser.py +++ b/tests/entrypoints/tool_parsers/test_openai_tool_parser.py @@ -117,10 +117,7 @@ MESSAGES_INVALID_CALL = [ # Expected outputs FUNC_CALC = "calculator" -FUNC_ARGS_CALC = '{"expression":"123 + 456"}' - FUNC_TIME = "get_time" -FUNC_ARGS_TIME = '{"city": "New York"}' # ========================================================== diff --git a/tests/kernels/attention/test_attention.py b/tests/kernels/attention/test_attention.py index e78ede88f0..e3c3ac0b24 100644 --- a/tests/kernels/attention/test_attention.py +++ b/tests/kernels/attention/test_attention.py @@ -26,7 +26,6 @@ NUM_BLOCKS = 4321 # Arbitrary values for testing PARTITION_SIZE_ROCM = 256 DTYPES = [torch.bfloat16, torch.float16] NUM_GEN_SEQS = [7] # Arbitrary values for testing -NUM_PREFILL_SEQS = [3] # Arbitrary values for testing NUM_HEADS = [(32, 8), (40, 40), (64, 8)] # Arbitrary values for testing # Head sizes supported by the ROCm paged attention kernel. diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index 71b28a9375..1536c4cec9 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -7,7 +7,6 @@ import torch import vllm._custom_ops as ops from tests.kernels.quant_utils import per_block_cast_to_int8 from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX -from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.all2all_utils import ( maybe_make_prepare_finalize, @@ -552,97 +551,6 @@ class TestMLP(torch.nn.Module): return x -class RealMLP(torch.nn.Module): - def __init__( - self, - hidden_size: int, - intermediate_size: int, - w1: torch.Tensor, - w2: torch.Tensor, - hidden_act: str = "silu", - quant_config=None, - reduce_results: bool = True, - prefix: str = "", - w1_s: torch.Tensor | None = None, - w2_s: torch.Tensor | None = None, - ) -> None: - from vllm.model_executor.layers.linear import ( - MergedColumnParallelLinear, - RowParallelLinear, - ) - - super().__init__() - self.gate_up_proj = MergedColumnParallelLinear( - hidden_size, - [intermediate_size] * 2, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.gate_up_proj", - ) - self.gate_up_proj.register_parameter( - "weight", torch.nn.Parameter(w1, requires_grad=False) - ) - self.gate_up_proj.register_parameter( - "weight_scale", torch.nn.Parameter(w1_s, requires_grad=False) - ) - self.gate_up_proj.register_parameter( - "input_scale", None - ) # torch.nn.Parameter(None, requires_grad=False)) - self.down_proj = RowParallelLinear( - intermediate_size, - hidden_size, - bias=False, - quant_config=quant_config, - reduce_results=reduce_results, - prefix=f"{prefix}.down_proj", - ) - self.down_proj.register_parameter( - "weight", torch.nn.Parameter(w2, requires_grad=False) - ) - self.down_proj.register_parameter( - "weight_scale", torch.nn.Parameter(w2_s, requires_grad=False) - ) - self.down_proj.register_parameter( - "input_scale", None - ) # torch.nn.Parameter(None, requires_grad=False)) - if hidden_act != "silu": - raise ValueError( - f"Unsupported activation: {hidden_act}. Only silu is supported for now." - ) - self.act_fn = SiluAndMul() - - def forward(self, x): - gate_up, _ = self.gate_up_proj(x) - x = self.act_fn(gate_up) - x, _ = self.down_proj(x) - return x - - -def make_shared_experts_with_weights( - N: int, - K: int, - in_dtype: torch.dtype, - w1: torch.Tensor, - w2: torch.Tensor, - w1_s: torch.Tensor | None = None, - w2_s: torch.Tensor | None = None, - quant_dtype: torch.dtype | str | None = None, -) -> torch.nn.Module: - old_dtype = torch.get_default_dtype() - try: - torch.set_default_dtype(in_dtype) - if quant_dtype == torch.float8_e4m3fn: - from vllm.model_executor.layers.quantization.fp8 import Fp8Config - - quant_config = Fp8Config() - else: - quant_config = None - - return RealMLP(K, N, w1, w2, "silu", quant_config, w1_s=w1_s, w2_s=w2_s) - finally: - torch.set_default_dtype(old_dtype) - - def modular_triton_fused_moe( moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig, diff --git a/tests/lora/conftest.py b/tests/lora/conftest.py index f5ca44f174..59690401e5 100644 --- a/tests/lora/conftest.py +++ b/tests/lora/conftest.py @@ -76,18 +76,6 @@ def dist_init(): cleanup_dist_env_and_memory(shutdown_ray=True) -@pytest.fixture -def dist_init_torch_only(): - if torch.distributed.is_initialized(): - return - backend = current_platform.dist_backend - - temp_file = tempfile.mkstemp()[1] - torch.distributed.init_process_group( - world_size=1, rank=0, init_method=f"file://{temp_file}", backend=backend - ) - - class DummyLoRAModel(nn.Sequential, SupportsLoRA): pass @@ -208,12 +196,6 @@ def qwen2vl_lora_files(): return hf_api().snapshot_download(repo_id="jeeejeee/qwen2-vl-lora-pokemon") -@pytest.fixture(scope="session") -def qwen25vl_base_huggingface_id(): - # used as a base model for testing with qwen25vl lora adapter - return "Qwen/Qwen2.5-VL-3B-Instruct" - - @pytest.fixture(scope="session") def qwen25vl_lora_files(): return hf_api().snapshot_download(repo_id="jeeejeee/qwen25-vl-lora-pokemon") diff --git a/tests/models/multimodal/generation/test_phi4siglip.py b/tests/models/multimodal/generation/test_phi4siglip.py deleted file mode 100644 index f80b16c341..0000000000 --- a/tests/models/multimodal/generation/test_phi4siglip.py +++ /dev/null @@ -1,198 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence -from importlib.metadata import version - -import pytest -import regex as re -from packaging.version import Version -from transformers import AutoModelForCausalLM, AutoTokenizer - -from vllm.logprobs import SampleLogprobs -from vllm.multimodal.image import rescale_image_size - -from ....conftest import ( - IMAGE_ASSETS, - HfRunner, - PromptImageInput, - VllmRunner, -) -from ....utils import multi_gpu_test -from ...utils import check_logprobs_close - -pytestmark = pytest.mark.skipif( - Version("5.0") <= Version(version("transformers")), - reason=( - "vllm upgraded transformers above v5.4 where HF model custom code uses siglip2 " - "internals (filter_out_non_signature_kwargs) removed by " - "huggingface/transformers#43514" - ), -) - -MODEL_ID = "microsoft/Phi-4-reasoning-vision-15B" - -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts( - { - "stop_sign": "<|user|>\n\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501 - "cherry_blossom": "<|user|>\n\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501 - } -) -HF_MULTIIMAGE_IMAGE_PROMPT = ( - "<|user|>\n\n\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501 -) - -DTYPE = "half" -MAX_TOKENS = 128 -NUM_LOGPROBS = 10 - - -def vllm_to_hf_output( - vllm_output: tuple[list[int], str, SampleLogprobs | None], model: str -): - """Sanitize vllm output to be comparable with hf output.""" - _, output_str, out_logprobs = vllm_output - - output_str_without_image = re.sub(r"()+", "", output_str) - if output_str_without_image and output_str_without_image[0] == " ": - output_str_without_image = output_str_without_image[1:] - - hf_output_str = output_str_without_image + "<|end|><|endoftext|>" - - tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True) - hf_output_ids = tokenizer.encode(output_str_without_image) - if hf_output_ids and hf_output_ids[0] == tokenizer.bos_token_id: - hf_output_ids = hf_output_ids[1:] - - return hf_output_ids, hf_output_str, out_logprobs - - -def _build_single_image_inputs( - image_assets, -) -> list[tuple[list[str], PromptImageInput]]: - """Build single-image inputs for all size_factors at once.""" - images = [asset.pil_image for asset in image_assets] - all_inputs: list[tuple[list[str], PromptImageInput]] = [] - for size_factors in [[1.0], [0.25, 0.5, 1.0]]: - for image, prompt in zip(images, HF_IMAGE_PROMPTS): - all_inputs.append( - ( - [prompt for _ in size_factors], - [rescale_image_size(image, f) for f in size_factors], - ) - ) - return all_inputs - - -def _build_multi_image_inputs( - image_assets, -) -> list[tuple[list[str], PromptImageInput]]: - """Build multi-image inputs for all size_factors at once.""" - images = [asset.pil_image for asset in image_assets] - all_inputs: list[tuple[list[str], PromptImageInput]] = [] - for size_factors in [[0.5], [0.15, 0.30]]: - all_inputs.append( - ( - [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors], - [ - [rescale_image_size(image, factor) for image in images] - for factor in size_factors - ], - ) - ) - return all_inputs - - -def _run_and_compare( - hf_runner: type[HfRunner], - vllm_runner: type[VllmRunner], - all_inputs: Sequence[tuple[list[str], PromptImageInput]], - model: str, - max_model_len: int, - max_num_seqs: int, - mm_limit: int, - gpu_memory_utilization: float, -): - """Load each runner once, run all inputs, then compare.""" - # NOTE: run vLLM first, then HF. vLLM needs a fresh process without - # cuda initialization; running HF first would break the multiprocessing - # backend with fork method. - with vllm_runner( - model, - runner="generate", - max_model_len=max_model_len, - max_num_seqs=max_num_seqs, - gpu_memory_utilization=gpu_memory_utilization, - dtype=DTYPE, - limit_mm_per_prompt={"image": mm_limit}, - tensor_parallel_size=2, - trust_remote_code=True, - enforce_eager=True, - ) as vllm_model: - vllm_outputs_per_case = [ - vllm_model.generate_greedy_logprobs( - prompts, - MAX_TOKENS, - num_logprobs=NUM_LOGPROBS, - images=images, - ) - for prompts, images in all_inputs - ] - - hf_model_kwargs = {"_attn_implementation": "sdpa", "device_map": "auto"} - with hf_runner( - model, - dtype=DTYPE, - model_kwargs=hf_model_kwargs, - auto_cls=AutoModelForCausalLM, - trust_remote_code=True, - ) as hf_model: - hf_outputs_per_case = [ - hf_model.generate_greedy_logprobs_limit( - prompts, - MAX_TOKENS, - num_logprobs=NUM_LOGPROBS, - images=images, - ) - for prompts, images in all_inputs - ] - - for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case): - check_logprobs_close( - outputs_0_lst=hf_outputs, - outputs_1_lst=vllm_outputs, - name_0="hf", - name_1="vllm", - ) - - -@multi_gpu_test(num_gpus=2) -@pytest.mark.parametrize("model", [MODEL_ID]) -def test_models(hf_runner, vllm_runner, image_assets, model) -> None: - all_inputs = _build_single_image_inputs(image_assets) - _run_and_compare( - hf_runner, - vllm_runner, - all_inputs, - model, - max_model_len=8192, - max_num_seqs=2, - mm_limit=1, - gpu_memory_utilization=0.80, - ) - - -@multi_gpu_test(num_gpus=2) -@pytest.mark.parametrize("model", [MODEL_ID]) -def test_multi_images_models(hf_runner, vllm_runner, image_assets, model) -> None: - all_inputs = _build_multi_image_inputs(image_assets) - _run_and_compare( - hf_runner, - vllm_runner, - all_inputs, - model, - max_model_len=8192, - max_num_seqs=2, - mm_limit=2, - gpu_memory_utilization=0.80, - ) diff --git a/tests/reasoning/test_kimi_k2_reasoning_parser.py b/tests/reasoning/test_kimi_k2_reasoning_parser.py index dc0947fdb2..e9690b2faa 100644 --- a/tests/reasoning/test_kimi_k2_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k2_reasoning_parser.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from unittest.mock import MagicMock import pytest @@ -13,20 +12,6 @@ from vllm.tokenizers import get_tokenizer REASONING_MODEL_NAME = "moonshotai/Kimi-K2.5" -@pytest.fixture -def mock_kimi_k2_tokenizer(): - tokenizer = MagicMock() - tokenizer.get_vocab.return_value = { - "": 100, - "": 101, - "<|tool_calls_section_begin|>": 200, - "<|tool_calls_section_end|>": 201, - "<|tool_call_begin|>": 202, - "<|tool_call_end|>": 203, - } - return tokenizer - - @pytest.fixture(scope="module") def kimi_k2_tokenizer(): return get_tokenizer(tokenizer_name=REASONING_MODEL_NAME, trust_remote_code=True) diff --git a/tests/tracing/conftest.py b/tests/tracing/conftest.py index 635d4fd257..97b2afa0c8 100644 --- a/tests/tracing/conftest.py +++ b/tests/tracing/conftest.py @@ -142,9 +142,3 @@ def trace_service() -> Generator[FakeTraceService, None, None]: yield service server.stop(grace=None) - - -@pytest.fixture -def trace_server_address() -> str: - """Returns the address of the fake trace server.""" - return FAKE_TRACE_SERVER_ADDRESS diff --git a/tests/v1/engine/conftest.py b/tests/v1/engine/conftest.py index 1cfe671a62..e382b9fdca 100644 --- a/tests/v1/engine/conftest.py +++ b/tests/v1/engine/conftest.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -import torch from transformers import AutoTokenizer from tests.v1.engine.utils import ( @@ -19,9 +18,6 @@ from vllm.engine.arg_utils import EngineArgs from ...distributed.conftest import publisher_config, random_port # noqa: F401 -EngineCoreSampleLogprobsType = list[tuple[torch.Tensor, torch.Tensor]] -EngineCorePromptLogprobsType = tuple[torch.Tensor, torch.Tensor] - def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors: """Generate output processor dummy test vectors, without logprobs. diff --git a/tests/v1/engine/test_llm_engine.py b/tests/v1/engine/test_llm_engine.py index e0e6004b0e..c9832f7d8e 100644 --- a/tests/v1/engine/test_llm_engine.py +++ b/tests/v1/engine/test_llm_engine.py @@ -49,13 +49,6 @@ def vllm_model(vllm_runner, request): yield vllm_model -@pytest.fixture(scope="function") -def vllm_model_apc(vllm_runner): - """VllmRunner test fixture with APC.""" - with _vllm_model(True, vllm_runner) as vllm_model: - yield vllm_model - - @pytest.fixture( # Function scope decouples tests & allows # env var adjustment via monkeypatch From a3af78384a1f14f4769f8fa2c64c7ced5216b2ae Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Wed, 30 Sep 2026 00:59:48 -0400 Subject: [PATCH 080/248] [Agents] Add API compatibility checking skill (#58003) Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Claude Fable 5.1 --- .agents/skills/check-api-compat/SKILL.md | 112 ++++++++++++++++++ .../check-api-compat/agents/openai.yaml | 4 + .claude/skills/check-api-compat | 1 + 3 files changed, 117 insertions(+) create mode 100644 .agents/skills/check-api-compat/SKILL.md create mode 100644 .agents/skills/check-api-compat/agents/openai.yaml create mode 120000 .claude/skills/check-api-compat diff --git a/.agents/skills/check-api-compat/SKILL.md b/.agents/skills/check-api-compat/SKILL.md new file mode 100644 index 0000000000..36fdb48cdd --- /dev/null +++ b/.agents/skills/check-api-compat/SKILL.md @@ -0,0 +1,112 @@ +--- +name: check-api-compat +description: Determine the correct behavior of vLLM's OpenAI-compatible or Anthropic-compatible API by checking the official upstream spec, instead of guessing from vLLM's own code. Use for any bug, issue, PR, review, or question touching /v1/chat/completions, /v1/completions, /v1/responses, /v1/embeddings, /v1/messages, or their request/response models — including "is this a bug or intended", "what should this field default to", "does upstream allow this", and any change that adds, removes, or renames a field on those models. Do not use for vLLM-internal APIs (engine, scheduler, LLM class) or for endpoints with no upstream counterpart. +--- + +# Checking vLLM API compatibility against the upstream spec + +vLLM's OpenAI and Anthropic server surfaces are *reimplementations* of someone +else's spec. vLLM's own code, tests, and docstrings are not evidence of correct +behavior — they are the thing under test. Establish what upstream says first, +then compare. + +Never answer "what should this field do" from memory. Field names, types, and +especially defaults drift between spec versions. + +## 1. Pin down the question + +Name three things before looking anything up: + +- **Which surface** — OpenAI-compatible or Anthropic-compatible. They are separate + code paths in vLLM and separate specs. +- **Which endpoint**, by path. +- **Which field or behavior** — a request field, a response field, a streaming + event shape, an error code, or a status code. + +## 2. Read vLLM's implementation + +In a vLLM checkout, the request/response models live beside each endpoint: + +| surface | path | +| --- | --- | +| OpenAI chat | `vllm/entrypoints/openai/chat_completion/` | +| OpenAI completions | `vllm/entrypoints/openai/completion/` | +| OpenAI responses | `vllm/entrypoints/openai/responses/` | +| OpenAI embeddings | `vllm/entrypoints/pooling/embed/` | +| OpenAI shared base models, errors, usage | `vllm/entrypoints/serve/engine/protocol.py` | +| Anthropic messages | `vllm/entrypoints/anthropic/` | +| audio | `vllm/entrypoints/speech_to_text/*/protocol.py` | + +Each directory holds `protocol.py` (the Pydantic models), `serving.py` (behavior), +and `api_router.py` (routing and status codes). Read the field declaration *and* +the code that consumes it — a field can be declared faithfully and then ignored, +which is still an incompatibility. + +## 3. Fetch the upstream spec + +Prefer a source that can actually be read as text, in this order. + +### OpenAI + +1. The OpenAPI spec — authoritative, machine-readable, and fetches cleanly: + `https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml` +2. The installed `openai` Python SDK's typed params, which track the spec closely. + Locate them with `python -c "import openai,os;print(os.path.dirname(openai.__file__))"`, + then read e.g. `types/chat/completion_create_params.py`, + `types/chat/chat_completion.py`, `types/chat/chat_completion_chunk.py`, + `types/responses/`. Note the SDK version — an old SDK is an old spec. +3. The prose reference at `https://platform.openai.com/docs/api-reference/...` + for semantics the types do not carry. It is script-heavy and often fetches + poorly; treat a failed fetch as a fetch failure, not as absence of a field. + +### Anthropic + +1. `https://docs.claude.com/en/api/messages` and + `https://docs.claude.com/en/api/messages-streaming` — these render as clean + text and are the primary source for both fields and streaming event order. +2. The installed `anthropic` SDK's `types/message_create_params.py`, + `types/message.py`, `types/raw_message_stream_event.py`. + +Say which source and which version answered the question, so the finding can be +re-checked later. + +## 4. Compare on all four axes + +A field is compatible only when **name**, **type**, **default**, and **semantics** +all match. Most real bugs hide in the last two: a field that exists with the right +type but the wrong default, or one that is accepted and silently dropped. + +Also check, when relevant: + +- **Streaming** — chunk shape, the order of events, and the terminal event + (`data: [DONE]` for Chat Completions and Completions; `response.completed`, + `response.failed`, or `response.incomplete` with no `[DONE]` for Responses; + `message_stop` for Anthropic). +- **Errors** — HTTP status *and* the error body's `type`/`code` strings. +- **Omitted vs null** — upstream often distinguishes an absent field from an + explicit `null`. Pydantic defaults erase that distinction unless the model is + written to preserve it. + +## 5. Classify before calling it a bug + +Three outcomes, and they are not the same: + +- **Matches upstream** — no action. +- **vLLM extension** — a field upstream has no name for at all (`vllm_xargs`, + `chat_template_kwargs`, sampling knobs like `min_p`). These are intentional and + allowed. Do not report them as violations. Check per surface: `top_k` is an + extension on OpenAI but a real upstream field on Anthropic Messages. +- **Incompatibility** — a field that *shares a name with upstream* but differs in + type, default, or behavior, or an upstream field that is missing, ignored, or + malformed. This is the reportable case. + +An unsupported-but-declared field is its own subcase: decide whether vLLM rejects +it explicitly or accepts and ignores it, and report which, since silently ignoring +a documented field is the more damaging failure. + +## 6. Report + +State: the endpoint and field, what upstream specifies (with the source), what +vLLM does (with `file:line`), which of the three outcomes it is, and the smallest +fix. When behavior turns on a spec detail that is easy to disbelieve, quote the +upstream line rather than paraphrasing it. diff --git a/.agents/skills/check-api-compat/agents/openai.yaml b/.agents/skills/check-api-compat/agents/openai.yaml new file mode 100644 index 0000000000..ef94d20b7d --- /dev/null +++ b/.agents/skills/check-api-compat/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Check API Compat" + short_description: "Check vLLM's OpenAI/Anthropic-compatible API against the upstream spec." + default_prompt: "Use the vLLM API compatibility skill to check this endpoint or field against the upstream OpenAI or Anthropic spec." diff --git a/.claude/skills/check-api-compat b/.claude/skills/check-api-compat new file mode 120000 index 0000000000..b33d69cc59 --- /dev/null +++ b/.claude/skills/check-api-compat @@ -0,0 +1 @@ +../../.agents/skills/check-api-compat \ No newline at end of file From ac68c3087215e0a4f3cdfa218508c6aada57235d Mon Sep 17 00:00:00 2001 From: Shijie Lyu <115741859+shijie-lyu@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:21:09 -0700 Subject: [PATCH 081/248] [Bugfix][Frontend] Ignore reused prompt token ids for media in /v1/responses (#59173) Signed-off-by: Shijie Lyu --- .../responses/test_serving_responses.py | 97 +++++++++++++++++++ vllm/entrypoints/chat_utils.py | 26 +++++ .../openai/chat_completion/protocol.py | 37 ++----- vllm/renderers/online_renderer.py | 19 +++- 4 files changed, 146 insertions(+), 33 deletions(-) diff --git a/tests/entrypoints/openai/responses/test_serving_responses.py b/tests/entrypoints/openai/responses/test_serving_responses.py index d07a89ec9c..591ba5c42a 100644 --- a/tests/entrypoints/openai/responses/test_serving_responses.py +++ b/tests/entrypoints/openai/responses/test_serving_responses.py @@ -34,6 +34,7 @@ from openai_harmony import Message as OpenAIHarmonyMessage from openai_harmony import Role import vllm.envs as envs +from tests.entrypoints.openai.chat_completion.test_serving_chat import MockModelConfig from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, @@ -294,6 +295,102 @@ async def test_online_renderer_renders_non_harmony_responses_with_explicit_histo assert preprocess_kwargs["default_template_kwargs"]["request_default"] == "kept" +_REUSED_IDS = [10, 20, 30] +_IMAGE_PART = { + "type": "input_image", + "image_url": "https://example.com/a.png", + "detail": "auto", +} + + +async def _render_responses_with_reuse(request, previous_messages=None): + online_renderer = OnlineRenderer( + model_config=MockModelConfig(), + renderer=MagicMock(), + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + render_chat = AsyncMock(return_value=([[]], [tokens_input([1, 2])])) + online_renderer.renderer.render_chat_async = render_chat + + result = await online_renderer.render_responses( + request, previous_messages=previous_messages + ) + + assert not isinstance(result, ErrorResponse) + return render_chat, result.engine_input + + +@pytest.mark.asyncio +@pytest.mark.parametrize("ids", [_REUSED_IDS, [1.5]]) +@pytest.mark.parametrize( + "request_input", + [ + [ + { + "role": "user", + "content": [{"type": "input_text", "text": "look"}, _IMAGE_PART], + } + ], + [ + { + "type": "function_call", + "call_id": "call_1", + "name": "shot", + "arguments": "{}", + }, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [_IMAGE_PART], + }, + ], + ], + ids=["input_image", "tool_output_image"], +) +async def test_responses_kv_transfer_prompt_token_ids_ignored_with_non_text_input( + request_input, ids +): + request = ResponsesRequest( + input=request_input, + kv_transfer_params={"do_remote_prefill": True, "prompt_token_ids": ids}, + ) + + render_chat, _ = await _render_responses_with_reuse(request) + + render_chat.assert_awaited_once() + assert request.kv_transfer_params == {"do_remote_prefill": True} + + +@pytest.mark.asyncio +async def test_responses_kv_transfer_prompt_token_ids_ignored_with_media_in_history(): + request = ResponsesRequest( + input="next", kv_transfer_params={"prompt_token_ids": _REUSED_IDS} + ) + + render_chat, _ = await _render_responses_with_reuse( + request, previous_messages=[{"role": "user", "content": [_IMAGE_PART]}] + ) + + render_chat.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_responses_kv_transfer_prompt_token_ids_kept_with_text_input(): + request = ResponsesRequest( + input=[ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"role": "assistant", "content": "ok"}, + ], + kv_transfer_params={"prompt_token_ids": _REUSED_IDS}, + ) + + _, engine_input = await _render_responses_with_reuse(request) + + assert engine_input["prompt_token_ids"] == _REUSED_IDS + + @pytest.mark.asyncio async def test_online_renderer_rejects_untrusted_responses_chat_template(): renderer = _new_online_renderer() diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index a419ffca81..10c4866f89 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1814,6 +1814,32 @@ PART_TYPES_TO_SKIP_NONE_CONTENT = ( TEXT_PART_TYPES = frozenset( {"text", "input_text", "output_text", "refusal", "thinking"} ) +# Content part types that carry no multimodal data. +_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} +# Keys that mark a content part as multimodal, whatever its ``type``. +_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES + + +def has_non_text_content(messages: Any) -> bool: + """Whether any message in ``messages`` has a non-text content part. + + Only list content is inspected, so validated chat content that is a + one-shot iterator is never consumed. + """ + if not isinstance(messages, list): + return False + for msg in messages: + content = msg.get("content") if isinstance(msg, dict) else None + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and ( + any(key in part for key in _MEDIA_CONTENT_PART_KEYS) + or not isinstance(part_type := part.get("type", "text"), str) + or part_type not in _TEXT_CONTENT_PART_TYPES + ): + return True + return False def _parse_chat_message_content_parts( diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 8ad7534d6d..228472ab6b 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -20,10 +20,9 @@ from pydantic import ( from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( - MM_PARSER_MAP, - TEXT_PART_TYPES, ChatCompletionMessageParam, ChatTemplateContentFormatOption, + has_non_text_content, ) from vllm.entrypoints.generate.base.protocol import ( AnyResponseFormat, @@ -60,11 +59,6 @@ logger = init_logger(__name__) _INT64_MIN = -(2**63) _INT64_MAX = 2**63 - 1 -# Content part types that carry no multimodal data. -_TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} -# Keys that mark a content part as multimodal, whatever its ``type``. -_MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES - class ChatMessage(OpenAIBaseModel): role: str @@ -991,29 +985,16 @@ class ChatCompletionRequest(OpenAIBaseModel): if not isinstance(data, dict): return data kv_transfer_params = data.get("kv_transfer_params") - messages = data.get("messages") if ( - not isinstance(kv_transfer_params, dict) - or kv_transfer_params.get("prompt_token_ids") is None - or not isinstance(messages, list) + isinstance(kv_transfer_params, dict) + and kv_transfer_params.get("prompt_token_ids") is not None + and has_non_text_content(data.get("messages")) ): - return data - for msg in messages: - content = msg.get("content") if isinstance(msg, dict) else None - if not isinstance(content, list): - continue - for part in content: - if isinstance(part, dict) and ( - any(key in part for key in _MEDIA_CONTENT_PART_KEYS) - or not isinstance(part_type := part.get("type", "text"), str) - or part_type not in _TEXT_CONTENT_PART_TYPES - ): - logger.debug( - "Ignoring kv_transfer_params['prompt_token_ids']: " - "messages have non-text content and are rendered instead." - ) - kv_transfer_params.pop("prompt_token_ids") - return data + logger.debug( + "Ignoring kv_transfer_params['prompt_token_ids']: " + "messages have non-text content and are rendered instead." + ) + kv_transfer_params.pop("prompt_token_ids") return data @model_validator(mode="before") diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 5108adfe81..bbe24f7df3 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -16,6 +16,7 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, ConversationMessage, + has_non_text_content, ) from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -101,16 +102,18 @@ def _extract_allowed_tools_from_mcp_requests( return allowed_tools_map -def _reused_prompt_token_ids(request: Any) -> list[int] | None: +def _reused_prompt_token_ids( + request: Any, messages: list[Any] | None = None +) -> list[int] | None: """Pop prompt token ids forwarded for decode-side reuse, if any. Disaggregated serving carries the prefill stage's ids in ``kv_transfer_params`` so the decode stage can skip re-tokenizing. Removing the key keeps the id list out of the engine's sampling metadata. - Returns None without checking the ids when ``echo`` is set, since echo - needs ``messages`` to be rendered. Otherwise raises VLLMValidationError if - the ids are malformed. + Returns None without checking the ids when ``echo`` is set or ``messages`` + has non-text content, since both need ``messages`` to be rendered. + Otherwise raises VLLMValidationError if the ids are malformed. """ kv = getattr(request, "kv_transfer_params", None) if not isinstance(kv, dict): @@ -124,6 +127,12 @@ def _reused_prompt_token_ids(request: Any) -> list[int] | None: "echo is set, so messages are rendered instead." ) return None + if has_non_text_content(messages): + logger.debug( + "Ignoring kv_transfer_params['prompt_token_ids']: " + "messages have non-text content and are rendered instead." + ) + return None # bool is an int subclass, hence the exact type check. if ( not isinstance(ids, list) @@ -772,7 +781,7 @@ class OnlineRenderer: default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), ) - reuse_ids = _reused_prompt_token_ids(request) + reuse_ids = _reused_prompt_token_ids(request, messages) if reuse_ids: # Decode-side token reuse: feed the forwarded ids straight to the # engine, skipping templating and tokenization. ``messages`` are not From 329e6eee3f3cb1d028666afaf23c65394536efb6 Mon Sep 17 00:00:00 2001 From: Ting SUN Date: Wed, 30 Sep 2026 13:32:38 +0800 Subject: [PATCH 082/248] [Bugfix][Frontend] Report named Anthropic tool calls as tool_use (#47598) Signed-off-by: Ting Sun Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../test_anthropic_messages_conversion.py | 79 +++++++++++++- vllm/entrypoints/anthropic/serving.py | 102 +++++++++++++----- 2 files changed, 147 insertions(+), 34 deletions(-) diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 06efefa099..ccd88c462b 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -37,6 +37,8 @@ from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, DeltaToolCall, + FunctionCall, + ToolCall, ) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionResponse, @@ -999,11 +1001,6 @@ class TestInlineSystemMessageInMessagesArray: def _make_stream_converter(): obj = MagicMock(spec=AnthropicServingMessages) - obj.stop_reason_map = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - } obj.message_stream_converter = ( AnthropicServingMessages.message_stream_converter.__get__(obj) ) @@ -1671,6 +1668,78 @@ class TestStopSequenceReason: assert msg_deltas[0]["delta"]["stop_sequence"] is None +class TestToolUseStopReason: + """A response carrying a tool_use block reports ``stop_reason="tool_use"`` + even when the chat layer says ``finish_reason="stop"``, as it does for a + forced (named) tool. A call cut short by max_tokens or a stop string does not. + """ + + @pytest.mark.parametrize( + ("finish_reason", "stop_reason", "expected"), + [ + ("stop", None, "tool_use"), + ("tool_calls", "", "tool_use"), + ], + ) + def test_non_streaming(self, finish_reason, stop_reason, expected): + tool_call = ToolCall( + function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}') + ) + response = ChatCompletionResponse( + id="chatcmpl-test", + model="test-model", + choices=[ + ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", tool_calls=[tool_call]), + finish_reason=finish_reason, + stop_reason=stop_reason, + ) + ], + usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), + ) + + result = _make_full_converter().messages_full_converter(response) + + assert result.content[0].type == "tool_use" + assert result.stop_reason == expected + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("args", "finish_reason", "stop_reason", "expected"), + [ + ('{"city": "Paris"}', "stop", None, "tool_use"), + ('{"city": "Pa', "length", None, "max_tokens"), + ('{"city": "Paris"}', "tool_calls", "", "tool_use"), + ('{"city": "Pa', "tool_calls", "", "stop_sequence"), + ], + ) + async def test_streaming(self, args, finish_reason, stop_reason, expected): + async def sse_input(): + yield _make_stream_chunk(delta=DeltaMessage(role="assistant")) + yield _make_stream_chunk( + delta=DeltaMessage( + tool_calls=[_tc(id="call_1", name="get_weather", args=args)] + ) + ) + yield _make_stream_chunk( + finish_reason=finish_reason, stop_reason=stop_reason + ) + yield _make_stream_chunk( + choices=[], + usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + events = _parse_sse_events( + [e async for e in converter.message_stream_converter(sse_input())] + ) + + msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"] + assert msg_deltas[0]["delta"]["stop_reason"] == expected + + # ====================================================================== # Client-caused errors are 4xx, not 500 (Issue #52088) # ====================================================================== diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index cd31908508..91820de2fd 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -10,7 +10,7 @@ import logging import time import uuid from collections.abc import AsyncGenerator -from typing import Any, get_args +from typing import Any, Literal, get_args import jinja2 from fastapi import Request @@ -97,6 +97,46 @@ def _build_anthropic_usage( return AnthropicUsage(**kwargs) +def _to_anthropic_stop_reason( + finish_reason: str | None, + stop_reason: int | str | None, + tool_used: bool, + tool_complete: bool, +) -> tuple[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None, + str | None, +]: + """Map an OpenAI finish reason to Anthropic ``(stop_reason, stop_sequence)``. + + Anthropic reports ``tool_use`` whenever the model invoked a tool. The chat + layer reports ``finish_reason="stop"`` for a named (forced) tool call, so + ``tool_used`` records whether a ``tool_use`` block was produced. + ``max_tokens`` always wins. A matched stop string wins unless the tool + calls are complete (``tool_complete``): a call it cut short must not run. + """ + if finish_reason == "length": + return "max_tokens", None + if finish_reason not in ("stop", "tool_calls"): + return None, None + if tool_complete: + return "tool_use", None + # vLLM reports the matched stop string in stop_reason (a str); + # an int stop-token-id or None (natural EOS) is not a stop sequence. + if isinstance(stop_reason, str): + return "stop_sequence", stop_reason + if finish_reason == "tool_calls" or tool_used: + return "tool_use", None + return "end_turn", None + + +def _is_json(text: str) -> bool: + try: + json.loads(text) + except json.JSONDecodeError: + return False + return True + + def wrap_data_with_event(data: str, event: str): return f"event: {event}\ndata: {data}\n\n" @@ -141,11 +181,6 @@ class AnthropicServingMessages(OpenAIServingChat): enable_force_include_usage=enable_force_include_usage, default_chat_template_kwargs=default_chat_template_kwargs, ) - self.stop_reason_map = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - } self._merge_inline_system = self._should_merge_inline_system(online_renderer) # Resolved lazily from the renderer when "auto". self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = ( @@ -751,18 +786,13 @@ class AnthropicServingMessages(OpenAIServingChat): ec_transfer_params=generator.ec_transfer_params, ) choice = generator.choices[0] - if choice.finish_reason == "stop": - # vLLM reports the matched stop string in stop_reason (a str); - # an int stop-token-id or None (natural EOS) maps to end_turn. - if isinstance(choice.stop_reason, str): - result.stop_reason = "stop_sequence" - result.stop_sequence = choice.stop_reason - else: - result.stop_reason = "end_turn" - elif choice.finish_reason == "length": - result.stop_reason = "max_tokens" - elif choice.finish_reason == "tool_calls": - result.stop_reason = "tool_use" + result.stop_reason, result.stop_sequence = _to_anthropic_stop_reason( + choice.finish_reason, + choice.stop_reason, + tool_used=bool(choice.message.tool_calls), + # Their arguments are json-decoded below, so parsed calls are complete. + tool_complete=bool(choice.message.tool_calls), + ) content: list[AnthropicContentBlock] = [] if choice.message.reasoning: @@ -845,6 +875,10 @@ class AnthropicServingMessages(OpenAIServingChat): # Matched stop string, when generation stopped on one (a str); # int stop-token-id / None are not stop sequences. stop_sequence: int | str | None = None + tool_used = False + # Arguments of the latest tool_use block; only the last call can be + # cut short by a stop string. + last_tool_args: list[str] = [] state = _ActiveBlockState() # Map from tool call index to tool_use_id tool_index_to_id: dict[int, str] = {} @@ -954,19 +988,21 @@ class AnthropicServingMessages(OpenAIServingChat): if len(origin_chunk.choices) == 0: for event in stop_and_flush(): yield event - if isinstance(stop_sequence, str): - stop_delta = AnthropicDelta( - stop_reason="stop_sequence", - stop_sequence=stop_sequence, - ) - else: - stop_delta = AnthropicDelta( - stop_reason=self.stop_reason_map.get( - finish_reason or "stop" + anthropic_stop_reason, anthropic_stop_sequence = ( + _to_anthropic_stop_reason( + finish_reason or "stop", + stop_sequence, + tool_used=tool_used, + tool_complete=( + tool_used and _is_json("".join(last_tool_args)) ), - # Set explicitly so exclude_unset=True keeps it. - stop_sequence=None, ) + ) + stop_delta = AnthropicDelta( + stop_reason=anthropic_stop_reason, + # Set explicitly so exclude_unset=True keeps it. + stop_sequence=anthropic_stop_sequence, + ) chunk = AnthropicStreamEvent( type="message_delta", delta=stop_delta, @@ -1060,6 +1096,8 @@ class AnthropicServingMessages(OpenAIServingChat): ): for event in stop_and_flush(): yield event + tool_used = True + last_tool_args = [] start_event = start_block( AnthropicContentBlock( type="tool_use", @@ -1075,6 +1113,9 @@ class AnthropicServingMessages(OpenAIServingChat): and tool_call.function.arguments and state.tool_use_id == tool_call.id ): + last_tool_args.append( + tool_call.function.arguments + ) chunk = AnthropicStreamEvent( index=( state.block_index @@ -1100,6 +1141,9 @@ class AnthropicServingMessages(OpenAIServingChat): and tool_call.function.arguments and state.tool_use_id == tool_use_id ): + last_tool_args.append( + tool_call.function.arguments + ) chunk = AnthropicStreamEvent( index=( state.block_index From 1117140edbaf3d1e1f73b01b09cde3cf98ec24ff Mon Sep 17 00:00:00 2001 From: Fabian Joswig Date: Wed, 30 Sep 2026 07:34:26 +0200 Subject: [PATCH 083/248] [Bugfix][Model] Fix DiffusionGemma silently freezing attention mask under CUDA graph replay (#51994) Signed-off-by: Fabian Joswig Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/diffusion_gemma.py | 15 +++++++++------ vllm/v1/attention/backends/flash_attn.py | 6 +++++- vllm/v1/attention/ops/triton_unified_attention.py | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/vllm/model_executor/models/diffusion_gemma.py b/vllm/model_executor/models/diffusion_gemma.py index 8f26edf9e4..404c154b54 100644 --- a/vllm/model_executor/models/diffusion_gemma.py +++ b/vllm/model_executor/models/diffusion_gemma.py @@ -909,9 +909,12 @@ class DiffusionGemmaModelState(ModelState): self._req_id_to_index: dict[str, int] = {} # Persistent buffer for per-request causal flags, updated in-place - # so FULL CUDA graph replay sees the latest values. + # so FULL CUDA graph replay sees the latest values. Must be int32: + # FlashAttentionMetadataBuilder.build() rejects other dtypes, since + # an out-of-place cast there would detach the captured graph from + # this buffer and freeze replay at the capture-time snapshot. self._causal_buf = torch.zeros( - self.max_num_reqs, dtype=torch.bool, device=device + self.max_num_reqs, dtype=torch.int32, device=device ) # Persistent inputs_embeds buffer — required so FULL CUDA graph @@ -1116,11 +1119,11 @@ class DiffusionGemmaModelState(ModelState): # Invariant: the sampler flips is_encoder_phase to False only after a # request's FINAL prompt chunk, so a prompt spanning multiple chunks # (longer than the token budget) stays causal for every chunk. - self._causal_buf[:actual_num_reqs] = self.diffusion_states.is_encoder_phase[ - slots - ] + self._causal_buf[:actual_num_reqs].copy_( + self.diffusion_states.is_encoder_phase[slots] + ) if actual_num_reqs < num_reqs: - self._causal_buf[actual_num_reqs:num_reqs] = False + self._causal_buf[actual_num_reqs:num_reqs] = 0 causal: bool | torch.Tensor = self._causal_buf[:num_reqs] return build_attn_metadata( diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 6035678394..6179b27bb6 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -987,7 +987,11 @@ class FlashAttentionMetadataBuilder(AttentionMetadataBuilder[FlashAttentionMetad scheduler_metadata = self._store_scheduler_metadata(scheduler_metadata) if isinstance(causal, torch.Tensor) and causal.dtype != torch.int32: - causal = causal.to(torch.int32) + raise ValueError( + f"Per-request causal tensor must be int32, got {causal.dtype}. " + "Casting here would allocate a fresh tensor each build() and " + "break FULL CUDA graph replay; allocate the buffer as int32." + ) attn_metadata = FlashAttentionMetadata( num_actual_tokens=num_actual_tokens, diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index b0231dd0c0..4d73e49e91 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -217,7 +217,7 @@ def kernel_unified_attention( SLIDING_WINDOW: tl.constexpr, # int USE_CAUSAL: tl.constexpr, # bool USE_PER_SEQ_CAUSAL: tl.constexpr, # bool - per_seq_causal_ptr, # [num_seqs] bool, or None + per_seq_causal_ptr, # [num_seqs] int32, or None USE_MM_PREFIX: tl.constexpr, # bool MAX_MM_RANGES: tl.constexpr, # int mm_prefix_range_ptr, From 6945d142e51cb9b4ea1ad0d054280b80734339ae Mon Sep 17 00:00:00 2001 From: liuzhenwei Date: Wed, 30 Sep 2026 13:43:21 +0800 Subject: [PATCH 084/248] [XPU] Use encoder-only model runner for EC producer instances (#59320) Signed-off-by: zhenwei-intel --- vllm/v1/worker/xpu_model_runner.py | 13 +++++++++++++ vllm/v1/worker/xpu_worker.py | 15 +++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/vllm/v1/worker/xpu_model_runner.py b/vllm/v1/worker/xpu_model_runner.py index f6747cb349..15fa8e1e76 100644 --- a/vllm/v1/worker/xpu_model_runner.py +++ b/vllm/v1/worker/xpu_model_runner.py @@ -11,6 +11,7 @@ from vllm.v1.worker.gpu.model_runner import ( GPUModelRunner as GPUModelRunnerV2, ) from vllm.v1.worker.gpu_model_runner import GPUModelRunner +from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner class XPUModelRunner(GPUModelRunner): @@ -39,6 +40,18 @@ class XPUModelRunnerV2(GPUModelRunnerV2): super().__init__(vllm_config, device) +class XPUMMEncoderModelRunner(MMEncoderModelRunner): + """An encoder-only model runner for XPU devices.""" + + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + ): + with _torch_cuda_wrapper(): + super().__init__(vllm_config, device) + + @contextmanager def _torch_cuda_wrapper(): # Replace cuda APIs with xpu APIs. Each callable gets its own functools.partial diff --git a/vllm/v1/worker/xpu_worker.py b/vllm/v1/worker/xpu_worker.py index 02cc2c9e87..8a8166c435 100644 --- a/vllm/v1/worker/xpu_worker.py +++ b/vllm/v1/worker/xpu_worker.py @@ -13,7 +13,11 @@ from vllm.utils.torch_utils import set_random_seed from vllm.v1.utils import report_usage_stats from vllm.v1.worker.gpu_worker import Worker, init_worker_distributed_environment from vllm.v1.worker.workspace import init_workspace_manager -from vllm.v1.worker.xpu_model_runner import XPUModelRunner, XPUModelRunnerV2 +from vllm.v1.worker.xpu_model_runner import ( + XPUMMEncoderModelRunner, + XPUModelRunner, + XPUModelRunnerV2, +) from .utils import request_memory @@ -215,7 +219,14 @@ class XPUWorker(Worker): init_workspace_manager(self.device, num_ubatches) # Construct the model runner - model_runner = XPUModelRunnerV2 if self.use_v2_model_runner else XPUModelRunner + if self.use_v2_model_runner: + model_runner = ( + XPUMMEncoderModelRunner + if self.vllm_config.is_mm_encoder_only + else XPUModelRunnerV2 + ) + else: + model_runner = XPUModelRunner # type: ignore[assignment] self.model_runner = model_runner( # type: ignore self.vllm_config, self.device ) From fcf4750466b07c2ba921a7da3831f585cbc2b8d7 Mon Sep 17 00:00:00 2001 From: Ting SUN Date: Wed, 30 Sep 2026 13:44:35 +0800 Subject: [PATCH 085/248] [Bugfix][Frontend] Constrain forced named tool choice with empty parameters to a JSON object (#45290) Signed-off-by: Ting Sun Co-authored-by: Flora Feng <4florafeng@gmail.com> --- tests/tool_use/test_tool_choice_required.py | 26 ++++++++++++++++++++- vllm/tool_parsers/utils.py | 12 ++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/tests/tool_use/test_tool_choice_required.py b/tests/tool_use/test_tool_choice_required.py index aaa499da16..a859659417 100644 --- a/tests/tool_use/test_tool_choice_required.py +++ b/tests/tool_use/test_tool_choice_required.py @@ -5,7 +5,7 @@ from copy import deepcopy import pytest import regex as re -from openai.types.responses import FunctionTool, WebSearchTool +from openai.types.responses import FunctionTool, ToolChoiceFunction, WebSearchTool from pydantic import TypeAdapter from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -456,3 +456,27 @@ class TestParallelToolCallsConstraint: ) assert isinstance(schema, dict) assert "maxItems" not in schema + + +class TestForcedNamedToolChoiceEmptyParams: + """A forced named tool_choice with missing/empty parameters must still + constrain the generated arguments to a JSON object, like the + `tool_choice="required"` path, instead of leaving them unconstrained.""" + + @pytest.mark.parametrize("params", [None, {}]) + def test_chat_empty_params_constrains_object(self, params): + tool = ChatCompletionToolsParam.model_validate( + {"type": "function", "function": {"name": "ping", "parameters": params}} + ) + choice = ChatCompletionNamedToolChoiceParam.model_validate( + {"type": "function", "function": {"name": "ping"}} + ) + schema = get_json_schema_from_tools(choice, [tool]) + assert schema == {"type": "object", "properties": {}} + + @pytest.mark.parametrize("params", [None, {}]) + def test_responses_empty_params_constrains_object(self, params): + tool = FunctionTool(type="function", name="ping", parameters=params) + choice = ToolChoiceFunction(type="function", name="ping") + schema = get_json_schema_from_tools(choice, [tool]) + assert schema == {"type": "object", "properties": {}} diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py index 081e58da04..a2992261da 100644 --- a/vllm/tool_parsers/utils.py +++ b/vllm/tool_parsers/utils.py @@ -308,14 +308,18 @@ def find_tool_name( return False +def _params_or_empty_object(params: dict[str, Any] | None) -> dict[str, Any]: + # Empty/missing parameters still constrain arguments to a JSON object. + return params if params else {"type": "object", "properties": {}} + + def _get_tool_schema_from_name_and_params( name: str, params: dict[str, Any] | None ) -> dict: - params = params if params else {"type": "object", "properties": {}} return { "properties": { "name": {"type": "string", "enum": [name]}, - "parameters": params, + "parameters": _params_or_empty_object(params), }, "required": ["name", "parameters"], } @@ -403,7 +407,7 @@ def get_json_schema_from_tools( responses_tool_map.setdefault(name.rsplit("__", 1)[1], params) if tool_name not in responses_tool_map: raise ValueError(f"Tool '{tool_name}' has not been passed in `tools`.") - return responses_tool_map[tool_name] + return _params_or_empty_object(responses_tool_map[tool_name]) # tool_choice: Forced Function (ChatCompletion) if (not isinstance(tool_choice, str)) and isinstance( tool_choice, ChatCompletionNamedToolChoiceParam @@ -416,7 +420,7 @@ def get_json_schema_from_tools( } if tool_name not in chat_tool_map: raise ValueError(f"Tool '{tool_name}' has not been passed in `tools`.") - return chat_tool_map[tool_name].function.parameters + return _params_or_empty_object(chat_tool_map[tool_name].function.parameters) # tool_choice: "required" if tool_choice == "required": return _get_json_schema_from_tools(tools, parallel_tool_calls) From 5faf81a4297921c699f9aa58f9c6395e95ede718 Mon Sep 17 00:00:00 2001 From: Harshal Adhav Date: Wed, 30 Sep 2026 11:22:53 +0530 Subject: [PATCH 086/248] [CPU][Whisper] Support W4A16 quantized Whisper on the CPU WNA16 kernel (#58268) Signed-off-by: Harshal Adhav Co-authored-by: Cursor Co-authored-by: Li, Jiang --- tests/kernels/test_awq_int4_to_int8.py | 9 ++-- .../multimodal/generation/test_whisper.py | 44 +++++++++++++++++ tests/quantization/test_cpu_wna16.py | 45 +++++++++++++++++ vllm/_custom_ops.py | 13 +++-- vllm/model_executor/models/whisper.py | 48 ++++++++++++++++--- 5 files changed, 144 insertions(+), 15 deletions(-) diff --git a/tests/kernels/test_awq_int4_to_int8.py b/tests/kernels/test_awq_int4_to_int8.py index dc7124e9ae..0eea482433 100644 --- a/tests/kernels/test_awq_int4_to_int8.py +++ b/tests/kernels/test_awq_int4_to_int8.py @@ -203,7 +203,7 @@ class TestInt4ScaledMmCpu: print(f" [PASS] INT4 GEMM with bias: M={M}") def test_gemm_3d_input(self): - """apply() reshapes 3D input [B, S, K] -> [B*S, K] -> back to 3D.""" + """int4_scaled_mm_cpu flattens 3D [B, S, K] to 2D and reshapes back.""" K, N, group_size = 256, 128, 128 (packed_qweight, packed_qzeros, scales, float_ref, _, _) = ( make_awq_checkpoint_data(K, N, group_size) @@ -220,12 +220,13 @@ class TestInt4ScaledMmCpu: x_3d = torch.randn(B, S, K, dtype=torch.bfloat16) x_2d = x_3d.reshape(-1, K) + out_3d = int4_scaled_mm_cpu(x_3d, blocked_w, blocked_zp, blocked_s, None) out_2d = int4_scaled_mm_cpu(x_2d, blocked_w, blocked_zp, blocked_s, None) - out_3d = out_2d.reshape(B, S, N) - - ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N) assert out_3d.shape == (B, S, N) + torch.testing.assert_close(out_3d.reshape(-1, N), out_2d) + + ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N) abs_diff = (out_3d.float() - ref_out).abs() mean_abs = abs_diff.mean().item() ref_mag = ref_out.abs().mean().item() + 1e-6 diff --git a/tests/models/multimodal/generation/test_whisper.py b/tests/models/multimodal/generation/test_whisper.py index fef6b69911..db27fa86de 100644 --- a/tests/models/multimodal/generation/test_whisper.py +++ b/tests/models/multimodal/generation/test_whisper.py @@ -5,6 +5,7 @@ from collections.abc import Sequence from typing import Any import pytest +import torch from transformers import AutoModelForSpeechSeq2Seq from vllm.assets.audio import AudioAsset @@ -264,6 +265,49 @@ def test_parse_language_detection_output(): cls.parse_language_detection_output([], make_tokenizer("anything")) +def test_create_fake_bias_for_k_proj_injects_out_features(): + """Packed k_proj weights get a zeros bias of length ``out_features``.""" + from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj + + packed = torch.ones(2, 16) + result = dict( + _create_fake_bias_for_k_proj( + [("model.encoder.layers.0.self_attn.k_proj.weight_packed", packed)], + ".k_proj.weight_packed", + out_features=8, + ) + ) + bias_name = "model.encoder.layers.0.self_attn.k_proj.bias" + assert bias_name in result + assert "model.encoder.layers.0.self_attn.k_proj.bias_packed" not in result + assert tuple(result[bias_name].shape) == (8,) + assert torch.equal(result[bias_name], torch.zeros(8)) + + +def test_create_fake_bias_for_k_proj_forwards_real_bias(): + """A checkpoint bias is forwarded and no extra fake bias is emitted.""" + from vllm.model_executor.models.whisper import _create_fake_bias_for_k_proj + + weight = torch.ones(8, 16) + real_bias = torch.ones(8) + items = list( + _create_fake_bias_for_k_proj( + [ + ("layers.0.self_attn.k_proj.weight", weight), + ("layers.0.self_attn.k_proj.bias", real_bias), + ], + ".k_proj.weight", + out_features=8, + ) + ) + names = [n for n, _ in items] + assert names == [ + "layers.0.self_attn.k_proj.weight", + "layers.0.self_attn.k_proj.bias", + ] + assert items[1][1] is real_bias + + @pytest.mark.core_model @pytest.mark.cpu_model @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"]) diff --git a/tests/quantization/test_cpu_wna16.py b/tests/quantization/test_cpu_wna16.py index ffdc93f454..2af633be6b 100644 --- a/tests/quantization/test_cpu_wna16.py +++ b/tests/quantization/test_cpu_wna16.py @@ -1,12 +1,57 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +import torch from vllm.platforms import current_platform if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True) +from vllm._custom_ops import cpu_gemm_wna16 # noqa: E402 +from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( # noqa: E402 + _get_isa_hint, +) + +# Isolated runs of cpu_gemm_wna16 can hang unless AMX tiles are initialized. +# Guard so non-AMX hosts still take the vec/rvv path. +if torch.cpu._is_amx_tile_supported(): + torch.cpu._init_amx() + + +@pytest.mark.cpu_test +def test_cpu_gemm_wna16_3d_input(): + """cpu_gemm_wna16 flattens 3D [B, T, K] to 2D and reshapes the output back.""" + if not hasattr(torch.ops._C, "cpu_gemm_wna16"): + pytest.skip("cpu_gemm_wna16 op is not available") + + B, T, K, N = 2, 5, 64, 64 + pack_factor = 8 + group_size = 32 + dtype = torch.bfloat16 + q_weight = torch.zeros(N // 16, K * 16 // pack_factor, dtype=torch.int32) + scales = torch.ones(K // group_size, N, dtype=dtype) + + x_3d = torch.randn(B, T, K, dtype=dtype) + x_2d = x_3d.reshape(-1, K) + isa_hint = _get_isa_hint(dtype) + kwargs = dict( + q_weight=q_weight, + scales=scales, + zeros=None, + bias=None, + pack_factor=pack_factor, + isa_hint=isa_hint, + ) + + out_3d = cpu_gemm_wna16(input=x_3d, **kwargs) + out_2d = cpu_gemm_wna16(input=x_2d, **kwargs) + + assert out_3d.shape == (B, T, N) + assert out_2d.shape == (B * T, N) + torch.testing.assert_close(out_3d.reshape(-1, N), out_2d) + + MODELS = [ "TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ", "Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index be9a111a58..c21f85c029 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -4434,18 +4434,23 @@ def cpu_gemm_wna16( pack_factor: int, isa_hint: str, ) -> torch.Tensor: - output = torch.empty((input.size(0), scales.size(1)), dtype=input.dtype) + # Match int4_scaled_mm_cpu: flatten >2-D activations to [M, K] for the + # C++ kernel, then restore the original leading dims on the output. + x_shape = input.shape + x_2d = input.reshape(-1, x_shape[-1]) if len(x_shape) > 2 else input + out = torch.empty((x_2d.size(0), scales.size(1)), dtype=input.dtype) torch.ops._C.cpu_gemm_wna16( - input, + x_2d, q_weight, - output, + out, scales, zeros, bias, pack_factor, isa_hint, ) - return output + out = out.reshape(x_shape[:-1] + (out.size(-1),)) if len(x_shape) > 2 else out + return out def cpu_activation_lut_bf16(input: torch.Tensor, activation: str) -> torch.Tensor: diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 09f5f85453..9e5d74cba2 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -1039,20 +1039,54 @@ class WhisperForConditionalGeneration( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) - # add fake zeros bias for k_proj to state_dict - weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight") + # add fake zeros bias for k_proj to state_dict. Covers the + # ``.k_proj.weight`` and the compressed-tensors packed + # ``.k_proj.weight_packed`` checkpoints. + k_proj_weight_suffixes = (".k_proj.weight", ".k_proj.weight_packed") + for suffix in k_proj_weight_suffixes: + weights = _create_fake_bias_for_k_proj( + weights, suffix, out_features=self.config.d_model + ) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def _create_fake_bias_for_k_proj( - weights: Iterable[tuple[str, torch.Tensor]], fake_bias_key_name: str + weights: Iterable[tuple[str, torch.Tensor]], + fake_bias_key_name: str, + out_features: int | None = None, ) -> Iterable[tuple[str, torch.Tensor]]: """Create full zeros bias for k_proj weight in self-attn and x-attn layers. - So that the bias for k_proj in qkv_proj can be initialized with zeros. + So that the bias for k_proj in qkv_proj or kv_proj can be initialized with + zeros. + + If the checkpoint already provides a real ``.bias`` entry for the given + weight, it is forwarded as-is and no fake bias is injected for that layer. """ + # Map the weight-name suffix to the corresponding bias-name suffix, e.g. + # ".k_proj.weight_packed" / ".k_proj.weight" -> ".k_proj.bias", + # ".wk.weight" -> ".wk.bias". ``.weight_packed`` is stripped first so the + # longer suffix wins. + bias_key_name = ( + fake_bias_key_name.removesuffix(".weight_packed").removesuffix(".weight") + + ".bias" + ) + + real_bias_names: set[str] = set() + pending: dict[str, torch.Tensor] = {} for name, weight in weights: yield name, weight + + if name.endswith(bias_key_name): + real_bias_names.add(name) + pending.pop(name, None) + continue + if name.endswith(fake_bias_key_name): - bias = torch.zeros(weight.size(0)) - bias_name = name.replace("weight", "bias") - yield bias_name, bias + bias_name = name[: -len(fake_bias_key_name)] + bias_key_name + if bias_name not in real_bias_names: + pending[bias_name] = torch.zeros( + out_features if out_features is not None else weight.size(0) + ) + + for bias_name, bias in pending.items(): + yield bias_name, bias From 866fa130fac1e1252330a680fbed45ad05eba64f Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 30 Sep 2026 01:02:55 -0500 Subject: [PATCH 087/248] [Core] Bound UniProc EngineCore startup threads to available CPUs (#58946) Signed-off-by: Andreas Karatzas --- .../v1/engine/test_startup_watch_processes.py | 123 ++++++++++++++++++ vllm/v1/engine/utils.py | 94 +++++++++---- 2 files changed, 189 insertions(+), 28 deletions(-) diff --git a/tests/v1/engine/test_startup_watch_processes.py b/tests/v1/engine/test_startup_watch_processes.py index 0b346efb65..feb1e04268 100644 --- a/tests/v1/engine/test_startup_watch_processes.py +++ b/tests/v1/engine/test_startup_watch_processes.py @@ -1,14 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import os +from contextlib import nullcontext from multiprocessing import connection from threading import Event from types import SimpleNamespace import pytest +import torch import zmq import vllm.platforms as platforms +from vllm.utils import torch_utils from vllm.v1.engine import core as core_module from vllm.v1.engine import utils as engine_utils from vllm.v1.engine.core import EngineCoreProc, EngineShutdownState @@ -19,10 +23,129 @@ from vllm.v1.engine.utils import ( EngineZmqAddresses, wait_for_engine_startup, ) +from vllm.v1.executor import UniProcExecutor, multiproc_executor pytestmark = pytest.mark.skip_global_cleanup +@pytest.mark.parametrize( + ( + "executor_class", + "local_engine_count", + "initial_threads", + "user_threads", + "fail_start", + "expected", + ), + [ + (UniProcExecutor, 1, 192, None, False, (12, "12", "1")), + (UniProcExecutor, 2, 192, None, False, (6, "6", "1")), + (UniProcExecutor, 2, 192, None, True, (6, "6", "1")), + (UniProcExecutor, 2, 5, None, False, (5, "5", "1")), + (UniProcExecutor, 2, 1, None, False, (1, "1", "1")), + (UniProcExecutor, 2, 5, "5", False, (5, "5", None)), + ( + multiproc_executor.MultiprocExecutor, + 2, + 192, + None, + False, + (192, None, None), + ), + ], +) +def test_engine_core_startup_threads_are_scoped_to_launch( + monkeypatch: pytest.MonkeyPatch, + executor_class, + local_engine_count: int, + initial_threads: int, + user_threads: str | None, + fail_start: bool, + expected: tuple[int, str | None, str | None], +): + """UniProc workers inherit their CPU share and restore the parent.""" + marker = torch_utils.OMP_NUM_THREADS_SET_BY_VLLM + monkeypatch.delenv(marker, raising=False) + monkeypatch.delenv("OMP_NUM_THREADS", raising=False) + if user_threads is not None: + monkeypatch.setenv("OMP_NUM_THREADS", user_threads) + monkeypatch.setattr(torch_utils, "available_cpu_count", lambda: 12) + platform = SimpleNamespace( + is_cpu=lambda: False, is_cuda_alike=lambda: True, is_xpu=lambda: False + ) + monkeypatch.setattr(engine_utils, "current_platform", platform) + threads = SimpleNamespace(count=initial_threads) + setter_calls = [] + + def set_threads(count): + setter_calls.append(count) + threads.count = count + + monkeypatch.setattr(torch, "get_num_threads", lambda: threads.count) + monkeypatch.setattr(torch, "set_num_threads", set_threads) + start_states = [] + + def thread_state(): + return ( + threads.count, + os.environ.get("OMP_NUM_THREADS"), + os.environ.get(marker), + ) + + def start_process(): + start_states.append(thread_state()) + if fail_start: + raise RuntimeError("start failed") + + def get_context(): + assert thread_state() == (initial_threads, user_threads, None) + return SimpleNamespace( + Process=lambda **kwargs: SimpleNamespace( + name=kwargs["name"], + exitcode=1 if fail_start else None, + start=start_process, + ) + ) + + monkeypatch.setattr(engine_utils, "get_mp_context", get_context) + monkeypatch.setattr(engine_utils, "shutdown", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + engine_utils.numa_utils, + "configure_subprocess", + lambda *args, **kwargs: nullcontext(), + ) + with ( + pytest.raises(RuntimeError, match="start failed") + if fail_start + else nullcontext() + ): + manager = CoreEngineProcManager( + local_engine_count=local_engine_count, + start_index=0, + local_start_index=0, + vllm_config=SimpleNamespace( + shutdown_timeout=0, + parallel_config=SimpleNamespace( + data_parallel_size=local_engine_count, + assigned_physical_gpu_ids=None, + use_ray=False, + ), + ), + local_client=True, + handshake_address="unused", + executor_class=executor_class, + log_stats=False, + ) + manager._finalizer.detach() + assert start_states == [expected] * (1 if fail_start else local_engine_count) + assert thread_state() == (initial_threads, user_threads, None) + assert setter_calls == ( + [expected[0], initial_threads] + if executor_class is UniProcExecutor and expected[0] < initial_threads + else [] + ) + + @pytest.mark.parametrize( ("is_rocm", "request_timeout", "manager_timeout", "process_timeout"), [ diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index e786cf2c5b..9df749f365 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -28,9 +28,9 @@ from vllm.utils.network_utils import ( get_tcp_uri, zmq_socket_ctx, ) -from vllm.utils.system_utils import get_mp_context +from vllm.utils.system_utils import get_mp_context, set_env_var from vllm.v1.engine.coordinator import DPCoordinator -from vllm.v1.executor import Executor +from vllm.v1.executor import Executor, UniProcExecutor from vllm.v1.executor.ray_utils import WORKER_SPECIFIC_ENV_VARS from vllm.v1.utils import _SubprocessWrapper, get_engine_client_zmq_addr, shutdown @@ -166,6 +166,42 @@ def _node_ip_from_resources(node_resources: dict) -> str | None: return None +@contextlib.contextmanager +def _configure_uniproc_startup_threads( + executor_class: type[Executor], local_engine_count: int +) -> Iterator[None]: + if ( + not issubclass(executor_class, UniProcExecutor) + or current_platform.is_cpu() + or "OMP_NUM_THREADS" in os.environ + ): + yield + return + + import torch + + from vllm.utils.torch_utils import ( + OMP_NUM_THREADS_SET_BY_VLLM, + set_default_torch_num_threads, + startup_omp_num_threads, + ) + + original_threads = torch.get_num_threads() + num_threads = min(original_threads, startup_omp_num_threads(local_engine_count)) + torch_threads = ( + set_default_torch_num_threads(num_threads) + if num_threads < original_threads + else contextlib.nullcontext() + ) + # Fork inherits Torch's setting; spawn reads OMP_NUM_THREADS on import. + with ( + set_env_var("OMP_NUM_THREADS", str(num_threads)), + set_env_var(OMP_NUM_THREADS_SET_BY_VLLM, "1"), + torch_threads, + ): + yield + + class CoreEngineProcManager: """Utility class to handle creation, readiness, and shutdown of background processes used by the AsyncLLM and LLMEngine. @@ -229,34 +265,36 @@ class CoreEngineProcManager: # pickles process args at start() time, sequentially per rank. user_assigned_gpu_ids = vllm_config.parallel_config.assigned_physical_gpu_ids try: - for proc, local_dp_rank in zip(self.processes, local_dp_ranks): - # Populate the logical-to-physical GPU mapping in DP for - # platforms that cannot rely on - # torch.accelerator.set_device_index(), and for Ray. - needs_device_env_isolation = not ( - current_platform.is_cuda_alike() or current_platform.is_xpu() - ) - if is_dp and ( - needs_device_env_isolation or vllm_config.parallel_config.use_ray - ): - set_assigned_physical_gpu_ids_for_dp_rank( - vllm_config, local_dp_rank, user_assigned_gpu_ids + with _configure_uniproc_startup_threads(executor_class, local_engine_count): + for proc, local_dp_rank in zip(self.processes, local_dp_ranks): + # Populate the logical-to-physical GPU mapping in DP for + # platforms that cannot rely on + # torch.accelerator.set_device_index(), and for Ray. + needs_device_env_isolation = not ( + current_platform.is_cuda_alike() or current_platform.is_xpu() ) + if is_dp and ( + needs_device_env_isolation + or vllm_config.parallel_config.use_ray + ): + set_assigned_physical_gpu_ids_for_dp_rank( + vllm_config, local_dp_rank, user_assigned_gpu_ids + ) - with numa_utils.configure_subprocess( - # EngineCore itself does not have a TP/PP-local rank. - # When DP is enabled, set_assigned_physical_gpu_ids_for_dp_rank() - # populates the logical-to-physical mapping for this DP - # shard, so local_rank=0 means "the first local GPU in - # this shard". The actual TP/PP worker processes spawned - # by the executor are bound separately with their own - # local_rank values. - vllm_config, - local_rank=0, - dp_local_rank=local_dp_rank, - process_kind="EngineCore", - ): - proc.start() + with numa_utils.configure_subprocess( + # EngineCore itself does not have a TP/PP-local rank. + # set_assigned_physical_gpu_ids_for_dp_rank() populates + # the logical-to-physical mapping for this DP + # shard, so local_rank=0 means "the first local GPU in + # this shard". The actual TP/PP worker processes spawned + # by the executor are bound separately with their own + # local_rank values. + vllm_config, + local_rank=0, + dp_local_rank=local_dp_rank, + process_kind="EngineCore", + ): + proc.start() finally: # Kill other procs if not all are running. if self.finished_procs(): From 29773c722b8afd051b499954e51444f07e8b97f6 Mon Sep 17 00:00:00 2001 From: Ning Xie Date: Wed, 30 Sep 2026 14:13:24 +0800 Subject: [PATCH 088/248] [Observability] log sub process killing in process manager force kill (#52314) Signed-off-by: Andy Xie --- vllm/utils/system_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/utils/system_utils.py b/vllm/utils/system_utils.py index ae09ed2dbe..5b90438dca 100644 --- a/vllm/utils/system_utils.py +++ b/vllm/utils/system_utils.py @@ -221,6 +221,11 @@ def kill_process_tree(pid: int): # Send SIGKILL to all children first for child in children: with contextlib.suppress(ProcessLookupError): + logger.warning( + "[shutdown] force killing sub process %s pid %d", + child.name(), + child.pid, + ) os.kill(child.pid, signal.SIGKILL) # Finally kill the parent From e5f3d08d722870a5301560fad0cfa65b7e333900 Mon Sep 17 00:00:00 2001 From: Ronald Date: Wed, 30 Sep 2026 14:16:38 +0800 Subject: [PATCH 089/248] [Frontend][RL] Align sleep-mode API responses and operation metrics (#52864) Signed-off-by: Ronald1995 Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- .../engine-core-client/src/tests/client.rs | 83 +++++ rust/src/metrics/src/api_server.rs | 119 +++++++- rust/src/server/src/middleware/metrics.rs | 4 - rust/src/server/src/routes/sleep.rs | 59 +++- rust/src/server/src/routes/tests.rs | 284 +++++++++++++++++- tests/entrypoints/serve/dev/test_sleep.py | 184 +++++++++++- tests/v1/engine/test_wake_up_contract.py | 32 ++ vllm/engine/protocol.py | 4 +- vllm/entrypoints/llm.py | 7 +- .../entrypoints/serve/dev/sleep/api_router.py | 39 ++- vllm/entrypoints/serve/dev/sleep/metrics.py | 57 ++++ vllm/v1/engine/async_llm.py | 4 +- vllm/v1/engine/core_client.py | 2 +- vllm/v1/engine/llm_engine.py | 4 +- 14 files changed, 834 insertions(+), 48 deletions(-) create mode 100644 tests/v1/engine/test_wake_up_contract.py create mode 100644 vllm/entrypoints/serve/dev/sleep/metrics.py diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 5f2dc9c81e..0ac81764a9 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2290,6 +2290,89 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() { client.shutdown().await.unwrap(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn wake_up_requires_every_engine_to_be_fully_awake() { + init_tracing(); + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + let mut engines = Vec::new(); + for engine_index in 0..2 { + let engine = spawn_mock_engine_task( + handshake_address.clone(), + EngineId::from_engine_index(engine_index).into_frame().to_vec(), + move |dealer, push| { + Box::pin(async move { + let results = if engine_index == 0 { + [true, false, true] + } else { + [false, true, true] + }; + for (attempt, fully_awake) in results.into_iter().enumerate() { + let utility = recv_engine_message(dealer).await; + assert_eq!(utility[0].as_ref(), &[0x03]); + let payload = decode_value(&utility[1]); + let array = payload.as_array().expect("utility payload array"); + let call_id = array[1].as_u64().expect("call_id"); + assert_eq!(array[2], Value::from("wake_up")); + let tags = if attempt == 1 { + Value::Array(vec![Value::from("weights")]) + } else { + Value::Nil + }; + assert_eq!(array[3], Value::Array(vec![tags])); + send_outputs( + push, + UtilityCallOutput { + engine_index: u32::from(engine_index), + timestamp: 0.0, + output: UtilityOutput { + call_id: call_id.into(), + failure_message: None, + result: Some(utility_result_value(fully_awake)), + }, + } + .into(), + ) + .await; + } + }) + }, + ); + engines.push(engine); + } + let client = connect_client_with_ipc( + handshake_test_config( + handshake_address, + 2, + "test-model", + Duration::from_secs(2), + 5, + None, + ), + &ipc, + ) + .await; + + for (tags, expected) in [ + (None, false), + (Some(vec!["weights".to_owned()]), false), + (None, true), + ] { + assert_eq!( + timeout(Duration::from_secs(2), client.wake_up(tags)) + .await + .expect("wake timeout") + .expect("wake result"), + expected + ); + } + for (shutdown_tx, engine_task) in engines { + let _ = shutdown_tx.send(()); + engine_task.await.unwrap(); + } + client.shutdown().await.unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn collective_rpc_flattens_results_from_all_engines() { init_tracing(); diff --git a/rust/src/metrics/src/api_server.rs b/rust/src/metrics/src/api_server.rs index 855e9fbbb9..1dc7154998 100644 --- a/rust/src/metrics/src/api_server.rs +++ b/rust/src/metrics/src/api_server.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project +use std::time::Instant; + use prometheus_client::encoding::EncodeLabelSet; use prometheus_client::metrics::family::Family; use prometheus_client::registry::Registry; -use crate::{Histogram, U64Counter}; +use crate::{Histogram, U64Counter, U64Gauge}; const HTTP_REQUEST_DURATION_BUCKETS: [f64; 3] = [0.1, 0.5, 1.0]; const HTTP_REQUEST_DURATION_HIGHR_BUCKETS: [f64; 21] = [ @@ -21,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram { Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied()) } +fn sleep_mode_operation_duration_histogram() -> Histogram { + Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0]) +} + #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] pub struct HttpRequestLabels { pub method: String, @@ -34,15 +40,25 @@ pub struct HttpHandlerLabels { pub handler: String, } +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct SleepModeOperationLabels { + pub operation: &'static str, +} + pub(crate) type HttpRequestCounterFamily = Family; pub(crate) type HttpHandlerHistogramFamily = Family Histogram>; +pub(crate) type SleepModeOperationHistogramFamily = + Family Histogram>; +pub(crate) type SleepModeOperationGaugeFamily = Family; /// API-server Prometheus families exported from the HTTP middleware layer. pub struct ApiServerMetrics { pub http_requests: HttpRequestCounterFamily, pub http_request_duration_seconds: HttpHandlerHistogramFamily, pub http_request_duration_highr_seconds: Histogram, + pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily, + pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily, } impl ApiServerMetrics { @@ -71,10 +87,111 @@ impl ApiServerMetrics { http_request_duration_highr_seconds.clone(), ); + let sleep_mode_operation_duration_seconds = Family::new_with_constructor( + sleep_mode_operation_duration_histogram as fn() -> Histogram, + ); + registry.register( + "vllm:sleep_mode_operation_duration_seconds", + "Duration of one sleep-mode operation.", + sleep_mode_operation_duration_seconds.clone(), + ); + + let sleep_mode_operations_in_flight = SleepModeOperationGaugeFamily::default(); + registry.register( + "vllm:sleep_mode_operations_in_flight", + "Sleep-mode operations currently awaited.", + sleep_mode_operations_in_flight.clone(), + ); + Self { http_requests, http_request_duration_seconds, http_request_duration_highr_seconds, + sleep_mode_operation_duration_seconds, + sleep_mode_operations_in_flight, + } + } + + /// Record one dispatched sleep-mode engine operation. + pub fn record_sleep_mode_operation( + &self, + operation: &'static str, + ) -> SleepModeOperationRecorder { + let labels = SleepModeOperationLabels { operation }; + let in_flight = self.sleep_mode_operations_in_flight.get_or_create_owned(&labels); + in_flight.inc(); + SleepModeOperationRecorder { + started_at: Instant::now(), + duration: self.sleep_mode_operation_duration_seconds.get_or_create_owned(&labels), + in_flight, + } + } +} + +/// Records duration and clears in-flight tracking on completion or cancellation. +pub struct SleepModeOperationRecorder { + started_at: Instant, + duration: Histogram, + in_flight: U64Gauge, +} + +impl Drop for SleepModeOperationRecorder { + fn drop(&mut self) { + self.in_flight.dec(); + self.duration.observe(self.started_at.elapsed().as_secs_f64()); + } +} + +#[cfg(test)] +mod tests { + use prometheus_client::encoding::text::encode; + use prometheus_client::registry::Registry; + + use super::ApiServerMetrics; + + fn rendered_metrics(registry: &Registry) -> String { + let mut rendered = String::new(); + encode(&mut rendered, registry).expect("encode metrics"); + rendered + } + + #[test] + fn sleep_mode_recorder_tracks_duration_and_concurrency() { + let mut registry = Registry::default(); + let metrics = ApiServerMetrics::register(&mut registry); + for operation in ["sleep", "release_kv_cache_memory", "wake"] { + let recorder = metrics.record_sleep_mode_operation(operation); + let active = rendered_metrics(®istry); + assert!(active.contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1" + ))); + + drop(recorder); + let succeeded = rendered_metrics(®istry); + assert!(succeeded.contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0" + ))); + assert!(succeeded.contains(&format!( + "vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 1" + ))); + + let first = metrics.record_sleep_mode_operation(operation); + let second = metrics.record_sleep_mode_operation(operation); + assert!(rendered_metrics(®istry).contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 2" + ))); + drop(first); + assert!(rendered_metrics(®istry).contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 1" + ))); + drop(second); + let rendered = rendered_metrics(®istry); + assert!(rendered.contains(&format!( + "vllm:sleep_mode_operations_in_flight{{operation=\"{operation}\"}} 0" + ))); + assert!(rendered.contains(&format!( + "vllm:sleep_mode_operation_duration_seconds_count{{operation=\"{operation}\"}} 3" + ))); } } } diff --git a/rust/src/server/src/middleware/metrics.rs b/rust/src/server/src/middleware/metrics.rs index de07620cf7..46f10934ec 100644 --- a/rust/src/server/src/middleware/metrics.rs +++ b/rust/src/server/src/middleware/metrics.rs @@ -25,10 +25,6 @@ const EXCLUDED_HANDLERS: &[&str] = &[ "/reset_mm_cache", "/reset_encoder_cache", "/collective_rpc", - "/sleep", - "/release_kv_cache_memory", - "/wake_up", - "/is_sleeping", ]; /// Map the request method to a bounded Prometheus label. diff --git a/rust/src/server/src/routes/sleep.rs b/rust/src/server/src/routes/sleep.rs index d64a0850b6..b6239dc2d4 100644 --- a/rust/src/server/src/routes/sleep.rs +++ b/rust/src/server/src/routes/sleep.rs @@ -6,9 +6,9 @@ use std::sync::Arc; use axum::Json; use axum::extract::rejection::QueryRejection; use axum::extract::{Query, RawQuery, State}; -use axum::http::StatusCode; use serde::{Deserialize, Serialize}; use vllm_engine_core_client::protocol::utility::PauseMode; +use vllm_metrics::METRICS; use crate::error::ApiError; use crate::state::AppState; @@ -19,6 +19,24 @@ pub(crate) struct IsSleepingResponse { is_sleeping: bool, } +#[derive(Serialize)] +pub struct SleepResponse { + status: &'static str, + level: u32, +} + +#[derive(Serialize)] +pub struct ReleaseKvCacheMemoryResponse { + status: &'static str, +} + +#[derive(Serialize)] +pub struct WakeUpResponse { + status: &'static str, + /// Tags requested by the caller; null means wake all resources. + tags: Option>, +} + #[derive(Debug, Deserialize)] pub(crate) struct SleepParams { #[serde(default = "default_sleep_level")] @@ -32,43 +50,56 @@ const fn default_sleep_level() -> u32 { } fn invalid_query(error: QueryRejection) -> ApiError { - ApiError::invalid_request(error.body_text(), Some("mode")) + ApiError::invalid_request(error.body_text(), None) } /// Put the engine to sleep. pub async fn sleep( State(state): State>, params: Result, QueryRejection>, -) -> Result { - let Query(params) = params.map_err(invalid_query)?; +) -> Result, ApiError> { + let Query(SleepParams { level, mode }) = params.map_err(invalid_query)?; + if level > 2 { + return Err(ApiError::invalid_request( + "level must be between 0 and 2", + Some("level"), + )); + } + let _recorder = METRICS.api_server.record_sleep_mode_operation("sleep"); state .engine_core_client() - .sleep(params.level, params.mode) + .sleep(level, mode) .await .map_err(|error| utility_call_error("sleep", error))?; - Ok(StatusCode::OK) + Ok(Json(SleepResponse { + status: "sleeping", + level, + })) } /// Release KV cache memory while keeping model weights resident. pub async fn release_kv_cache_memory( State(state): State>, -) -> Result { +) -> Result, ApiError> { + let _recorder = METRICS.api_server.record_sleep_mode_operation("release_kv_cache_memory"); state .engine_core_client() .release_kv_cache_memory() .await .map_err(|error| utility_call_error("release_kv_cache_memory", error))?; - Ok(StatusCode::OK) + Ok(Json(ReleaseKvCacheMemoryResponse { + status: "kv_cache_released", + })) } /// Wake the engine from sleep mode. pub async fn wake_up( State(state): State>, RawQuery(query): RawQuery, -) -> Result { +) -> Result, ApiError> { let tags = query.and_then(|query| { let tags = url::form_urlencoded::parse(query.as_bytes()) .filter(|(key, _)| key == "tags") @@ -77,13 +108,17 @@ pub async fn wake_up( (!tags.is_empty()).then_some(tags) }); - state + let _recorder = METRICS.api_server.record_sleep_mode_operation("wake"); + let fully_awake = state .engine_core_client() - .wake_up(tags) + .wake_up(tags.clone()) .await .map_err(|error| utility_call_error("wake_up", error))?; - Ok(StatusCode::OK) + Ok(Json(WakeUpResponse { + status: if fully_awake { "awake" } else { "sleeping" }, + tags, + })) } /// Return whether the engine is currently sleeping at any level. diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index b016810b59..d229be25fa 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -5892,6 +5892,7 @@ async fn weight_checker_route_rejects_invalid_requests_before_engine_calls() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn sleep_route_uses_python_compatible_default_query_values() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5927,13 +5928,95 @@ async fn sleep_route_uses_python_compatible_default_query_values() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "sleeping", "level": 1 }) + ); engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"2xx\",handler=\"/sleep\""), + ), + 1.0 + ); + assert_eq!( + metric_delta( + &before, + &after, + "vllm:sleep_mode_operation_duration_seconds_count", + Some("operation=\"sleep\""), + ), + 1.0 + ); + assert_eq!( + metric_value( + &after, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"sleep\""), + ), + Some(0.0) + ); + assert_eq!( + metric_delta( + &before, + &after, + "http_request_duration_seconds_count", + Some("method=\"POST\",handler=\"/sleep\""), + ), + 1.0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn sleep_route_rejects_invalid_query_before_engine_dispatch() { + let before = METRICS.render().expect("render metrics"); + let (app, engine_task) = + test_admin_app_with_engine_script(|_dealer, _push| boxed_test_future(async move {})).await; + for query in ["level=invalid", "level=-1", "level=3", "mode=invalid"] { + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri(format!("/sleep?{query}")) + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{query}"); + } + engine_task.abort_and_join().await; + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"4xx\",handler=\"/sleep\""), + ), + 4.0 + ); + for metric in [ + "vllm:sleep_mode_operation_duration_seconds_count", + "vllm:sleep_mode_operations_in_flight", + ] { + assert_eq!( + metric_delta(&before, &after, metric, Some("operation=\"sleep\"")), + 0.0 + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn release_kv_cache_memory_route_sends_expected_utility_call() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -5966,13 +6049,177 @@ async fn release_kv_cache_memory_route_sends_expected_utility_call() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "kv_cache_released" }) + ); engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"2xx\",handler=\"/release_kv_cache_memory\""), + ), + 1.0 + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn release_kv_cache_memory_error_records_http_status_and_operation_duration() { + let before = METRICS.render().expect("render metrics"); + let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + let call_id = + payload.as_array().expect("utility payload array")[1].as_u64().expect("call id"); + send_outputs( + push, + UtilityCallOutput { + output: UtilityOutput { + call_id: call_id.into(), + failure_message: Some("requires a completed pause first".to_owned()), + result: None, + }, + ..Default::default() + } + .into(), + ) + .await; + }) + }) + .await; + let response = app + .clone() + .call( + Request::builder() + .method("POST") + .uri("/release_kv_cache_memory") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"5xx\",handler=\"/release_kv_cache_memory\""), + ), + 1.0 + ); + assert_eq!( + metric_delta( + &before, + &after, + "vllm:sleep_mode_operation_duration_seconds_count", + Some("operation=\"release_kv_cache_memory\""), + ), + 1.0 + ); + assert_eq!( + metric_value( + &after, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"release_kv_cache_memory\""), + ), + Some(0.0) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn cancelled_wake_operation_clears_in_flight_and_is_scraped() { + let before = METRICS.render().expect("render metrics"); + let (dispatched_tx, dispatched_rx) = tokio::sync::oneshot::channel(); + let (app, engine_task) = test_admin_app_with_engine_script(move |dealer, _push| { + boxed_test_future(async move { + let utility = recv_engine_message(dealer).await; + let payload = decode_value(&utility[1]).expect("decode utility payload"); + assert_eq!( + payload.as_array().expect("utility payload array")[2], + Value::from("wake_up") + ); + dispatched_tx.send(()).expect("notify dispatch"); + std::future::pending::<()>().await; + }) + }) + .await; + let mut request_app = app.clone(); + let request_task = tokio::spawn(async move { + request_app + .call( + Request::builder() + .method("POST") + .uri("/wake_up") + .body(Body::empty()) + .expect("build request"), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), dispatched_rx) + .await + .expect("wake dispatch timeout") + .expect("wake dispatched"); + let active = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &active, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"wake\""), + ), + 1.0 + ); + request_task.abort(); + assert!(request_task.await.expect_err("request cancelled").is_cancelled()); + engine_task.abort_and_join().await; + + let response = app + .clone() + .call( + Request::builder() + .uri("/metrics") + .body(Body::empty()) + .expect("build scrape request"), + ) + .await + .expect("scrape metrics"); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read metrics"); + let after = String::from_utf8(body.to_vec()).expect("utf8 metrics"); + assert_eq!( + metric_value( + &after, + "vllm:sleep_mode_operations_in_flight", + Some("operation=\"wake\""), + ), + Some(0.0) + ); + assert_eq!( + metric_delta( + &before, + &after, + "vllm:sleep_mode_operation_duration_seconds_count", + Some("operation=\"wake\""), + ), + 1.0 + ); + assert!(!after.contains("rl_sleep_mode")); + assert!(!after.contains("sleep_mode_operations_total")); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn wake_up_route_without_tags_accepts_bool_result() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -6005,8 +6252,21 @@ async fn wake_up_route_without_tags_accepts_bool_result() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "awake", "tags": null }) + ); engine_task.await.expect("mock engine task"); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"POST\",status=\"2xx\",handler=\"/wake_up\""), + ), + 1.0 + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -6030,7 +6290,7 @@ async fn wake_up_route_accepts_repeated_tags() { ])]) ); - send_outputs(push, utility_outputs(call_id, utility_result_value(true))).await; + send_outputs(push, utility_outputs(call_id, utility_result_value(false))).await; }) }) .await; @@ -6050,13 +6310,17 @@ async fn wake_up_route_accepts_repeated_tags() { let status = response.status(); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&body)); - assert!(body.is_empty()); + assert_eq!( + serde_json::from_slice::(&body).expect("decode json"), + json!({ "status": "sleeping", "tags": ["weights", "kv_cache"] }) + ); engine_task.await.expect("mock engine task"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn is_sleeping_route_returns_json_payload() { + let before = METRICS.render().expect("render metrics"); let (app, engine_task) = test_admin_app_with_engine_script(|dealer, push| { boxed_test_future(async move { let utility = recv_engine_message(dealer).await; @@ -6094,6 +6358,16 @@ async fn is_sleeping_route_returns_json_payload() { serde_json::from_slice::(&body).expect("decode json"), json!({ "is_sleeping": true }) ); + let after = METRICS.render().expect("render metrics"); + assert_eq!( + metric_delta( + &before, + &after, + "http_requests_total", + Some("method=\"GET\",status=\"2xx\",handler=\"/is_sleeping\""), + ), + 1.0 + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/tests/entrypoints/serve/dev/test_sleep.py b/tests/entrypoints/serve/dev/test_sleep.py index 34e69c9ece..1f91945327 100644 --- a/tests/entrypoints/serve/dev/test_sleep.py +++ b/tests/entrypoints/serve/dev/test_sleep.py @@ -1,35 +1,209 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +from types import SimpleNamespace from unittest.mock import AsyncMock import pytest import requests from fastapi import FastAPI from fastapi.testclient import TestClient +from prometheus_client import CollectorRegistry from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.serve.dev.sleep import metrics as sleep_metrics from vllm.entrypoints.serve.dev.sleep.api_router import attach_router +from vllm.entrypoints.serve.dev.sleep.metrics import SleepModeOperationMetrics +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler +from vllm.entrypoints.serve.instrumentator.metrics import ( + attach_router as attach_metrics_router, +) +from vllm.v1.metrics import prometheus as prometheus_metrics MODEL_NAME = "meta-llama/Llama-3.2-1B" +@pytest.fixture +def sleep_route_app(monkeypatch): + app = FastAPI() + app.state.args = SimpleNamespace(log_error_stack=False) + app.state.engine_client = AsyncMock() + metrics = SleepModeOperationMetrics(CollectorRegistry()) + monkeypatch.setattr( + "vllm.entrypoints.serve.dev.sleep.api_router.sleep_mode_operation_metrics", + lambda: metrics, + ) + + attach_router(app) + init_exception_handler(app) + return app, metrics + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("level", [0, 1, 2]) +def test_sleep_route_response_and_engine_arguments(sleep_route_app, level): + app, _ = sleep_route_app + with TestClient(app) as client: + response = client.post("/sleep", params={"level": level}) + assert response.status_code == 200 + assert response.json() == {"status": "sleeping", "level": level} + app.state.engine_client.sleep.assert_awaited_once_with(level, "abort") + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("query", "expected_param"), + [ + ("level=invalid", "query.level"), + ("level=-1", "query.level"), + ("level=3", "query.level"), + ("mode=invalid", "query.mode"), + ], +) +def test_sleep_route_rejects_invalid_query_before_dispatch( + sleep_route_app, query, expected_param +): + app, metrics = sleep_route_app + with TestClient(app) as client: + response = client.post(f"/sleep?{query}") + assert response.status_code == 400 + assert response.json()["error"]["param"] == expected_param + app.state.engine_client.sleep.assert_not_awaited() + assert list(metrics.duration.collect()[0].samples) == [] + assert list(metrics.in_flight.collect()[0].samples) == [] + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("fully_awake", "tags", "expected"), + [ + (True, "", {"status": "awake", "tags": None}), + (False, "?tags=weights", {"status": "sleeping", "tags": ["weights"]}), + ], +) +def test_wake_route_maps_engine_result_without_state_query( + sleep_route_app, fully_awake, tags, expected +): + app, _ = sleep_route_app + app.state.engine_client.wake_up.return_value = fully_awake + with TestClient(app) as client: + response = client.post(f"/wake_up{tags}") + assert response.status_code == 200 + assert response.json() == expected + app.state.engine_client.wake_up.assert_awaited_once_with(expected["tags"]) + app.state.engine_client.is_sleeping.assert_not_awaited() + + @pytest.mark.cpu_test @pytest.mark.parametrize("fails", [False, True]) -def test_release_kv_cache_memory_route(fails): - app = FastAPI() - app.state.engine_client = AsyncMock() +def test_release_kv_cache_memory_route(sleep_route_app, fails): + app, metrics = sleep_route_app release = app.state.engine_client.release_kv_cache_memory if fails: release.side_effect = RuntimeError("requires a completed pause first") - attach_router(app) - with TestClient(app, raise_server_exceptions=False) as client: response = client.post("/release_kv_cache_memory") assert response.status_code == (500 if fails else 200) + if not fails: + assert response.json() == {"status": "kv_cache_released"} release.assert_awaited_once_with() + assert metrics.in_flight.labels("release_kv_cache_memory")._value.get() == 0 + assert _duration_count(metrics, "release_kv_cache_memory") == 1 + + +@pytest.mark.cpu_test +@pytest.mark.parametrize("operation", ["sleep", "release_kv_cache_memory", "wake"]) +@pytest.mark.parametrize("exception", [RuntimeError, asyncio.CancelledError]) +def test_sleep_mode_recorder_tracks_duration_and_cleans_up(operation, exception): + metrics = SleepModeOperationMetrics(CollectorRegistry()) + with metrics.record(operation): + assert metrics.in_flight.labels(operation)._value.get() == 1 + with pytest.raises(exception), metrics.record(operation): + raise exception() + assert metrics.in_flight.labels(operation)._value.get() == 0 + assert metrics.duration.labels(operation)._sum.get() >= 0 + assert _duration_count(metrics, operation) == 2 + + +def _duration_count(metrics, operation): + return next( + sample.value + for sample in metrics.duration.collect()[0].samples + if sample.name.endswith("_count") and sample.labels == {"operation": operation} + ) + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + ("path", "method", "operation", "status", "fails"), + [ + ("/sleep", "POST", "sleep", 200, False), + ("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 200, False), + ("/wake_up", "POST", "wake", 200, False), + ("/is_sleeping", "GET", None, 200, False), + ("/sleep?level=3", "POST", None, 400, False), + ("/release_kv_cache_memory", "POST", "release_kv_cache_memory", 500, True), + ], +) +def test_sleep_routes_visible_on_production_metrics_endpoint( + monkeypatch, path, method, operation, status, fails +): + registry = CollectorRegistry() + monkeypatch.delenv("PROMETHEUS_MULTIPROC_DIR", raising=False) + monkeypatch.setattr(sleep_metrics, "REGISTRY", registry) + monkeypatch.setattr(sleep_metrics, "_metrics", None) + monkeypatch.setattr(prometheus_metrics, "REGISTRY", registry) + + app = FastAPI() + app.state.args = SimpleNamespace(log_error_stack=False) + app.state.engine_client = AsyncMock() + app.state.engine_client.wake_up.return_value = True + app.state.engine_client.is_sleeping.return_value = False + if fails: + app.state.engine_client.release_kv_cache_memory.side_effect = RuntimeError( + "engine failed" + ) + attach_router(app) + init_exception_handler(app) + attach_metrics_router(app) + + with TestClient(app, raise_server_exceptions=False) as client: + assert client.request(method, path).status_code == status + response = client.get("/metrics") + + assert response.status_code == 200 + samples = [ + sample + for family in text_string_to_metric_families(response.text) + for sample in family.samples + ] + assert any( + sample.name == "http_requests_total" + and sample.labels + == { + "handler": path.split("?")[0], + "method": method, + "status": f"{status // 100}xx", + } + and sample.value == 1 + for sample in samples + ) + assert not any("sleep_mode_operations_total" in sample.name for sample in samples) + assert any( + sample.name == "vllm:sleep_mode_operations_in_flight" + and sample.labels == {"operation": operation} + and sample.value == 0 + for sample in samples + ) == (operation is not None) + assert any( + sample.name == "vllm:sleep_mode_operation_duration_seconds_count" + and sample.labels == {"operation": operation} + and sample.value == 1 + for sample in samples + ) == (operation is not None) def test_sleep_mode(): diff --git a/tests/v1/engine/test_wake_up_contract.py b/tests/v1/engine/test_wake_up_contract.py new file mode 100644 index 0000000000..1369cece95 --- /dev/null +++ b/tests/v1/engine/test_wake_up_contract.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import AsyncMock, call + +import pytest + +from vllm.v1.engine.core_client import DPLBAsyncMPClient + +pytestmark = pytest.mark.cpu_test + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "results, expected", + [ + ([True, True], True), + ([True, False], False), + ([False, True], False), + ([False, False], False), + ], +) +async def test_dp_wake_up_requires_all_engines_awake(results, expected): + client = object.__new__(DPLBAsyncMPClient) + client.core_engines = [b"engine-0", b"engine-1"] + client._call_utility_async = AsyncMock(side_effect=results) + + assert await client.wake_up_async(["weights"]) is expected + assert client._call_utility_async.await_args_list == [ + call("wake_up", ["weights"], engine=b"engine-0"), + call("wake_up", ["weights"], engine=b"engine-1"), + ] diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 71a61b1648..a34981d651 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -199,8 +199,8 @@ class EngineClient(ABC): ... @abstractmethod - async def wake_up(self, tags: list[str] | None = None) -> None: - """Wake up the engine.""" + async def wake_up(self, tags: list[str] | None = None) -> bool: + """Wake the tagged resources; return whether every engine is fully awake.""" ... @abstractmethod diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 37348533b8..3de7bab133 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -839,7 +839,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): """ self.llm_engine.release_kv_cache_memory() - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] | None = None) -> bool: """Wake up the engine from sleep mode. See the [sleep][vllm.LLM.sleep] method for more details. @@ -851,8 +851,11 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): (or None) before the engine is used again. Use tags=["scheduling"] to resume from level 0 sleep. + Returns: + Whether the engine is fully awake. + """ - self.llm_engine.wake_up(tags) + return self.llm_engine.wake_up(tags) def get_metrics(self) -> list["Metric"]: """Return a snapshot of aggregated metrics from Prometheus. diff --git a/vllm/entrypoints/serve/dev/sleep/api_router.py b/vllm/entrypoints/serve/dev/sleep/api_router.py index 846a487706..eb2ad9d7d4 100644 --- a/vllm/entrypoints/serve/dev/sleep/api_router.py +++ b/vllm/entrypoints/serve/dev/sleep/api_router.py @@ -2,11 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from fastapi import APIRouter, FastAPI, Request -from fastapi.responses import JSONResponse, Response +from typing import Annotated + +from fastapi import APIRouter, FastAPI, Query, Request +from fastapi.responses import JSONResponse from vllm.engine.protocol import EngineClient from vllm.logger import init_logger +from vllm.v1.engine import PauseMode + +from .metrics import sleep_mode_operation_metrics logger = init_logger(__name__) @@ -19,29 +24,35 @@ router = APIRouter() @router.post("/sleep") -async def sleep(raw_request: Request): - # get POST params - level = raw_request.query_params.get("level", "1") - mode = raw_request.query_params.get("mode", "abort") - await engine_client(raw_request).sleep(int(level), mode) - return Response(status_code=200) +async def sleep( + raw_request: Request, + level: Annotated[int, Query(ge=0, le=2)] = 1, + mode: Annotated[PauseMode, Query()] = "abort", +) -> JSONResponse: + with sleep_mode_operation_metrics().record("sleep"): + await engine_client(raw_request).sleep(level, mode) + return JSONResponse(content={"status": "sleeping", "level": level}) @router.post("/release_kv_cache_memory") -async def release_kv_cache_memory(raw_request: Request): - await engine_client(raw_request).release_kv_cache_memory() - return Response(status_code=200) +async def release_kv_cache_memory(raw_request: Request) -> JSONResponse: + with sleep_mode_operation_metrics().record("release_kv_cache_memory"): + await engine_client(raw_request).release_kv_cache_memory() + return JSONResponse(content={"status": "kv_cache_released"}) @router.post("/wake_up") -async def wake_up(raw_request: Request): +async def wake_up(raw_request: Request) -> JSONResponse: tags = raw_request.query_params.getlist("tags") if tags == []: # set to None to wake up all tags if no tags are provided tags = None logger.info("wake up the engine with tags: %s", tags) - await engine_client(raw_request).wake_up(tags) - return Response(status_code=200) + with sleep_mode_operation_metrics().record("wake"): + fully_awake = await engine_client(raw_request).wake_up(tags) + return JSONResponse( + content={"status": "awake" if fully_awake else "sleeping", "tags": tags} + ) @router.get("/is_sleeping") diff --git a/vllm/entrypoints/serve/dev/sleep/metrics.py b/vllm/entrypoints/serve/dev/sleep/metrics.py new file mode 100644 index 0000000000..e1dbeaf175 --- /dev/null +++ b/vllm/entrypoints/serve/dev/sleep/metrics.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Metrics for dispatched sleep-mode frontend operations.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from threading import Lock +from time import perf_counter +from typing import Literal + +from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram + +Operation = Literal["sleep", "release_kv_cache_memory", "wake"] +BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600) + + +class SleepModeOperationMetrics: + def __init__(self, registry: CollectorRegistry | None = None): + registry = REGISTRY if registry is None else registry + self.duration = Histogram( + "vllm:sleep_mode_operation_duration_seconds", + "Duration of one sleep-mode operation.", + ["operation"], + buckets=BUCKETS, + registry=registry, + ) + self.in_flight = Gauge( + "vllm:sleep_mode_operations_in_flight", + "Sleep-mode operations currently awaited.", + ["operation"], + multiprocess_mode="livesum", + registry=registry, + ) + + @contextmanager + def record(self, operation: Operation) -> Iterator[None]: + active = self.in_flight.labels(operation) + started = perf_counter() + active.inc() + try: + yield + finally: + active.dec() + self.duration.labels(operation).observe(perf_counter() - started) + + +_metrics: SleepModeOperationMetrics | None = None +_metrics_lock = Lock() + + +def sleep_mode_operation_metrics() -> SleepModeOperationMetrics: + global _metrics + if _metrics is None: + with _metrics_lock: + if _metrics is None: + _metrics = SleepModeOperationMetrics() + return _metrics diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index ad9420247f..b1cc2edbc4 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -1110,12 +1110,14 @@ class AsyncLLM(EngineClient): if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, 0) - async def wake_up(self, tags: list[str] | None = None) -> None: + async def wake_up(self, tags: list[str] | None = None) -> bool: fully_awake = await self.engine_core.wake_up_async(tags) if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) + return fully_awake + async def checkpoint_prepare(self) -> None: await self.collective_rpc("checkpoint_prepare") diff --git a/vllm/v1/engine/core_client.py b/vllm/v1/engine/core_client.py index 9372c8a172..cb6705966a 100644 --- a/vllm/v1/engine/core_client.py +++ b/vllm/v1/engine/core_client.py @@ -1320,7 +1320,7 @@ class AsyncMPClient(MPClient): await self.call_utility_async("release_kv_cache_memory") async def wake_up_async(self, tags: list[str] | None = None) -> bool: - return await self.call_utility_async("wake_up", tags) + return all(await self.call_utility_all_async("wake_up", tags)) async def is_sleeping_async(self) -> bool: return await self.call_utility_consensus_async("is_sleeping") diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 73f6c18b34..d14b3b9755 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -386,12 +386,14 @@ class LLMEngine: if self.logger_manager is not None: self.logger_manager.record_sleep_state(1, 0) - def wake_up(self, tags: list[str] | None = None): + def wake_up(self, tags: list[str] | None = None) -> bool: fully_awake = self.engine_core.wake_up(tags) if self.logger_manager is not None and fully_awake: self.logger_manager.record_sleep_state(0, 0) + return fully_awake + def is_sleeping(self) -> bool: return self.engine_core.is_sleeping() From 77e8b02c8d69f25867227fb9883797996fbfcd7b Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:07:44 -0400 Subject: [PATCH 090/248] [ROCm][Perf] Add opt-in a4w4 (FP4 activation) MoE for DeepSeek V4.1 on AITER (#58819) Signed-off-by: fai Signed-off-by: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Co-authored-by: Cursor Agent (Claude Sonnet 5) --- tests/kernels/moe/test_rocm_aiter_moe.py | 109 ++++++++++++++++++ vllm/_aiter_ops.py | 25 ++++ vllm/envs.py | 8 ++ .../model_executor/layers/fused_moe/config.py | 35 ++++++ .../fused_moe/experts/rocm_aiter_moe.py | 21 ++-- .../layers/fused_moe/oracle/mxfp4.py | 15 ++- .../layers/quantization/mxfp4.py | 1 + .../layers/quantization/online/mxfp4.py | 1 + 8 files changed, 204 insertions(+), 11 deletions(-) diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index fbc8586fd3..05692852e1 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -638,6 +638,115 @@ def test_aiter_moe_padding_env_var( assert envs.VLLM_ROCM_MOE_PADDING is moe_padding +# a4w4 (FP4 activation) opt-in gating test --------------------------------- + + +@pytest.mark.parametrize( + ("env_value", "model_type", "expected"), + [ + # Unset/"0" is opt-in-only: a4w4 never turns on by itself, even on + # a validated model type. + (None, "deepseek_v41", False), + ("0", "deepseek_v41", False), + ("1", "deepseek_v41", True), + ("1", "deepseek_v41_text", True), + ], +) +def test_aiter_moe_a4w4_dsv4_is_opt_in( + env_value: str | None, + model_type: str, + expected: bool, + monkeypatch: pytest.MonkeyPatch, +): + """vllm-project/vllm#58819's a4w4 MoE activation path must stay a plain + explicit opt-in (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1), with no + default-on behavior, but is still only permitted on the model types + this flag has been validated for.""" + import types + + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm._aiter_ops import rocm_aiter_ops + + _assert_aiter_supported() + + fake_vllm_config = types.SimpleNamespace( + model_config=types.SimpleNamespace( + hf_config=types.SimpleNamespace(model_type=model_type) + ) + ) + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", "1") + mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") + if env_value is None: + mp.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False) + else: + mp.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", env_value) + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + with mock.patch( + "vllm.config.get_current_vllm_config_or_none", + return_value=fake_vllm_config, + ): + moe_config = make_dummy_moe_config( + num_experts=4, + experts_per_token=2, + hidden_dim=256, + intermediate_size=512, + in_dtype=torch.bfloat16, + ) + + assert moe_config.use_mxfp4_w4a4_dsv4 is expected + + +@pytest.mark.parametrize("model_type", ["deepseek_v4", "gpt_oss", None]) +def test_aiter_moe_a4w4_dsv4_rejects_unvalidated_model_type( + model_type: str | None, + monkeypatch: pytest.MonkeyPatch, +): + """Forcing a4w4 on a model type this flag hasn't been GSM8K/AgentX + validated for must raise: it also flips the MXFP4 weight-shuffle layout + (ATOM SEPARATED vs INTERLEAVE), which silently produces garbled output + on the wrong model rather than just being a slower path — the same bug + class vllm-project/vllm#58819's own second commit fixed for DeepSeek + V4.1 itself.""" + import types + + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm._aiter_ops import rocm_aiter_ops + + _assert_aiter_supported() + + fake_vllm_config = types.SimpleNamespace( + model_config=types.SimpleNamespace( + hf_config=types.SimpleNamespace(model_type=model_type) + ) + ) + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_USE_AITER", "1") + mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1") + mp.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1") + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + with ( + mock.patch( + "vllm.config.get_current_vllm_config_or_none", + return_value=fake_vllm_config, + ), + pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4"), + ): + make_dummy_moe_config( + num_experts=4, + experts_per_token=2, + hidden_dim=256, + intermediate_size=512, + in_dtype=torch.bfloat16, + ) + + # Dispatch-policy forwarding test ------------------------------------------ diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 906349af9a..f2d313bdd0 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -345,6 +345,7 @@ def _rocm_aiter_fused_moe_impl( shared_w1_scale: torch.Tensor | None = None, shared_w2_scale: torch.Tensor | None = None, shared_expert_id: int = -1, + q_dtype_a: torch.dtype | None = None, ) -> torch.Tensor: has_shared_expert = _validate_rocm_aiter_fused_moe_shared_expert_args( shared_w1, @@ -378,6 +379,12 @@ def _rocm_aiter_fused_moe_impl( shared_w2_scale=shared_w2_scale, shared_expert_id=shared_expert_id, ) + if q_dtype_a is not None: + # DeepSeek V4.1 a4w4 override (use_mxfp4_w4a4_dsv4 in + # rocm_aiter_moe.py). rocm_aiter_ops.fused_moe_supports_quant_dtype_a() + # is checked at config time, so this is only reached on an AITER + # build new enough to accept it. + extra_kwargs["quant_dtype_a"] = q_dtype_a return fused_moe( hidden_states, @@ -435,6 +442,7 @@ def _rocm_aiter_fused_moe_fake( shared_w1_scale: torch.Tensor | None = None, shared_w2_scale: torch.Tensor | None = None, shared_expert_id: int = -1, + q_dtype_a: torch.dtype | None = None, ) -> torch.Tensor: if output_dtype is not None: return torch.empty_like(hidden_states, dtype=output_dtype) @@ -2485,6 +2493,21 @@ class rocm_aiter_ops: return "gate_mode" in inspect.signature(fused_moe).parameters + @classmethod + @if_aiter_supported + @functools.cache + def fused_moe_supports_quant_dtype_a(cls) -> bool: + """Probe whether the installed aiter.fused_moe accepts `quant_dtype_a`. + + Added in https://github.com/ROCm/aiter/pull/5439 (unreleased at + merge time). Older AITER can't override the activation quant dtype. + """ + import inspect + + from aiter.fused_moe import fused_moe + + return "quant_dtype_a" in inspect.signature(fused_moe).parameters + @staticmethod def _probe_dsv4_i384_fhmoe_capability(num_tokens: int) -> bool: """Probe AITER's CSV-backed DSV4 native-I384 FHMoE contract.""" @@ -3005,6 +3028,7 @@ class rocm_aiter_ops: shared_w1_scale: torch.Tensor | None = None, shared_w2_scale: torch.Tensor | None = None, shared_expert_id: int = -1, + q_dtype_a: torch.dtype | None = None, ) -> torch.Tensor: return torch.ops.vllm.rocm_aiter_fused_moe( hidden_states, @@ -3036,6 +3060,7 @@ class rocm_aiter_ops: shared_w1_scale, shared_w2_scale, shared_expert_id, + q_dtype_a, ) @staticmethod diff --git a/vllm/envs.py b/vllm/envs.py index 428d6d3c94..67e0207492 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -141,6 +141,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False VLLM_ROCM_USE_AITER_MOE: bool = True VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 + VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4: bool | None = None VLLM_ROCM_USE_AITER_MOE_SITUV2: Literal["auto", "a4w4", "a8w4", "a16w4"] = "auto" VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True @@ -1291,6 +1292,13 @@ environment_variables: dict[str, Callable[[], Any]] = { ["auto", "a4w4", "a8w4", "a16w4", "0", "1"], case_sensitive=False, ), + # Opt-in switch for a4w4 (FP4 activation) MoE on DeepSeek V4.1, AITER + # MXFP4 backend. Default is a8w4 (FP8); set to "1" to enable a4w4 + # ("true" is not accepted -- only "0"/"1"). Raises if set for other + # models. + "VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4": lambda: maybe_convert_bool( + os.getenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4") + ), # MoE sorting dispatch policy for AITER fused MoE kernels. # 0 = auto (default): single-pass for small batches, multi-pass # for large batches diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 11e2a57520..302ed121fe 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -6,6 +6,7 @@ from typing import Union import torch +import vllm.envs as envs from vllm.config import ParallelConfig, SchedulerConfig from vllm.config.kernel import MoEBackend from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank @@ -1247,6 +1248,12 @@ class FusedMoEParallelConfig: ) +# Model types validated for VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1. This flag +# also changes the MXFP4 weight shuffle layout, so using it on other models +# can produce garbled output. +_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES = ("deepseek_v41", "deepseek_v41_text") + + # Adapted from pplx-kernels tests/all_to_all_utils.py @dataclass class FusedMoEConfig: @@ -1314,6 +1321,11 @@ class FusedMoEConfig: tp_shard_with_padding: bool = False rocm_aiter_fmoe_enabled: bool = False aiter_fmoe_shared_expert_enabled: bool = False + # Whether to force MXFP4 (a4w4) MoE activations for DeepSeek V4.1 on + # ROCm/AITER. Opt-in via VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1; rejected + # for any other model type. Resolved here, not in the forward path, + # because get_current_vllm_config() isn't set there. + use_mxfp4_w4a4_dsv4: bool = False def __post_init__(self): from vllm._aiter_ops import rocm_aiter_ops @@ -1345,6 +1357,29 @@ class FusedMoEConfig: rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() ) + if self.rocm_aiter_fmoe_enabled and envs.VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4: + from vllm.config import get_current_vllm_config_or_none + + vllm_config = get_current_vllm_config_or_none() + model_type = ( + getattr(vllm_config.model_config.hf_config, "model_type", None) + if vllm_config is not None + else None + ) + if model_type not in _AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES: + raise ValueError( + f"VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 only supports " + f"model_type in {_AITER_MOE_A4W4_DSV4_VALIDATED_MODEL_TYPES}, " + f"got {model_type!r}. Unset this env var for this model." + ) + if not rocm_aiter_ops.fused_moe_supports_quant_dtype_a(): + raise ValueError( + "VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1 needs an AITER build " + "with fused_moe(quant_dtype_a=...) support " + "(ROCm/aiter#5439+). Upgrade AITER or unset this env var." + ) + self.use_mxfp4_w4a4_dsv4 = True + if self.use_mori_kernels: assert self.rocm_aiter_fmoe_enabled, ( "Mori needs to be used with aiter fused_moe for now." diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 4f2f760ad3..07eddbf4a0 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -375,15 +375,13 @@ def rocm_aiter_fused_experts( intermediate_pad // 64 * 64 * (2 if moe_config.tp_size == 1 else 1) ) - # https://github.com/ROCm/aiter/pull/3123 specialized the AITER stage1 GEMMs - # for interleaved vs separated gate and up weights. - # For gpt-oss i.e. use_mxfp4_w4a16=True, the weights are shuffled by - # `rocm_aiter_ops.shuffle_weight_a16w4` in `oracle/mxfp4.py`, - # which always sets `is_guinterleave=True`. - # Hence, we pass in GateMode.INTERLEAVE to match the weight shuffling. + # AITER's stage1 GEMM needs gate_mode to match how weights were + # shuffled at load time (oracle/mxfp4.py). gpt-oss uses INTERLEAVE; + # DeepSeek V4.1 a4w4 uses SEPARATED (see below). from aiter.ops.flydsl.moe_common import GateMode gate_mode = "" + q_dtype_a = None if activation == MoEActivation.SITU: gate_mode = ( GateMode.INTERLEAVE.value @@ -391,7 +389,15 @@ def rocm_aiter_fused_experts( else GateMode.SEPARATED.value ) elif quant_config.use_mxfp4_w4a16: - gate_mode = GateMode.INTERLEAVE.value + if moe_config.use_mxfp4_w4a4_dsv4: + # Opt-in a4w4 for DeepSeek V4.1 (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1). + # Must match the SEPARATED weight shuffle from load time. + from aiter import dtypes + + gate_mode = GateMode.SEPARATED.value + q_dtype_a = dtypes.fp4x2 + else: + gate_mode = GateMode.INTERLEAVE.value elif activation_interleave is not None: gate_mode = ( GateMode.INTERLEAVE.value @@ -433,6 +439,7 @@ def rocm_aiter_fused_experts( shared_w1_scale=shared_w1_scale, shared_w2_scale=shared_w2_scale, shared_expert_id=shared_expert_id, + q_dtype_a=q_dtype_a, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 56ba3981b7..27d59ba2e0 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -1382,6 +1382,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( w2_bias: torch.Tensor | None = None, _cache_permute_indices: dict[torch.Size, torch.Tensor] | None = None, activation: MoEActivation | None = None, + use_separated_a4w4: bool = False, ) -> tuple[ torch.Tensor, torch.Tensor, @@ -1626,10 +1627,16 @@ def convert_weight_to_mxfp4_moe_kernel_format( from aiter.ops.shuffle import shuffle_scale as _shuf_s from aiter.ops.shuffle import shuffle_weight as _shuf_w + # DeepSeek V4.1 a4w4 uses ATOM's SEPARATED gate/up layout instead of + # the default INTERLEAVE shuffle (INTERLEAVE + fp4x2 has no tuned + # kernel and produces garbage output). Must match GateMode.SEPARATED + # in rocm_aiter_moe.py. + is_guinterleave = not use_separated_a4w4 + w13_weight = torch.nn.Parameter( _shuf_w( w13_weight.data.view(torch.float4_e2m1fn_x2), - is_guinterleave=True, + is_guinterleave=is_guinterleave, gate_up=True, ), requires_grad=False, @@ -1637,14 +1644,14 @@ def convert_weight_to_mxfp4_moe_kernel_format( shuffled_w13_scale = _shuf_s( w13_weight_scale.reshape(-1, w13_weight_scale.shape[-1]), num_experts, - True, + is_guinterleave, True, ) w2_weight = torch.nn.Parameter( _shuf_w( w2_weight.data.view(torch.float4_e2m1fn_x2), - is_guinterleave=True, + is_guinterleave=is_guinterleave, gate_up=False, ), requires_grad=False, @@ -1653,7 +1660,7 @@ def convert_weight_to_mxfp4_moe_kernel_format( shuffled_w2_scale = _shuf_s( w2_weight_scale.reshape(-1, w2_weight_scale.shape[-1]), num_experts, - True, + is_guinterleave, False, ) diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index dcba98c7e4..f9c7f37ff3 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -736,6 +736,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): w2_bias=w2_bias, _cache_permute_indices=self._cache_permute_indices, activation=self.moe.activation, + use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4, ) ) diff --git a/vllm/model_executor/layers/quantization/online/mxfp4.py b/vllm/model_executor/layers/quantization/online/mxfp4.py index 0115988e92..72914a9726 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp4.py +++ b/vllm/model_executor/layers/quantization/online/mxfp4.py @@ -220,6 +220,7 @@ class Mxfp4OnlineMoEMethod(OnlineMoEMethodBase): w2_weight_scale=w2_scale, w13_bias=w13_bias, w2_bias=w2_bias, + use_separated_a4w4=self.moe.use_mxfp4_w4a4_dsv4, ) ) From 583637f126a5f5193952a81ca9e0630ae9fc9986 Mon Sep 17 00:00:00 2001 From: Rehan Khan Date: Wed, 30 Sep 2026 12:46:28 +0530 Subject: [PATCH 091/248] [CPU][Perf] Add vectorized Sampler Kernel (#53913) Signed-off-by: Rehan Khan Co-authored-by: Li, Jiang --- benchmarks/kernels/bench_cpu_sampling.py | 217 +++++++++++++++++++++++ cmake/cpu_extension.cmake | 3 + csrc/cpu/sampling_kernels.cpp | 151 ++++++++++++++++ csrc/cpu/torch_bindings.cpp | 12 ++ tests/kernels/test_cpu_fused_sampling.py | 120 +++++++++++++ vllm/v1/sample/ops/topk_topp_sampler.py | 33 ++-- vllm/v1/sample/sampler.py | 3 + 7 files changed, 528 insertions(+), 11 deletions(-) create mode 100644 benchmarks/kernels/bench_cpu_sampling.py create mode 100644 csrc/cpu/sampling_kernels.cpp create mode 100644 tests/kernels/test_cpu_fused_sampling.py diff --git a/benchmarks/kernels/bench_cpu_sampling.py b/benchmarks/kernels/bench_cpu_sampling.py new file mode 100644 index 0000000000..fb7fdeb3dd --- /dev/null +++ b/benchmarks/kernels/bench_cpu_sampling.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Micro-benchmark for CPU sampling kernels. + +Compares fused Gumbel-max / greedy argmax against the baseline +(softmax → exponential → div → argmax) across vocab and batch sizes. + +Usage: + .venv/bin/python benchmarks/kernels/bench_cpu_sampling.py + .venv/bin/python benchmarks/kernels/bench_cpu_sampling.py --profile + .venv/bin/python benchmarks/kernels/bench_cpu_sampling.py --vocab 128256 --batch 16 +""" + +import argparse +import time + +import torch +import vllm._C # noqa: F401 +from torch.profiler import ProfilerActivity, profile, record_function + + +def baseline_random_sample(logits: torch.Tensor) -> torch.Tensor: + probs = logits.softmax(dim=-1, dtype=torch.float32) + q = torch.empty_like(probs) + q.exponential_() + return probs.div(q).argmax(dim=-1).view(-1) + + +def baseline_greedy_sample(logits: torch.Tensor) -> torch.Tensor: + return logits.argmax(dim=-1).view(-1) + + +def bench_latency(fn, args, n_warmup=20, n_iters=500): + for _ in range(n_warmup): + fn(*args) + + t0 = time.perf_counter() + for _ in range(n_iters): + fn(*args) + elapsed = time.perf_counter() - t0 + return elapsed / n_iters * 1e6 # µs + + +def run_profile(logits, seeds, n_iters=50): + """Run torch.profiler and print comparison tables.""" + # Profile baseline random sampling + with profile( + activities=[ProfilerActivity.CPU], + record_shapes=True, + ) as prof_base_rand: + for _ in range(n_iters): + with record_function("baseline_random"): + baseline_random_sample(logits) + + # Profile fused Gumbel-max + with profile( + activities=[ProfilerActivity.CPU], + record_shapes=True, + ) as prof_fused: + for _ in range(n_iters): + with record_function("fused_gumbel_argmax"): + torch.ops._C.fused_gumbel_argmax(logits, seeds) + + # Profile baseline greedy + with profile( + activities=[ProfilerActivity.CPU], + record_shapes=True, + ) as prof_base_grdy: + for _ in range(n_iters): + with record_function("baseline_greedy"): + baseline_greedy_sample(logits) + + # Profile custom greedy + with profile( + activities=[ProfilerActivity.CPU], + record_shapes=True, + ) as prof_cust_grdy: + for _ in range(n_iters): + with record_function("custom_greedy_argmax"): + torch.ops._C.greedy_argmax(logits) + + B, V = logits.shape + print(f"\n{'=' * 80}") + print(f"torch.profiler breakdown (batch={B}, vocab={V}, iters={n_iters})") + print(f"{'=' * 80}") + + print("\n--- Baseline Random (softmax → exp → div → argmax) ---") + print(prof_base_rand.key_averages().table(sort_by="cpu_time_total", row_limit=15)) + + print("\n--- Fused Gumbel-max (table lookup + add + argmax) ---") + print(prof_fused.key_averages().table(sort_by="cpu_time_total", row_limit=15)) + + print("\n--- Baseline Greedy (torch.argmax) ---") + print(prof_base_grdy.key_averages().table(sort_by="cpu_time_total", row_limit=15)) + + print("\n--- Custom Greedy (vec_op argmax) ---") + print(prof_cust_grdy.key_averages().table(sort_by="cpu_time_total", row_limit=15)) + + # Summary comparison + def avg_us(prof, label): + for e in prof.key_averages(): + if e.key == label: + return e.cpu_time_total / e.count + return 0.0 + + t_br = avg_us(prof_base_rand, "baseline_random") + t_fg = avg_us(prof_fused, "fused_gumbel_argmax") + t_bg = avg_us(prof_base_grdy, "baseline_greedy") + t_cg = avg_us(prof_cust_grdy, "custom_greedy_argmax") + + print(f"\n{'=' * 60}") + print(f" Summary (batch={B}, vocab={V})") + print(f"{'=' * 60}") + print(f" {'Kernel':<30} {'Avg (µs)':>10} {'Speedup':>10}") + print(f" {'-' * 50}") + print(f" {'baseline random':<30} {t_br:>10.1f} {'—':>10}") + print( + f" {'fused gumbel-max':<30} {t_fg:>10.1f} " + f"{t_br / t_fg if t_fg > 0 else 0:>9.2f}x" + ) + print(f" {'baseline greedy':<30} {t_bg:>10.1f} {'—':>10}") + print( + f" {'custom greedy':<30} {t_cg:>10.1f} {t_bg / t_cg if t_cg > 0 else 0:>9.2f}x" + ) + print(f"{'=' * 60}") + + +def main(): + parser = argparse.ArgumentParser(description="Benchmark CPU sampling kernels") + parser.add_argument( + "--vocab", + type=int, + nargs="+", + default=[32000, 49152, 128256], + help="Vocab sizes to benchmark", + ) + parser.add_argument( + "--batch", + type=int, + nargs="+", + default=[1, 4, 16], + help="Batch sizes to benchmark", + ) + parser.add_argument( + "--profile", + action="store_true", + help="Run torch.profiler and export chrome trace", + ) + parser.add_argument( + "--iters", type=int, default=500, help="Iterations per measurement" + ) + args = parser.parse_args() + + header = ( + f"{'batch':>5} {'vocab':>7} " + f"{'base_rand':>10} {'fused_rand':>10} {'rand_spdup':>10} " + f"{'base_grdy':>10} {'cust_grdy':>10} {'grdy_spdup':>10}" + ) + units = ( + f"{'':>5} {'':>7} " + f"{'(µs)':>10} {'(µs)':>10} {'':>10} " + f"{'(µs)':>10} {'(µs)':>10} {'':>10}" + ) + print("\n" + "=" * len(header)) + print("CPU Sampling Kernel Benchmark") + print("=" * len(header)) + print(header) + print(units) + print("-" * len(header)) + + for V in args.vocab: + for B in args.batch: + logits = torch.randn(B, V, dtype=torch.float32) + seeds = torch.arange(B, dtype=torch.long) + + t_base_rand = bench_latency( + baseline_random_sample, (logits,), n_iters=args.iters + ) + t_fused = bench_latency( + torch.ops._C.fused_gumbel_argmax, (logits, seeds), n_iters=args.iters + ) + + t_base_grdy = bench_latency( + baseline_greedy_sample, (logits,), n_iters=args.iters + ) + t_cust_grdy = bench_latency( + torch.ops._C.greedy_argmax, (logits,), n_iters=args.iters + ) + + rand_speedup = t_base_rand / t_fused if t_fused > 0 else 0 + grdy_speedup = t_base_grdy / t_cust_grdy if t_cust_grdy > 0 else 0 + + print( + f"{B:>5} {V:>7} " + f"{t_base_rand:>10.1f} {t_fused:>10.1f} " + f"{rand_speedup:>9.2f}x " + f"{t_base_grdy:>10.1f} {t_cust_grdy:>10.1f} " + f"{grdy_speedup:>9.2f}x" + ) + + print("-" * len(header)) + + if args.profile: + print("\nRunning torch.profiler (batch=16, vocab=128256) ...") + logits = torch.randn(16, 128256, dtype=torch.float32) + seeds = torch.arange(16, dtype=torch.long) + + # warmup + for _ in range(10): + baseline_random_sample(logits) + torch.ops._C.fused_gumbel_argmax(logits, seeds) + + run_profile(logits, seeds) + + +if __name__ == "__main__": + main() diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 414fbc5e23..94dd188822 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -501,6 +501,7 @@ set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe.cpp" "csrc/cpu/cpu_attn.cpp" "csrc/cpu/cpu_isa.cpp" + "csrc/cpu/sampling_kernels.cpp" "csrc/cpu/torch_bindings.cpp") if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64" AND VLLM_RVV_VLEN AND @@ -589,6 +590,7 @@ if (ENABLE_X86_ISA) "csrc/cpu/cpu_isa.cpp" "csrc/cpu/dnnl_kernels.cpp" "csrc/cpu/mamba_cpu.cpp" + "csrc/cpu/sampling_kernels.cpp" "csrc/cpu/torch_bindings.cpp" # TODO: Remove these files "csrc/cpu/activation.cpp" @@ -606,6 +608,7 @@ if (ENABLE_X86_ISA) "csrc/cpu/cpu_isa.cpp" "csrc/cpu/mamba_cpu.cpp" "csrc/cpu/dnnl_kernels.cpp" + "csrc/cpu/sampling_kernels.cpp" "csrc/cpu/torch_bindings.cpp" # TODO: Remove these files "csrc/cpu/activation.cpp" diff --git a/csrc/cpu/sampling_kernels.cpp b/csrc/cpu/sampling_kernels.cpp new file mode 100644 index 0000000000..3e2f00a0ba --- /dev/null +++ b/csrc/cpu/sampling_kernels.cpp @@ -0,0 +1,151 @@ +#include "cpu_types.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace { + +constexpr int GUMBEL_TABLE_SIZE = 1 << 20; +constexpr int GUMBEL_TABLE_MASK = GUMBEL_TABLE_SIZE - 1; + +static float g_gumbel_table[GUMBEL_TABLE_SIZE]; +static std::once_flag g_gumbel_init_flag; + +static void init_gumbel_table() { + std::mt19937 rng(42); + std::uniform_real_distribution uniform(1.0e-7f, 1.0f - 1.0e-7f); + for (int i = 0; i < GUMBEL_TABLE_SIZE; ++i) { + float u = uniform(rng); + g_gumbel_table[i] = -std::log(-std::log(u)); + } +} + +static inline void ensure_gumbel_table() { + std::call_once(g_gumbel_init_flag, init_gumbel_table); +} + +// Fused Gumbel-max kernel + +static void fused_gumbel_argmax_kernel(int64_t* __restrict__ output, + const float* __restrict__ logits, + const int64_t* __restrict__ seeds, + const int64_t batch_size, + const int64_t vocab_size) { + ensure_gumbel_table(); + constexpr int VEC_ELEM_NUM = vec_op::FP32Vec8::VEC_ELEM_NUM; + const int64_t vec_end = vocab_size - (vocab_size % VEC_ELEM_NUM); + +#pragma omp parallel for schedule(static) + for (int64_t b = 0; b < batch_size; ++b) { + const float* row = logits + b * vocab_size; + const uint32_t seed = static_cast(seeds[b]); + + float best_score = -std::numeric_limits::infinity(); + int64_t best_idx = 0; + + for (int64_t i = 0; i < vec_end; i += VEC_ELEM_NUM) { + vec_op::FP32Vec8 logit_vec(row + i); + + float gumbel_buf[VEC_ELEM_NUM]; + for (int k = 0; k < VEC_ELEM_NUM; ++k) + gumbel_buf[k] = g_gumbel_table[(seed + i + k) & GUMBEL_TABLE_MASK]; + vec_op::FP32Vec8 gumbel_vec(gumbel_buf); + + vec_op::FP32Vec8 score = logit_vec + gumbel_vec; + + float score_buf[VEC_ELEM_NUM]; + score.save(score_buf); + for (int k = 0; k < VEC_ELEM_NUM; ++k) { + if (score_buf[k] > best_score) { + best_score = score_buf[k]; + best_idx = i + k; + } + } + } + + for (int64_t i = vec_end; i < vocab_size; ++i) { + float s = row[i] + g_gumbel_table[(seed + i) & GUMBEL_TABLE_MASK]; + if (s > best_score) { + best_score = s; + best_idx = i; + } + } + + output[b] = best_idx; + } +} + +static void greedy_argmax_kernel(int64_t* __restrict__ output, + const float* __restrict__ logits, + const int64_t batch_size, + const int64_t vocab_size) { + constexpr int VEC_ELEM_NUM = vec_op::FP32Vec16::VEC_ELEM_NUM; + const int64_t vec_end = vocab_size - (vocab_size % VEC_ELEM_NUM); + +#pragma omp parallel for schedule(static) + for (int64_t b = 0; b < batch_size; ++b) { + const float* row = logits + b * vocab_size; + + vec_op::FP32Vec16 vmax(-std::numeric_limits::infinity()); + for (int64_t i = 0; i < vec_end; i += VEC_ELEM_NUM) { + vmax = vmax.max(vec_op::FP32Vec16(row + i)); + } + float best_val = vmax.reduce_max(); + for (int64_t i = vec_end; i < vocab_size; ++i) { + if (row[i] > best_val) { + best_val = row[i]; + } + } + + int64_t best_idx = 0; + for (int64_t i = 0; i < vocab_size; ++i) { + if (row[i] == best_val) { + best_idx = i; + break; + } + } + output[b] = best_idx; + } +} + +} // namespace + +torch::Tensor fused_gumbel_argmax(const torch::Tensor& logits, + const torch::Tensor& seeds) { + TORCH_CHECK(logits.dim() == 2, "logits must be 2-D [batch, vocab]"); + TORCH_CHECK(logits.scalar_type() == torch::kFloat32, + "logits must be float32"); + TORCH_CHECK(seeds.dim() == 1 && seeds.size(0) == logits.size(0), + "seeds must be 1-D with batch_size elements"); + + auto logits_contig = logits.contiguous(); + auto output = torch::empty({logits_contig.size(0)}, torch::kInt64); + fused_gumbel_argmax_kernel( + output.data_ptr(), logits_contig.data_ptr(), + seeds.data_ptr(), logits_contig.size(0), logits_contig.size(1)); + return output; +} + +torch::Tensor greedy_argmax(const torch::Tensor& logits) { + TORCH_CHECK(logits.dim() == 2, "logits must be 2-D [batch, vocab]"); + TORCH_CHECK(logits.scalar_type() == torch::kFloat32, + "logits must be float32"); + + // A single row is faster in ATen: OpenMP fork/join dominates the scan + // (~1 ms vs ~50 µs on s390x at vocab=32k). + if (logits.size(0) <= 1) { + return logits.argmax(/*dim=*/-1); + } + + auto logits_contig = logits.contiguous(); + auto output = torch::empty({logits_contig.size(0)}, torch::kInt64); + greedy_argmax_kernel(output.data_ptr(), + logits_contig.data_ptr(), logits_contig.size(0), + logits_contig.size(1)); + return output; +} diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index accd5fcec3..918daf5a8b 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -393,6 +393,11 @@ void mamba_chunk_scan_fwd_cpu_impl(at::Tensor& out, at::Tensor& final_states, const c10::optional& z, const at::Tensor& cu_seqlens); +torch::Tensor fused_gumbel_argmax(const torch::Tensor& logits, + const torch::Tensor& seeds); + +torch::Tensor greedy_argmax(const torch::Tensor& logits); + void init_cpu_memory_env(std::vector node_ids); namespace cpu_utils { @@ -969,6 +974,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def("init_cpu_memory_env(SymInt[] node_ids) -> ()", &init_cpu_memory_env); + // Fused sampling kernels + ops.def("fused_gumbel_argmax(Tensor logits, Tensor seeds) -> Tensor"); + ops.impl("fused_gumbel_argmax", torch::kCPU, &fused_gumbel_argmax); + + ops.def("greedy_argmax(Tensor logits) -> Tensor"); + ops.impl("greedy_argmax", torch::kCPU, &greedy_argmax); + // Speculative decoding kernels ops.def( "eagle_prepare_inputs_padded_kernel_impl(Tensor cu_num_draft_tokens, " diff --git a/tests/kernels/test_cpu_fused_sampling.py b/tests/kernels/test_cpu_fused_sampling.py new file mode 100644 index 0000000000..8a3857fe9e --- /dev/null +++ b/tests/kernels/test_cpu_fused_sampling.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import tracemalloc + +import pytest +import torch + +from vllm.platforms import current_platform + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +import vllm._C # noqa: F401, E402 + +VOCAB_SIZES = [32000, 49152, 128256] +BATCH_SIZES = [1, 4, 16] + + +def reference_greedy_sample(logits: torch.Tensor) -> torch.Tensor: + return logits.argmax(dim=-1).view(-1) + + +class TestGreedyArgmax: + @pytest.mark.parametrize("vocab_size", VOCAB_SIZES) + @pytest.mark.parametrize("batch_size", BATCH_SIZES) + def test_exact_match(self, vocab_size: int, batch_size: int): + logits = torch.randn(batch_size, vocab_size, dtype=torch.float32) + expected = reference_greedy_sample(logits) + result = torch.ops._C.greedy_argmax(logits) + torch.testing.assert_close(result, expected) + + def test_single_dominant(self): + logits = torch.full((1, 50000), -1e9, dtype=torch.float32) + logits[0, 42] = 100.0 + assert torch.ops._C.greedy_argmax(logits).item() == 42 + + def test_negative_logits(self): + logits = torch.randn(8, 32000, dtype=torch.float32) - 10.0 + expected = reference_greedy_sample(logits) + result = torch.ops._C.greedy_argmax(logits) + torch.testing.assert_close(result, expected) + + +class TestFusedGumbelArgmax: + @pytest.mark.parametrize("vocab_size", VOCAB_SIZES) + def test_distribution_chi_squared(self, vocab_size: int): + """Verify sampling distribution via chi-squared goodness of fit.""" + small_vocab = min(vocab_size, 100) + logits = torch.randn(1, small_vocab, dtype=torch.float32) + probs = logits.softmax(dim=-1).squeeze(0) + + n_samples = 100_000 + counts = torch.zeros(small_vocab) + for trial in range(n_samples): + seed = torch.tensor([trial * 7 + 13], dtype=torch.long) + tile = logits.expand(1, -1).contiguous() + idx = torch.ops._C.fused_gumbel_argmax(tile, seed) + counts[idx.item()] += 1 + + expected = probs * n_samples + mask = expected > 5 + chi2 = ((counts[mask] - expected[mask]) ** 2 / expected[mask]).sum() + dof = mask.sum().item() - 1 + from scipy.stats import chi2 as chi2_dist + + p_value = 1.0 - chi2_dist.cdf(chi2.item(), dof) + assert p_value > 0.001, ( + f"Chi-squared test failed: chi2={chi2.item():.1f}, " + f"dof={dof}, p={p_value:.6f}" + ) + + def test_deterministic_same_seed(self): + """Same seed produces same result.""" + logits = torch.randn(4, 32000, dtype=torch.float32) + seeds = torch.tensor([42, 123, 456, 789], dtype=torch.long) + r1 = torch.ops._C.fused_gumbel_argmax(logits, seeds) + r2 = torch.ops._C.fused_gumbel_argmax(logits, seeds) + torch.testing.assert_close(r1, r2) + + def test_different_seeds_differ(self): + logits = torch.zeros(16, 50000, dtype=torch.float32) + seeds_a = torch.arange(16, dtype=torch.long) + seeds_b = torch.arange(16, dtype=torch.long) + 1_000_000 + r_a = torch.ops._C.fused_gumbel_argmax(logits, seeds_a) + r_b = torch.ops._C.fused_gumbel_argmax(logits, seeds_b) + assert not torch.equal(r_a, r_b) + + @pytest.mark.parametrize("vocab_size", VOCAB_SIZES) + @pytest.mark.parametrize("batch_size", BATCH_SIZES) + def test_output_in_range(self, vocab_size: int, batch_size: int): + logits = torch.randn(batch_size, vocab_size, dtype=torch.float32) + seeds = torch.arange(batch_size, dtype=torch.long) + result = torch.ops._C.fused_gumbel_argmax(logits, seeds) + assert result.min() >= 0 + assert result.max() < vocab_size + + +class TestMemory: + def test_fused_no_intermediate_allocs(self): + """Verify that the fused kernel does not allocate large intermediates.""" + logits = torch.randn(16, 128256, dtype=torch.float32) + seeds = torch.arange(16, dtype=torch.long) + + torch.ops._C.fused_gumbel_argmax(logits, seeds) + + tracemalloc.start() + snap_before = tracemalloc.take_snapshot() + for _ in range(50): + torch.ops._C.fused_gumbel_argmax(logits, seeds) + snap_after = tracemalloc.take_snapshot() + tracemalloc.stop() + + diff = snap_after.compare_to(snap_before, "lineno") + total_new_bytes = sum(s.size_diff for s in diff if s.size_diff > 0) + vocab_bytes = 16 * 128256 * 4 + assert total_new_bytes < vocab_bytes, ( + f"Fused kernel allocated {total_new_bytes} bytes, " + f"expected < {vocab_bytes} (one intermediate tensor)" + ) diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index 69ec368309..a3084a09c2 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -319,9 +319,11 @@ class TopKTopPSampler(nn.Module): k: torch.Tensor | None, p: torch.Tensor | None, ) -> tuple[torch.Tensor, torch.Tensor | None]: - """PyTorch-native implementation of top-k and top-p sampling for CPU. + """Fused Gumbel-max sampling for CPU. - The logits tensor may be updated in-place. + Uses a precomputed Gumbel table + single-pass argmax over logits, + skipping softmax and intermediate allocations entirely. + Falls back to the native path when fp64 Gumbel noise is requested. """ logits = apply_top_k_top_p(logits, k, p) logits_to_return = None @@ -330,16 +332,25 @@ class TopKTopPSampler(nn.Module): elif self.logprobs_mode == "processed_logprobs": logits_to_return = logits.log_softmax(dim=-1, dtype=torch.float32) - if not generators and not self.use_fp64_gumbel: - return compiled_random_sample(logits), logits_to_return + if self.use_fp64_gumbel: + probs = logits.softmax(dim=-1, dtype=torch.float32) + q = empty_exponential_noise_like(probs, self.use_fp64_gumbel) + q.exponential_() + for i, generator in generators.items(): + q[i].exponential_(generator=generator) + return sample_with_exponential_noise(probs, q), logits_to_return - probs = logits.softmax(dim=-1, dtype=torch.float32) - q = empty_exponential_noise_like(probs, self.use_fp64_gumbel) - q.exponential_() - for i, generator in generators.items(): - q[i].exponential_(generator=generator) - - return sample_with_exponential_noise(probs, q), logits_to_return + batch_size = logits.shape[0] + seeds = torch.randint(0, 2**31, (batch_size,), dtype=torch.long) + for i, gen in generators.items(): + seeds[i] = torch.randint( + 0, 2**31, (1,), generator=gen, dtype=torch.long + ).item() + logits_f32 = logits.to(dtype=torch.float32) + return ( + torch.ops._C.fused_gumbel_argmax(logits_f32, seeds), + logits_to_return, + ) def _init_aiter_ops(self) -> bool: if self._aiter_ops_import_failed: diff --git a/vllm/v1/sample/sampler.py b/vllm/v1/sample/sampler.py index 7e8d597d88..d19f65f6d8 100644 --- a/vllm/v1/sample/sampler.py +++ b/vllm/v1/sample/sampler.py @@ -6,6 +6,7 @@ import torch import torch.nn as nn from vllm.config.model import LogprobsMode +from vllm.platforms import current_platform from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.torch_utils import PIN_MEMORY from vllm.v1.outputs import LogprobsTensors, SamplerOutput @@ -239,6 +240,8 @@ class Sampler(nn.Module): @staticmethod def greedy_sample(logits: torch.Tensor) -> torch.Tensor: + if current_platform.is_cpu(): + return torch.ops._C.greedy_argmax(logits) return logits.argmax(dim=-1).view(-1) def sample( From fcd317cf33b5113768a8ba1570f5211627f6fd63 Mon Sep 17 00:00:00 2001 From: Zheng Gong Date: Wed, 30 Sep 2026 15:26:57 +0800 Subject: [PATCH 092/248] [ROCm][DSv4][Perf] Use the shared prefill chunk plan in the ROCm sparse prefill (#58405) Signed-off-by: Zheng Gong Co-authored-by: Cursor --- vllm/models/deepseek_v4/amd/rocm.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 292901dfe9..c4c4d416b8 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -1426,7 +1426,6 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): swa_only = attn_metadata is None - num_prefills = swa_metadata.num_prefills num_prefill_tokens = swa_metadata.num_prefill_tokens num_decodes = swa_metadata.num_decodes num_decode_tokens = swa_metadata.num_decode_tokens @@ -1458,26 +1457,22 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): topk_indices = attn_metadata.c128a_prefill_topk_indices assert topk_indices is not None top_k = topk_indices.shape[-1] - N = (self.max_model_len + self.compress_ratio - 1) // self.compress_ratio else: assert self.topk_indices_buffer is not None topk_indices = self.topk_indices_buffer[num_decode_tokens:] top_k = 0 - N = 0 - M = N + self.window_size + self.max_num_batched_tokens - num_chunks = (num_prefills + self.PREFILL_CHUNK_SIZE - 1) // ( - self.PREFILL_CHUNK_SIZE + chunk_plan = swa_metadata.get_prefill_chunk_plan( + compress_ratio=self.compress_ratio, + prefill_chunk_size=self.PREFILL_CHUNK_SIZE, ) - + assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0" workspace_manager = current_workspace_manager() - kv = workspace_manager.get_simultaneous( - ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), - )[0] - for chunk_idx in range(num_chunks): - chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE - chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills) + for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan: chunk_size = chunk_end - chunk_start + kv = workspace_manager.get_simultaneous( + ((chunk_size, chunk_M, q.shape[-1]), torch.bfloat16), + )[0] if not swa_only: assert attn_metadata is not None assert compressed_k_cache is not None @@ -1502,7 +1497,7 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): gather_lens=gather_lens[chunk_start:chunk_end], block_table=swa_block_table[chunk_start:chunk_end], block_size=swa_metadata.block_size, - offset=N, + offset=chunk_N, use_fnuz=current_platform.is_fp8_fnuz(), ) @@ -1523,8 +1518,8 @@ class DeepseekV4ROCMAiterMLAAttention(DeepseekV4Attention): self.window_size, self.compress_ratio, top_k, - M, - N, + chunk_M, + chunk_N, max_image_tokens=self.max_image_tokens, left_visible=( left_visible[query_start:query_end] From 87228cff01d7a56988e572c9a689444728f26568 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 30 Sep 2026 15:29:01 +0800 Subject: [PATCH 093/248] [Rust Frontend] Replay roundtrip output grammars through XGrammar (#59143) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/src/chat/Cargo.toml | 1 + rust/src/chat/tests/grammar_replay.py | 113 ++++ rust/src/chat/tests/grammar_replay.rs | 24 + .../grammar_replay/DeepSeek-V3.2-Exp.json | 274 ++++++++ .../grammar_replay/DeepSeek-V4-Flash.json | 274 ++++++++ .../grammar_replay/DeepSeek-V4.1-Flash.json | 274 ++++++++ .../tests/grammar_replay/GLM-4.7-Flash.json | 251 +++++++ .../tests/grammar_replay/GLM-5.2-FP8.json | 251 +++++++ rust/src/chat/tests/grammar_replay/Hy3.json | 625 ++++++++++++++++++ .../chat/tests/grammar_replay/Kimi-K2.5.json | 340 ++++++++++ .../chat/tests/grammar_replay/Kimi-K3.json | 587 ++++++++++++++++ .../grammar_replay/MiMo-V2.6-Flash-RL.json | 254 +++++++ .../tests/grammar_replay/MiniMax-M2.5.json | 301 +++++++++ .../chat/tests/grammar_replay/Qwen3-0.6B.json | 277 ++++++++ .../chat/tests/grammar_replay/Qwen3.5-4B.json | 268 ++++++++ rust/src/chat/tests/roundtrip.rs | 454 ++++++++++--- rust/src/parser/src/output_grammar.rs | 6 +- 17 files changed, 4484 insertions(+), 90 deletions(-) create mode 100755 rust/src/chat/tests/grammar_replay.py create mode 100644 rust/src/chat/tests/grammar_replay.rs create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json create mode 100644 rust/src/chat/tests/grammar_replay/Hy3.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K2.5.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K3.json create mode 100644 rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index a110020ec1..d145fc237a 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -46,6 +46,7 @@ clap.workspace = true expect-test.workspace = true paste.workspace = true rmp-serde.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } serial_test.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/rust/src/chat/tests/grammar_replay.py b/rust/src/chat/tests/grammar_replay.py new file mode 100755 index 0000000000..d34df627ba --- /dev/null +++ b/rust/src/chat/tests/grammar_replay.py @@ -0,0 +1,113 @@ +#!/usr/bin/env -S uv run --script +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "tiktoken>=0.6.0", +# # xgrammar imports torch unconditionally; only the CPU build is needed. +# "torch", +# "transformers>=5.10.4", +# # Keep in sync with requirements/common.txt. +# "xgrammar==0.2.7", +# ] +# +# [[tool.uv.index]] +# name = "pytorch-cpu" +# url = "https://download.pytorch.org/whl/cpu" +# explicit = true +# +# [tool.uv.sources] +# torch = { index = "pytorch-cpu" } +# /// +"""Replay the grammar cases exported by the roundtrip tests through XGrammar. + +Each file holds one model's cases, keyed by tool-choice variant. Every case is +compiled against the model tokenizer the same way as +`vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then +replayed from the first generated token: each token must be allowed by the +bitmask and accepted by the matcher, and the final stop token must terminate it. +""" + +import json +import sys +from functools import cache +from pathlib import Path + +import xgrammar as xgr +from transformers import AutoTokenizer + +CASES_DIR = Path(__file__).parent / "grammar_replay" + + +@cache +def load_tokenizer(model_id: str, vocab_size: int): + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) + tokenizer_info = xgr.TokenizerInfo.from_huggingface( + tokenizer, vocab_size=vocab_size + ) + return tokenizer, xgr.GrammarCompiler(tokenizer_info) + + +def replay(model: dict, case: dict) -> str: + """Check one case of a model and return a summary of what was checked.""" + tokenizer, compiler = load_tokenizer(model["model_id"], model["vocab_size"]) + generation = case["generation_token_ids"] + + # The case token IDs come from the Rust tokenizer; the runner must load the + # same vocabulary for the grammar to mean the same thing. + encoded = tokenizer.encode(case["completion"], add_special_tokens=False) + if encoded != generation[:-1]: + raise AssertionError( + "tokenizer mismatch: the completion encodes to " + f"{encoded}, but the case records {generation[:-1]}" + ) + + grammar = case["grammar"] + structural_tag = {"type": "structural_tag", "format": grammar["format"]} + compiled = compiler.compile_structural_tag(json.dumps(structural_tag)) + if grammar["coverage"] != "from_token_zero": + return f"compiled only ({grammar['coverage']} is gated by the engine)" + + matcher = xgr.GrammarMatcher( + compiled, override_stop_tokens=model["all_stop_token_ids"] + ) + bitmask = xgr.allocate_token_bitmask(1, model["vocab_size"]) + for index, token_id in enumerate(generation): + matcher.fill_next_token_bitmask(bitmask) + allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1 + if not allowed or not matcher.accept_token(token_id): + prefix = tokenizer.decode(generation[:index]) + raise AssertionError( + f"token {index} ({token_id}, {tokenizer.decode([token_id])!r}) " + f"rejected after {prefix!r}" + ) + if not matcher.is_terminated(): + raise AssertionError("matcher not terminated after the stop token") + return f"replayed {len(generation)} tokens from token zero" + + +def main() -> int: + paths = sorted(CASES_DIR.glob("*.json")) + if not paths: + print(f"no grammar replay cases in {CASES_DIR}", file=sys.stderr) + return 1 + + failures = 0 + for path in paths: + model = json.loads(path.read_text()) + for name, case in model["cases"].items(): + try: + summary = replay(model, case) + except Exception as error: + failures += 1 + coverage = case["grammar"]["coverage"] + print(f"FAIL {path.name} {name} [{coverage}]: {error}", file=sys.stderr) + else: + print(f"PASS {path.name} {name}: {summary}") + + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/src/chat/tests/grammar_replay.rs b/rust/src/chat/tests/grammar_replay.rs new file mode 100644 index 0000000000..4ddfa6bf24 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Replays the grammar cases exported by the roundtrip tests through the real +//! XGrammar, so that parser-built output grammars are checked against the +//! engine's compiler and matcher. + +use std::path::PathBuf; +use std::process::Command; + +#[test] +fn xgrammar_accepts_roundtrip_generations() { + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay.py"); + let output = Command::new(&script) + .output() + .unwrap_or_else(|error| panic!("failed to execute {:?}: {error}", script)); + assert!( + output.status.success(), + "grammar replay failed: status={:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json new file mode 100644 index 0000000000..4903b5900c --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json @@ -0,0 +1,274 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3.2-Exp", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|DSML|function_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|DSML|function_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|function_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|function_calls>\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|function_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json new file mode 100644 index 0000000000..1a286d8acd --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json @@ -0,0 +1,274 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4-Flash", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|DSML|tool_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|DSML|tool_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|tool_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML|invoke name=\"add\">\n<|DSML|parameter name=\"y\" string=\"false\">1.0\n<|DSML|parameter name=\"x\" string=\"false\">2\n<|DSML|parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML|tool_calls>\n" + }, + { + "type": "tag", + "begin": "<|DSML|invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML|tool_calls>\n<|DSML|invoke name=\"get_weather\">\n<|DSML|parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json new file mode 100644 index 0000000000..a21d8b90e5 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json @@ -0,0 +1,274 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|DSML| calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|DSML| calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML| calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n<|DSML| invoke name=\"add\">\n<|DSML| parameter name=\"y\" string=\"false\">1.0\n<|DSML| parameter name=\"x\" string=\"false\">2\n<|DSML| parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<|DSML| calls>\n" + }, + { + "type": "tag", + "begin": "<|DSML| invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<|DSML| calls>\n<|DSML| invoke name=\"get_weather\">\n<|DSML| parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json new file mode 100644 index 0000000000..b289ec62e6 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json @@ -0,0 +1,251 @@ +{ + "model_id": "zai-org/GLM-4.7-Flash", + "vocab_size": 154880, + "all_stop_token_ids": [154820, 154827, 154829], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json new file mode 100644 index 0000000000..404667c30a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json @@ -0,0 +1,251 @@ +{ + "model_id": "zai-org/GLM-5.2-FP8", + "vocab_size": 154880, + "all_stop_token_ids": [154820, 154827, 154829], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Hy3.json b/rust/src/chat/tests/grammar_replay/Hy3.json new file mode 100644 index 0000000000..f6ae7b0969 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Hy3.json @@ -0,0 +1,625 @@ +{ + "model_id": "tencent/Hy3", + "vocab_size": 120832, + "all_stop_token_ids": [120025], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + "end": "\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json new file mode 100644 index 0000000000..8ac6e689f7 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json @@ -0,0 +1,340 @@ +{ + "model_id": "moonshotai/Kimi-K2.5", + "vocab_size": 163840, + "all_stop_token_ids": [163585, 163586], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|tool_calls_section_begin|>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|tool_calls_section_begin|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|tool_calls_section_end|>" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "<|tool_call_begin|>" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json new file mode 100644 index 0000000000..ffe5061192 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -0,0 +1,587 @@ +{ + "model_id": "moonshotai/Kimi-K3", + "vocab_size": 163840, + "all_stop_token_ids": [163585, 163586], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json new file mode 100644 index 0000000000..582f29a1b9 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json @@ -0,0 +1,254 @@ +{ + "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", + "vocab_size": 152576, + "all_stop_token_ids": [151643, 151645, 151672], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "Need call the weather and add tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.Shanghai", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json new file mode 100644 index 0000000000..64f65e2f5a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json @@ -0,0 +1,301 @@ +{ + "model_id": "MiniMaxAI/MiniMax-M2.5", + "vocab_size": 200064, + "all_stop_token_ids": [200020], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json new file mode 100644 index 0000000000..5d171fc5f0 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json @@ -0,0 +1,277 @@ +{ + "model_id": "Qwen/Qwen3-0.6B", + "vocab_size": 151936, + "all_stop_token_ids": [151643, 151645], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "}\n" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json new file mode 100644 index 0000000000..8bd4c901e5 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json @@ -0,0 +1,268 @@ +{ + "model_id": "Qwen/Qwen3.5-4B", + "vocab_size": 248320, + "all_stop_token_ids": [248044, 248046], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "\n\n" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } +} diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index a0ccbe59ab..6925ae1b02 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -6,20 +6,31 @@ //! The invariant under test is that a structured assistant message rendered as history can be //! parsed from the generated assistant completion and then rendered back to the exact same //! assistant completion. +//! +//! The tool-call fixture also snapshots the output grammar built by the initialized parser for each +//! tool-choice variant, with the rendered completion as its generation, as one file per model under +//! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar. +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Debug; +use std::ops::Range; +use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use anyhow::{Context as _, Result, bail, ensure}; use futures::{Stream, StreamExt as _, stream}; +use serde::ser::Error as _; +use serde::{Serialize, Serializer}; use serde_json_fmt::JsonFormat as JsonFmt; use serial_test::file_serial; use vllm_chat::{ AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall, - ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode, - LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection, - load_model_backends, + ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason, + GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, + RendererSelection, ToolStrictLevel, load_model_backends, }; +use vllm_parser::output_grammar::BuiltOutputGrammar; use vllm_text::{DecodedTextEvent, Finished, Prompt}; use vllm_tokenizer::Tokenizer; @@ -114,6 +125,33 @@ impl ThinkingBehavior { } } +/// Tool-choice variant of the tool-call fixture. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum ToolChoiceVariant { + /// `auto`, with content before the tool calls. + Auto, + /// `required`, with tool calls only: builder grammars admit no content + /// before the calls. + Required, + /// Named `get_weather`, with a single call to it and no content. + Named, +} + +impl ToolChoiceVariant { + const ALL: [Self; 3] = [Self::Auto, Self::Required, Self::Named]; + + fn tool_choice(self) -> ChatToolChoice { + match self { + Self::Auto => ChatToolChoice::Auto, + Self::Required => ChatToolChoice::Required, + Self::Named => ChatToolChoice::Function { + name: "get_weather".to_string(), + }, + } + } +} + impl RoundtripCase { /// Qwen3 XML tool-call format with `qwen3` reasoning tags. fn qwen3() -> Self { @@ -451,6 +489,7 @@ async fn run_roundtrip_reasoning_and_content_inner( "roundtrip-reasoning-content", vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")], Vec::new(), + None, thinking, case.thinking_behavior, ); @@ -489,11 +528,25 @@ async fn run_roundtrip_reasoning_and_content_inner( Ok(()) } -/// Run the fixed reasoning+multiple-tools fixture for one model/parser case. +/// Run the fixed reasoning+multiple-tools fixture for one model/parser case, +/// once per tool-choice variant. async fn run_roundtrip_tool_call_mix( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, ) -> Result<()> { + let mut results = Vec::new(); + for variant in ToolChoiceVariant::ALL { + let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?; + results.push((variant, result)); + } + check_grammar_replay_file(case, backends, results) +} + +async fn run_roundtrip_tool_call_mix_inner( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + variant: ToolChoiceVariant, +) -> Result { let request = roundtrip_request( "roundtrip-reasoning-tools", vec![ChatMessage::text( @@ -501,41 +554,40 @@ async fn run_roundtrip_tool_call_mix( "Check Shanghai weather and add 1.0 plus 2.", )], test_tools(), + Some(variant.tool_choice()), Some(true), // always enable thinking in this fixture case.thinking_behavior, ); let expected_reasoning = "Need call the weather and add tools."; - let expected_text = "I will call the tools."; + let (expected_text, tool_call_count) = match variant { + ToolChoiceVariant::Auto => (Some("I will call the tools."), 2), + ToolChoiceVariant::Required => (None, 2), + ToolChoiceVariant::Named => (None, 1), + }; - let result = run_roundtrip( - case, - backends, - &request, - AssistantMessage { - content: vec![ - AssistantContentBlock::Reasoning { - text: expected_reasoning.to_string(), - }, - AssistantContentBlock::Text { - text: expected_text.to_string(), - }, - AssistantContentBlock::ToolCall(AssistantToolCall { - id: "functions.get_weather:0".to_string(), - name: "get_weather".to_string(), - arguments: r#"{"location":"Shanghai"}"#.to_string(), - }), - AssistantContentBlock::ToolCall(AssistantToolCall { - id: "functions.add:1".to_string(), - name: "add".to_string(), - // Intentionally use a non-lexical order of keys to verify text-level - // fidelity of the roundtrip where JSON formatting remains stable. The - // `items` key also exercises templates that call `arguments.items()`. - arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), - }), - ], - }, - ) - .await?; + let mut content = vec![AssistantContentBlock::Reasoning { + text: expected_reasoning.to_string(), + }]; + content.extend(expected_text.map(|text| AssistantContentBlock::Text { + text: text.to_string(), + })); + let tool_calls = [ + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.get_weather:0".to_string(), + name: "get_weather".to_string(), + arguments: r#"{"location":"Shanghai"}"#.to_string(), + }), + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.add:1".to_string(), + name: "add".to_string(), + // Intentionally use a non-lexical order of keys to verify text-level + // fidelity of the roundtrip where JSON formatting remains stable. The + // `items` key also exercises templates that call `arguments.items()`. + arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), + }), + ]; + content.extend(tool_calls.into_iter().take(tool_call_count)); + let result = run_roundtrip(case, backends, &request, AssistantMessage { content }).await?; assert_eq!( result.parsed_message.reasoning().as_deref(), @@ -543,12 +595,15 @@ async fn run_roundtrip_tool_call_mix( "parsed message: {:#?}", result.parsed_message ); - assert_eq!(result.parsed_message.text(), expected_text); + assert_eq!( + result.parsed_message.text(), + expected_text.unwrap_or_default() + ); let tool_calls = result.parsed_message.tool_calls().collect::>(); assert_eq!( tool_calls.len(), - 2, + tool_call_count, "parsed message: {:#?}", result.parsed_message ); @@ -557,18 +612,20 @@ async fn run_roundtrip_tool_call_mix( tool_calls[0].arguments, expected_arguments(case, r#"{"location": "Shanghai"}"#)?, ); - assert_eq!(tool_calls[1].name, "add"); - assert_eq!( - tool_calls[1].arguments, - expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?, - ); + if let Some(add) = tool_calls.get(1) { + assert_eq!(add.name, "add"); + assert_eq!( + add.arguments, + expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?, + ); + } assert_eq!( result.rerendered_closed_completion, result.closed_completion ); - Ok(()) + Ok(result) } /// Compact JSON argument formatting used by JSON-native parsers/renderers. @@ -645,6 +702,9 @@ struct RoundtripResult { /// Assistant-completion suffix cut after rendering the parsed assistant /// back as history. rerendered_closed_completion: Prompt, + /// Output grammar built by the same initialized parser that parsed the + /// completion. + output_grammar: Option, } /// Render, parse, and rerender one assistant turn through the production @@ -658,13 +718,17 @@ async fn run_roundtrip( let renderer = backends.chat_backend.chat_renderer(); let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?; - let parsed_message = parse_completion(case, backends, request, &rendered).await?; + let ParsedCompletion { + message: parsed_message, + output_grammar, + } = parse_completion(case, backends, request, &rendered).await?; let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?; Ok(RoundtripResult { parsed_message, closed_completion: rendered.completion, rerendered_closed_completion: rerendered.completion, + output_grammar, }) } @@ -675,7 +739,7 @@ struct RenderedTurn { } /// Render `history` as a production prompt and `history + assistant` as closed -/// history, then return the production prompt and assistant-completion suffix. +/// history, then return the production prompt and the assistant completion. fn render_closed_completion( renderer: &dyn vllm_chat::ChatRenderer, base_request: &ChatRequest, @@ -692,18 +756,15 @@ fn render_closed_completion( let completion = match (&prompt, full) { (Prompt::Text(prompt), Prompt::Text(full)) => { - ensure!( - full.starts_with(prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - Prompt::Text(full[prompt.len()..].to_string()) + let range = completion_range(prompt.as_bytes(), full.as_bytes()) + .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?; + let completion = full.get(range).context("completion splits a UTF-8 character")?; + Prompt::Text(completion.to_string()) } (Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => { - ensure!( - full.starts_with(prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - Prompt::TokenIds(full[prompt.len()..].to_vec()) + let range = completion_range(prompt, &full) + .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?; + Prompt::TokenIds(full[range].to_vec()) } (prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"), }; @@ -711,14 +772,81 @@ fn render_closed_completion( Ok(RenderedTurn { prompt, completion }) } +/// Locate the assistant completion in `full`, the history rendered with the +/// assistant message, given `prompt`, the same history rendered for generation. +/// +/// Some templates render request-scoped control messages after the whole +/// history, such as Kimi K3's `tool_choice=required` message. With history `A`, +/// control messages `C`, generation prompt `G`, and completion `B`, the renders +/// are `prompt = A + C + G` and `full = A + G + B + C`. `C` is empty for most +/// templates, which makes `prompt` a prefix of `full`. +fn completion_range(prompt: &[T], full: &[T]) -> Result> { + if full.starts_with(prompt) { + return Ok(prompt.len()..full.len()); + } + ensure!( + full.len() >= prompt.len(), + "full prompt is shorter than production prompt" + ); + + // `A` is a common prefix of both renders; try every split of the remaining + // `prompt` into `C + G` and require exactly one to fit `full`. + let common_prefix_len = prompt.iter().zip(full).take_while(|(p, f)| p == f).count(); + let mut control_lens = Vec::new(); + for history_len in 0..=common_prefix_len { + let rest = &prompt[history_len..]; + for control_len in 1..=rest.len() { + let (control, generation) = rest.split_at(control_len); + if full[history_len..].starts_with(generation) && full.ends_with(control) { + control_lens.push(control_len); + } + } + } + let [control_len] = control_lens[..] else { + bail!( + "full prompt must be `A + G + B + C` for production prompt `A + C + G`; found {} splits", + control_lens.len() + ); + }; + + Ok(prompt.len() - control_len..full.len() - control_len) +} + +#[test] +fn completion_range_skips_trailing_control_messages() { + let history = "hi"; + let control = "call tools"; + let generation = ""; + let completion = "call()"; + + let prompt = format!("{history}{generation}"); + let full = format!("{history}{generation}{completion}"); + let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap(); + assert_eq!(&full[range], completion); + + let prompt = format!("{history}{control}{generation}"); + let full = format!("{history}{generation}{completion}{control}"); + let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap(); + assert_eq!(&full[range], completion); +} + +/// Output of one production output processor over a rendered completion. +struct ParsedCompletion { + /// Terminal assistant message. + message: AssistantMessage, + /// Output grammar built by the processor after initialization. + output_grammar: Option, +} + /// Feed one rendered assistant completion body into the real output processor -/// and collect its terminal assistant message. +/// and collect its terminal assistant message, along with the output grammar +/// the initialized processor builds for the request. async fn parse_completion( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, base_request: &ChatRequest, rendered: &RenderedTurn, -) -> Result { +) -> Result { let tokenizer = backends.text_backend.tokenizer(); let prompt_token_ids = match &rendered.prompt { Prompt::Text(prompt) => tokenizer @@ -731,12 +859,15 @@ async fn parse_completion( let mut processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { - tool_strict_level: vllm_chat::ToolStrictLevel::Auto, + // Pin every tool schema so that builder-backed parsers build an + // output grammar for replay. The strict level does not affect parsing. + tool_strict_level: ToolStrictLevel::Parameter, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, )?; processor.initialize(&prompt_token_ids)?; + let output_grammar = processor.build_output_grammar()?; let decoded = decoded_completion_stream( tokenizer.as_ref(), @@ -748,7 +879,10 @@ async fn parse_completion( while let Some(event) = events.next().await { if let ChatEvent::Done { message, .. } = event? { - return Ok(message); + return Ok(ParsedCompletion { + message, + output_grammar, + }); } } @@ -772,14 +906,8 @@ fn decoded_completion_stream( prompt_logprobs: None, }]; - let chunks = match completion { - Prompt::Text(text) => { - let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { - format!( - "closed assistant completion did not end with {:?}: {:?}", - assistant_stop_suffix, text - ) - })?; + let chunks = match completion_body(tokenizer, completion, assistant_stop_suffix)? { + CompletionBody::Text(body) => { // The engine returns token ids, so attribute the rendered text // through the production decoder, then split it into small char // chunks to exercise marker splits. @@ -807,27 +935,12 @@ fn decoded_completion_stream( }) .collect() } - Prompt::TokenIds(token_ids) => { - let body = if assistant_stop_suffix.is_empty() { - token_ids.as_slice() - } else { - let stop_token_ids = tokenizer - .encode(assistant_stop_suffix, false) - .context("failed to encode token-id completion stop suffix")?; - token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| { - format!( - "token-id completion did not end with {:?}: {:?}", - assistant_stop_suffix, token_ids - ) - })? - }; - incremental_decode_chunks( - tokenizer, - &prompt_token_ids, - body, - TOKEN_COMPLETION_CHUNK_TOKENS, - )? - } + CompletionBody::TokenIds(body) => incremental_decode_chunks( + tokenizer, + &prompt_token_ids, + body, + TOKEN_COMPLETION_CHUNK_TOKENS, + )?, }; if chunks.is_empty() { @@ -868,6 +981,48 @@ fn decoded_completion_stream( Ok(stream::iter(events).map(Ok).boxed()) } +/// Generated part of a closed assistant completion, without the rendered stop +/// suffix. +enum CompletionBody<'a> { + Text(&'a str), + TokenIds(&'a [u32]), +} + +/// Strip the rendered stop suffix from a closed assistant completion. +fn completion_body<'a>( + tokenizer: &dyn Tokenizer, + completion: &'a Prompt, + assistant_stop_suffix: &str, +) -> Result> { + match completion { + Prompt::Text(text) => { + let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { + format!( + "closed assistant completion did not end with {:?}: {:?}", + assistant_stop_suffix, text + ) + })?; + Ok(CompletionBody::Text(body)) + } + Prompt::TokenIds(token_ids) => { + let body = if assistant_stop_suffix.is_empty() { + token_ids.as_slice() + } else { + let stop_token_ids = tokenizer + .encode(assistant_stop_suffix, false) + .context("failed to encode token-id completion stop suffix")?; + token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| { + format!( + "token-id completion did not end with {:?}: {:?}", + assistant_stop_suffix, token_ids + ) + })? + }; + Ok(CompletionBody::TokenIds(body)) + } + } +} + /// One decoded completion chunk fed into the output processor. struct DecodedCompletionChunk { /// Attributed for token-id completions, as the production decoder emits it. @@ -941,15 +1096,138 @@ fn incremental_decode_chunks( Ok(chunks) } +/// Grammar replay cases of one model, replayed through the real XGrammar by +/// `grammar_replay.py`. +#[derive(Serialize)] +struct GrammarReplayFile { + /// Hugging Face model id whose tokenizer the replay loads. + model_id: &'static str, + /// Model vocabulary size, which sizes the XGrammar token mask. + vocab_size: usize, + /// Request stop set, passed to the matcher as `override_stop_tokens`. + #[serde(serialize_with = "serialize_one_line")] + all_stop_token_ids: BTreeSet, + /// Cases keyed by tool-choice variant. + cases: BTreeMap, +} + +/// One parser-built output grammar with the generation it must accept. +#[derive(Serialize)] +struct GrammarReplayCase { + /// Parser-built grammar; only token-zero grammars are replayed from the + /// first generated token. + grammar: BuiltOutputGrammar, + /// Generated completion text, without the stop token. + completion: String, + /// Generated token IDs presented to the matcher, ending with a stop token. + #[serde(serialize_with = "serialize_one_line")] + generation_token_ids: Vec, +} + +/// Serialize a token ID list on one line inside the pretty-printed file. +fn serialize_one_line( + token_ids: &impl Serialize, + serializer: S, +) -> std::result::Result { + let json = spaced_json_fmt().format_to_string(token_ids).map_err(S::Error::custom)?; + serde_json::value::RawValue::from_string(json) + .map_err(S::Error::custom)? + .serialize(serializer) +} + +/// Directory of checked-in grammar replay files. +fn grammar_replay_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay") +} + +/// Snapshot the output grammars of one model's roundtrips, keyed by variant, +/// as its grammar replay file, or check that no stale file remains when the +/// parser builds none. +fn check_grammar_replay_file( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + results: Vec<(ToolChoiceVariant, RoundtripResult)>, +) -> Result<()> { + let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name); + let path = grammar_replay_dir().join(format!("{model_name}.json")); + let results = results + .into_iter() + .filter_map(|(variant, result)| { + Some((variant, result.output_grammar?, result.closed_completion)) + }) + .collect::>(); + if results.is_empty() { + ensure!( + !path.exists(), + "{} builds no output grammar, but grammar replay file {path:?} exists", + case.model_id + ); + return Ok(()); + } + + let tokenizer = backends.text_backend.tokenizer(); + let hints = backends.text_backend.sampling_hints()?; + // Lowered `_all_stop_token_ids` of a request without user stop tokens. + let mut all_stop_token_ids = hints.extra_eos_token_ids; + all_stop_token_ids.extend(hints.primary_eos_token_id); + + // The generation ends with the first token of the rendered stop suffix, or + // with the primary EOS for templates that render none. + let stop_token_id = if case.assistant_stop_suffix.is_empty() { + hints.primary_eos_token_id.context("model has no primary EOS token")? + } else { + tokenizer.encode(case.assistant_stop_suffix, false)?[0] + }; + ensure!( + all_stop_token_ids.contains(&stop_token_id), + "stop token {stop_token_id} of {:?} is not in the stop set {all_stop_token_ids:?}", + case.assistant_stop_suffix + ); + + let mut cases = BTreeMap::new(); + for (variant, grammar, closed_completion) in results { + let (completion, mut generation_token_ids) = match completion_body( + tokenizer.as_ref(), + &closed_completion, + case.assistant_stop_suffix, + )? { + CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?), + CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()), + }; + generation_token_ids.push(stop_token_id); + cases.insert( + variant, + GrammarReplayCase { + grammar, + completion, + generation_token_ids, + }, + ); + } + + let file = GrammarReplayFile { + model_id: case.model_id, + vocab_size: backends.text_backend.model_vocab_size(), + all_stop_token_ids, + cases, + }; + let mut json = serde_json::to_string_pretty(&file)?; + json.push('\n'); + expect_test::expect_file![path].assert_eq(&json); + + Ok(()) +} + /// Build a chat request fixture with parser-enabling tool-choice semantics. fn roundtrip_request( request_id: impl Into, messages: Vec, tools: Vec, + tool_choice: Option, thinking: Option, thinking_behavior: ThinkingBehavior, ) -> ChatRequest { - let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true) + let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, tool_choice, true) .expect("tool context should resolve"); let mut request = ChatRequest { request_id: request_id.into(), diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index f19254d364..db56f97708 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -5,6 +5,7 @@ use std::{fmt, str::FromStr}; +use serde::Serialize; use serde_with::{DeserializeFromStr, SerializeDisplay}; use thiserror::Error; use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions}; @@ -20,7 +21,8 @@ use crate::tool::Tool; pub type Result = std::result::Result; /// How much of the generated stream an output grammar covers. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] pub enum GrammarCoverage { /// The grammar covers reasoning and advances from the first generated token. FromTokenZero, @@ -29,7 +31,7 @@ pub enum GrammarCoverage { } /// One parser-built output grammar and its generated-stream coverage. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct BuiltOutputGrammar { /// Typed XGrammar structural-tag format. pub format: Format, From 8ee40690331c25f19ecad49696dd54930fcab696 Mon Sep 17 00:00:00 2001 From: Zheng Gong Date: Wed, 30 Sep 2026 15:31:51 +0800 Subject: [PATCH 094/248] [ROCm][DSv4.1][Perf] Use the shared prefill chunk plan in the ROCm sparse prefill (#58539) Signed-off-by: Zheng Gong --- vllm/models/deepseek_v41/amd/rocm.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index fe020f4061..99fcfa6702 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -1379,7 +1379,6 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): swa_only = attn_metadata is None - num_prefills = swa_metadata.num_prefills num_prefill_tokens = swa_metadata.num_prefill_tokens num_decodes = swa_metadata.num_decodes num_decode_tokens = swa_metadata.num_decode_tokens @@ -1408,21 +1407,23 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): N = 0 M = N + self.window_size + self.max_num_batched_tokens - num_chunks = (num_prefills + self.PREFILL_CHUNK_SIZE - 1) // ( - self.PREFILL_CHUNK_SIZE + chunk_plan = swa_metadata.get_prefill_chunk_plan( + compress_ratio=self.compress_ratio, + prefill_chunk_size=self.PREFILL_CHUNK_SIZE, + has_compressed=not swa_only, ) + assert chunk_plan, "prefill chunk plan must be non-empty when num_prefills > 0" workspace = current_workspace_manager().get_simultaneous( *self._prefill_workspace_shapes(M, num_prefill_tokens, q) ) - kv = workspace[0] + kv_rows = workspace[0].view(-1, q.shape[-1]) if mxfp8_out is not None: output = workspace[1] assert output is not None - for chunk_idx in range(num_chunks): - chunk_start = chunk_idx * self.PREFILL_CHUNK_SIZE - chunk_end = min(chunk_start + self.PREFILL_CHUNK_SIZE, num_prefills) + for chunk_start, chunk_end, chunk_N, chunk_M in chunk_plan: chunk_size = chunk_end - chunk_start + kv = kv_rows[: chunk_size * chunk_M].view(chunk_size, chunk_M, -1) if not swa_only: assert attn_metadata is not None assert compressed_k_cache is not None @@ -1447,7 +1448,7 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): gather_lens=gather_lens[chunk_start:chunk_end], block_table=swa_block_table[chunk_start:chunk_end], block_size=swa_metadata.block_size, - offset=N, + offset=chunk_N, use_fnuz=current_platform.is_fp8_fnuz(), ) @@ -1468,8 +1469,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): self.window_size, self.compress_ratio, top_k, - M, - N, + chunk_M, + chunk_N, ) rocm_sparse_attn_prefill( q=q[query_start:query_end], From 0a30bc3f9ac3cc1a9339e115377a99d32252aed0 Mon Sep 17 00:00:00 2001 From: cjackal <44624812+cjackal@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:44:46 +0900 Subject: [PATCH 095/248] [Model] Extend device-side mm normalization to GLM4V/GLM5Next (#55389) Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> --- docs/design/mm_processing.md | 3 ++ .../generation_ppl_test/test_glm.py | 40 +++++++++++++++++++ .../multimodal/processing/test_glm5next.py | 37 +++++++++++++++++ vllm/model_executor/models/glm4_1v.py | 23 +++++++---- vllm/model_executor/models/glm_ocr.py | 14 ++++++- vllm/models/glm5next/common/model.py | 2 + vllm/models/glm5next/common/multimodal.py | 5 ++- 7 files changed, 114 insertions(+), 10 deletions(-) create mode 100644 tests/models/multimodal/generation_ppl_test/test_glm.py diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 3a37a2d8a4..67b1a7742b 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -80,6 +80,9 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: | name | Architecture | Example HF Models | | -------------------------- | ------------------------------------------ | ----------------------------------------------------- | +| `glm-4v` | `Glm4vForConditionalGeneration` | `zai-org/GLM-4.1V-9B-Thinking`, etc. | +| `glm-5-next` | `Glm5NextForConditionalGeneration` | `zai-org/GLM-5.3-Flash` | +| `glm-ocr` | `GlmOcrForConditionalGeneration` | `zai-org/GLM-OCR` | | `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | | `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | | `llama-nemotron-vl-embed` | `LlamaNemotronVLForEmbedding` | `nvidia/llama-nemotron-embed-vl-1b-v2` | diff --git a/tests/models/multimodal/generation_ppl_test/test_glm.py b/tests/models/multimodal/generation_ppl_test/test_glm.py new file mode 100644 index 0000000000..e724ace657 --- /dev/null +++ b/tests/models/multimodal/generation_ppl_test/test_glm.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest + +from tests.models.utils import GenerateModelInfo + +from .ppl_utils import vqa_ppl_test + +MODELS = [ + (GenerateModelInfo("zai-org/GLM-OCR", hf_ppl=13.885052680969238), 28), +] + + +def _pixel_size(factor: int): + # Match max image token size with qwen's PPL test + return { + "size": { + "shortest_edge": 2 * factor**2, + "longest_edge": 2 * 1280 * factor**2, + } + } + + +@pytest.mark.parametrize("model_info,factor", MODELS) +@pytest.mark.parametrize("mm_device_do_normalize", [True, False]) +def test_ppl( + hf_runner, + vllm_runner, + model_info: GenerateModelInfo, + factor: int, + mm_device_do_normalize: bool, +): + vqa_ppl_test( + hf_runner, + vllm_runner, + model_info, + vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize}, + mm_processor_kwargs=_pixel_size(factor), + ) diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py index 6fb0e397c2..0e1e7b9876 100644 --- a/tests/models/multimodal/processing/test_glm5next.py +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -13,8 +13,12 @@ The checks below are arithmetic: no weights, no GPU. """ import pytest +import torch +from PIL import Image +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.platforms import current_platform from vllm.transformers_utils.processors.glm5next import ( Glm5NextVideoProcessor, _pixel_budget, @@ -168,3 +172,36 @@ def test_video_shorter_than_one_sampling_interval_is_rejected(processor): }, 2, ) + + +@pytest.mark.usefixtures("default_vllm_config") +def test_mm_device_do_normalize(): + device = current_platform.device_type + ctx = build_model_context( + "zai-org/GLM-5.3-Flash", + limit_mm_per_prompt={"image": 2}, + ) + assert ctx.model_config.multimodal_config.mm_device_do_normalize + + ctx.model_config.multimodal_config.mm_device_do_normalize = False + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + images = [ + Image.new("RGB", (310, 470), color=(17, 89, 231)), + Image.new("RGB", (480, 320), color=(201, 13, 127)), + ] + prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images) + mm_items = processor.info.parse_mm_data({"image": images}) + + normalized_inputs = processor(prompt, mm_items=mm_items) + normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"] + + ctx.model_config.multimodal_config.mm_device_do_normalize = True + raw_inputs = processor(prompt, mm_items=mm_items) + raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"] + assert raw_values.dtype == torch.uint8 + + input_norm = build_mm_input_norm(ctx.model_config).to(device) + output = input_norm(raw_values.to(device), normalized_values.dtype) + torch.testing.assert_close( + output, normalized_values.to(device), rtol=1e-5, atol=1e-6 + ) diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index 95cdcc5a20..be456a6a58 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -61,6 +61,10 @@ from vllm.model_executor.layers.attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import ( + IdentityInputNorm, + build_mm_input_norm, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -616,6 +620,7 @@ class Glm4vVisionTransformer(nn.Module): vision_config: Glm4vVisionConfig, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: super().__init__() @@ -642,6 +647,7 @@ class Glm4vVisionTransformer(nn.Module): in_channels=in_channels, hidden_size=self.hidden_size, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() norm_layer = partial(RMSNorm, eps=norm_eps) head_dim = self.hidden_size // self.num_heads @@ -924,7 +930,7 @@ class Glm4vVisionTransformer(nn.Module): encoder_metadata = self.prepare_encoder_metadata(grid_thw) # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = self.input_norm(x.to(device=self.device), self.dtype) x = self.patch_embed(x) x = self.post_conv_layernorm(x) @@ -1755,6 +1761,7 @@ class Glm4vForConditionalGeneration( supports_encoder_tp_data = True supports_tower_connector_lora = True + supports_mm_device_do_normalize = True @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: @@ -1786,6 +1793,7 @@ class Glm4vForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-5), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -1867,7 +1875,7 @@ class Glm4vForConditionalGeneration( if image_input["type"] == "image_embeds": image_embeds = image_input["image_embeds"].type(self.visual.dtype) else: - pixel_values = image_input["pixel_values"].type(self.visual.dtype) + pixel_values = image_input["pixel_values"] if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d" @@ -1888,9 +1896,7 @@ class Glm4vForConditionalGeneration( if video_input["type"] == "video_embeds": video_embeds = video_input["video_embeds"].type(self.visual.dtype) else: - pixel_values_videos = video_input["pixel_values_videos"].type( - self.visual.dtype - ) + pixel_values_videos = video_input["pixel_values_videos"] if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, @@ -2098,8 +2104,11 @@ class Glm4vForConditionalGeneration( flattened_patch_size = ( in_channels * temporal_patch_size * patch_size * patch_size ) - dummy_pixel_values = torch.randn( - total_patches, flattened_patch_size, device=device, dtype=dtype + dummy_pixel_values = torch.zeros( + total_patches, + flattened_patch_size, + device=device, + dtype=self.visual.input_norm.input_dtype or dtype, ) # Override max_seqlen with a safe upper bound for capture. diff --git a/vllm/model_executor/models/glm_ocr.py b/vllm/model_executor/models/glm_ocr.py index 407b08e8a1..ea773f658d 100644 --- a/vllm/model_executor/models/glm_ocr.py +++ b/vllm/model_executor/models/glm_ocr.py @@ -48,6 +48,7 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv2dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -254,9 +255,17 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer): vision_config: "GlmOcrVisionConfig", norm_eps: float = 1e-5, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: - super().__init__(text_config, vision_config, norm_eps, quant_config, prefix) + super().__init__( + text_config, + vision_config, + norm_eps, + quant_config, + input_norm, + prefix, + ) del self.post_conv_layernorm del self.embeddings @@ -334,7 +343,7 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer): grid_thw = torch.tensor(grid_thw, dtype=torch.int32) # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = self.input_norm(x.to(device=self.device), self.dtype) x = self.patch_embed(x) # compute position embedding @@ -393,5 +402,6 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration): config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-5), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/glm5next/common/model.py b/vllm/models/glm5next/common/model.py index 1c826f58be..3ff3476375 100644 --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py @@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe import ( GateLinear, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -1103,6 +1104,7 @@ class Glm5NextForConditionalGeneration( # and yields NaN image features. Mirrors the MLA/KDA proj # pattern (quant_config=None for BF16 submodules). quant_config=None, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index 03ac3fa664..60b5b47a60 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -19,6 +19,7 @@ from vllm.distributed import utils as dist_utils from vllm.model_executor.layers.activation import SiluAndMulWithClamp from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import IdentityInputNorm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -353,6 +354,7 @@ class Glm5NextVisionTransformer(nn.Module): vision_config, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: super().__init__() @@ -386,6 +388,7 @@ class Glm5NextVisionTransformer(nn.Module): in_channels=in_channels, hidden_size=self.hidden_size, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() norm_layer = partial(RMSNorm, eps=norm_eps) head_dim = self.hidden_size // self.num_heads @@ -566,7 +569,7 @@ class Glm5NextVisionTransformer(nn.Module): encoder_metadata: dict[str, torch.Tensor] | None = None, ) -> torch.Tensor: # patchify - x = x.to(device=self.device, dtype=self.dtype) + x = self.input_norm(x.to(device=self.device), self.dtype) x = self.patch_embed(x) if encoder_metadata is not None: From b81984879ab28d0f7c22d21c688689613510ef92 Mon Sep 17 00:00:00 2001 From: Alexandra Sidorova Date: Wed, 30 Sep 2026 12:03:37 +0400 Subject: [PATCH 096/248] [K2 Horizon] fold partial-RoPE permutation into q/k (and norm) weights (#55335) Signed-off-by: Alexandra Sidorova --- .../test_k2_horizon_rope_fold.py | 636 ++++++++++++++++++ vllm/model_executor/models/k2_horizon.py | 287 +++++--- 2 files changed, 845 insertions(+), 78 deletions(-) create mode 100644 tests/model_executor/test_k2_horizon_rope_fold.py diff --git a/tests/model_executor/test_k2_horizon_rope_fold.py b/tests/model_executor/test_k2_horizon_rope_fold.py new file mode 100644 index 0000000000..07e490a041 --- /dev/null +++ b/tests/model_executor/test_k2_horizon_rope_fold.py @@ -0,0 +1,636 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace + +import pytest +import torch +from torch import nn + +import vllm.model_executor.models.k2_horizon as k2_horizon +from tests.quantization.utils import is_quant_method_supported +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.distributed import ( + cleanup_dist_env_and_memory, + init_distributed_environment, + initialize_model_parallel, +) +from vllm.model_executor.layers.linear import ( + LinearBase, + UnquantizedLinearMethod, +) +from vllm.model_executor.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.models.k2_horizon import ( + K2HorizonDecoderLayer, + K2HorizonModel, + K2HorizonRMSNorm, + _rope_weight_perm, + apply_partial_rope, + is_rope_weights_folding_supported, +) +from vllm.platforms import current_platform +from vllm.utils.network_utils import get_open_port +from vllm.utils.torch_utils import set_default_torch_dtype, set_random_seed + +DEVICE = current_platform.device_type +DTYPE = torch.bfloat16 +SHAPES = [ + (128, 64), + (128, 128), + (192, 128), + (96, 32), +] + + +def get_gptj_rope( + positions: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + num_heads: int, + num_kv_heads: int, + head_dim: int, + rope_head_dim: int, + max_position: int, + base: float, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + rotary_emb = get_rope( + rope_head_dim, + max_position=max_position, + is_neox_style=True, + rope_parameters={"rope_theta": base}, + dtype=dtype, + ) + return apply_partial_rope( + rotary_emb, + positions, + q, + k, + num_heads, + num_kv_heads, + head_dim, + rope_head_dim, + fold_rope_weights=False, + ) + + +def _apply_head_perm( + x: torch.Tensor, num: int, head_dim: int, idx: torch.Tensor +) -> torch.Tensor: + return ( + x.reshape(*x.shape[:-1], num, head_dim)[..., idx].reshape(*x.shape).contiguous() + ) + + +def get_neox_rope( + positions: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + num_heads: int, + num_kv_heads: int, + head_dim: int, + rope_head_dim: int, + max_position: int, + base: float, + dtype: torch.dtype, +) -> tuple[torch.Tensor, torch.Tensor]: + rotary_emb = get_rope( + head_dim, + max_position=max_position, + is_neox_style=True, + rope_parameters={"rope_theta": base, "rope_dim": rope_head_dim}, + dtype=dtype, + ) + return apply_partial_rope( + rotary_emb, + positions, + q, + k, + num_heads, + num_kv_heads, + head_dim, + rope_head_dim, + fold_rope_weights=True, + ) + + +def fold_qk_proj_weight( + weight: torch.Tensor, head_dim: int, idx: torch.Tensor +) -> torch.Tensor: + hidden = weight.shape[-1] + return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous() + + +def fold_qk_channels( + vec: torch.Tensor, head_dim: int, idx: torch.Tensor +) -> torch.Tensor: + return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous() + + +@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES) +def test_rope_fold_matches_permute_then_rope( + head_dim, rope_head_dim, default_vllm_config +): + dtype = DTYPE + set_random_seed(0) + num_tokens = 17 + num_heads = 8 + num_kv_heads = 2 + hidden_size = 512 + max_position = 4096 + base = 10000.0 + + torch.set_default_dtype(dtype) + + positions = torch.randint(0, max_position, (num_tokens,), device=DEVICE) + + scale = hidden_size**-0.5 + x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE) + w_q = ( + torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE) + * scale + ) + w_k = ( + torch.randn(num_kv_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE) + * scale + ) + + idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE) + + q = torch.nn.functional.linear(x, w_q) + k = torch.nn.functional.linear(x, w_k) + q_gptj, k_gptj = get_gptj_rope( + positions, + q, + k, + num_heads, + num_kv_heads, + head_dim, + rope_head_dim, + max_position, + base, + dtype, + ) + + q_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_q, head_dim, idx)) + k_folded = torch.nn.functional.linear(x, fold_qk_proj_weight(w_k, head_dim, idx)) + q_neox, k_neox = get_neox_rope( + positions, + q_folded, + k_folded, + num_heads, + num_kv_heads, + head_dim, + rope_head_dim, + max_position, + base, + dtype, + ) + + q_gptj_perm = _apply_head_perm(q_gptj, num_heads, head_dim, idx) + k_gptj_perm = _apply_head_perm(k_gptj, num_kv_heads, head_dim, idx) + + atol, rtol = 1e-2, 1e-2 + + # Two complementary checks. First, an element-wise check that the folded + # NeoX path equals the GPT-J path up to the channel permutation P + # (neox_rope(Pq, Pk) == P @ gptj_rope(q, k)), hence permuting q_gptj/k_gptj. + # Second, the scores check below: P cancels in q @ k.T. + torch.testing.assert_close(q_neox, q_gptj_perm, atol=atol, rtol=rtol) + torch.testing.assert_close(k_neox, k_gptj_perm, atol=atol, rtol=rtol) + + def scores(qt, kt): + qh = qt.reshape(num_tokens, num_heads, head_dim) + kh = kt.reshape(num_tokens, num_kv_heads, head_dim) + g = num_heads // num_kv_heads + qh = qh.reshape(num_tokens, num_kv_heads, g, head_dim).float() + kh = kh.float() + return torch.einsum("tkgd,skd->tksg", qh, kh) + + torch.testing.assert_close( + scores(q_neox, k_neox), scores(q_gptj, k_gptj), atol=atol, rtol=rtol + ) + + +@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES) +@pytest.mark.parametrize("num_heads", [8, 2]) +def test_qk_bias_fold_matches_permute( + head_dim, rope_head_dim, num_heads, default_vllm_config +): + dtype = DTYPE + set_random_seed(0) + num_tokens = 17 + hidden_size = 512 + torch.set_default_dtype(dtype) + + scale = hidden_size**-0.5 + x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=DEVICE) + w = ( + torch.randn(num_heads * head_dim, hidden_size, dtype=dtype, device=DEVICE) + * scale + ) + b = torch.randn(num_heads * head_dim, dtype=dtype, device=DEVICE) + + idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE) + + ref = _apply_head_perm( + torch.nn.functional.linear(x, w, b), num_heads, head_dim, idx + ) + folded = torch.nn.functional.linear( + x, + fold_qk_proj_weight(w, head_dim, idx), + fold_qk_channels(b, head_dim, idx), + ) + + atol, rtol = 1e-2, 1e-2 + torch.testing.assert_close(folded, ref, atol=atol, rtol=rtol) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a GPU (CUDA/ROCm)" +) +@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES) +@pytest.mark.parametrize("num_heads", [8, 2]) +def test_qk_norm_scale_fold_matches_permute( + head_dim, rope_head_dim, num_heads, default_vllm_config +): + dtype = DTYPE + set_random_seed(0) + num_tokens = 13 + torch.set_default_dtype(dtype) + + hidden = num_heads * head_dim + idx = _rope_weight_perm(head_dim, rope_head_dim).to(DEVICE) + + x = torch.randn(num_tokens, hidden, dtype=dtype, device=DEVICE) + + norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE) + with torch.no_grad(): + norm.weight.copy_(torch.randn(hidden, dtype=dtype, device=DEVICE)) + + y_ref = _apply_head_perm(norm(x.clone()), num_heads, head_dim, idx) + + folded_norm = K2HorizonRMSNorm(hidden_size=hidden, n_groups=num_heads).to(DEVICE) + with torch.no_grad(): + folded_norm.weight.copy_(fold_qk_channels(norm.weight, head_dim, idx)) + # Permute x to emulate the folded q_proj/k_proj: at runtime the norm sees the + # already-permuted projection output (P @ x), not the raw activations. + y_folded = folded_norm(_apply_head_perm(x.clone(), num_heads, head_dim, idx)) + + atol, rtol = 1e-2, 1e-2 + torch.testing.assert_close(y_folded, y_ref, atol=atol, rtol=rtol) + + +class _Fp8QKVOnlyConfig(Fp8Config): + def get_quant_method(self, layer, prefix): + if isinstance(layer, LinearBase) and not ( + prefix.endswith("qkv_proj") or prefix.endswith("qk_proj") + ): + return UnquantizedLinearMethod() + return super().get_quant_method(layer, prefix) + + +class _DummyAttention(nn.Module): + def __init__(self, *args, **kwargs) -> None: + super().__init__() + + def forward(self, q, k, v): + return q + + +@pytest.fixture +def dist_init(): + with set_current_vllm_config(VllmConfig()): + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}", + local_rank=0, + backend="gloo", + ) + initialize_model_parallel(1, 1) + yield + cleanup_dist_env_and_memory() + + +def _make_hf_config( + *, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + head_dim: int, + rope_head_dim: int, + attention_bias: bool, + query_key_norm: bool, +) -> SimpleNamespace: + return SimpleNamespace( + hidden_size=hidden_size, + max_position_embeddings=4096, + dual_chunk_attention_config=None, + mlp_only_layers=[], + num_experts=0, # dense layer: avoids the MoE path + decoder_sparse_step=1, + mova_num_experts=0, + num_attention_heads=num_heads, + num_key_value_heads=num_kv_heads, + rope_parameters={"rope_theta": 10000.0}, + rope_head_dim=rope_head_dim, + query_key_norm=query_key_norm, + rms_norm_eps=1e-6, + attention_bias=attention_bias, + head_dim=head_dim, + attention_gate_func=None, + intermediate_size=2 * hidden_size, + hidden_act="silu", + layernorm_num_groups=1, + vocab_size=1000, + num_hidden_layers=1, + tie_word_embeddings=False, + ) + + +def _build_single_layer_model( + monkeypatch, hf_config: SimpleNamespace, *, quant_config=None +) -> K2HorizonModel: + monkeypatch.setattr(k2_horizon, "Attention", _DummyAttention) + + vllm_config = VllmConfig() + vllm_config.model_config = SimpleNamespace( + hf_text_config=hf_config, dtype=torch.get_default_dtype() + ) + vllm_config.quant_config = quant_config + + with set_current_vllm_config(vllm_config): + layer = K2HorizonDecoderLayer(vllm_config=vllm_config, prefix="model.layers.0") + + model = K2HorizonModel.__new__(K2HorizonModel) + nn.Module.__init__(model) + model.config = hf_config + model.quant_config = quant_config + model.num_redundant_experts = 0 + model.layers = nn.ModuleList([layer]) + return model + + +def _fold_rows(weight: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor: + hidden = weight.shape[-1] + return weight.view(-1, head_dim, hidden)[:, idx, :].reshape(-1, hidden).contiguous() + + +def _fold_channels(vec: torch.Tensor, head_dim: int, idx: torch.Tensor) -> torch.Tensor: + return vec.view(-1, head_dim)[:, idx].reshape(-1).contiguous() + + +def _non_qk_weights(prefix, *, hidden_size, intermediate_size, q_size, device=None): + return [ + ( + f"{prefix}.self_attn.o_proj.weight", + torch.randn(hidden_size, q_size, device=device), + ), + ( + f"{prefix}.mlp.gate_proj.weight", + torch.randn(intermediate_size, hidden_size, device=device), + ), + ( + f"{prefix}.mlp.up_proj.weight", + torch.randn(intermediate_size, hidden_size, device=device), + ), + ( + f"{prefix}.mlp.down_proj.weight", + torch.randn(hidden_size, intermediate_size, device=device), + ), + ( + f"{prefix}.input_layernorm.weight", + torch.randn(hidden_size, device=device), + ), + ( + f"{prefix}.post_attention_layernorm.weight", + torch.randn(hidden_size, device=device), + ), + ] + + +@pytest.mark.usefixtures("dist_init") +@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES) +@pytest.mark.parametrize("attention_bias", [False, True]) +def test_load_weights_dense_folds_qk( + head_dim, + rope_head_dim, + attention_bias, + monkeypatch, +): + dtype = DTYPE + set_random_seed(0) + num_heads = 8 + num_kv_heads = 2 + hidden_size = 128 + intermediate_size = 2 * hidden_size + q_size = num_heads * head_dim + kv_size = num_kv_heads * head_dim + prefix = "layers.0" + + with set_default_torch_dtype(dtype): + hf_config = _make_hf_config( + hidden_size=hidden_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + attention_bias=attention_bias, + query_key_norm=True, + ) + model = _build_single_layer_model(monkeypatch, hf_config, quant_config=None) + + assert model.layers[0].self_attn.fold_rope_weights is True + + raw = { + "q_proj.weight": torch.randn(q_size, hidden_size), + "k_proj.weight": torch.randn(kv_size, hidden_size), + "v_proj.weight": torch.randn(kv_size, hidden_size), + "q_norm.weight": torch.randn(q_size), + "k_norm.weight": torch.randn(kv_size), + } + if attention_bias: + raw.update( + { + "q_proj.bias": torch.randn(q_size), + "k_proj.bias": torch.randn(kv_size), + "v_proj.bias": torch.randn(kv_size), + "o_proj.bias": torch.randn(hidden_size), + } + ) + + weights: list[tuple[str, torch.Tensor]] = [ + (f"{prefix}.self_attn.{name}", tensor) for name, tensor in raw.items() + ] + weights += _non_qk_weights( + prefix, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + q_size=q_size, + ) + + loaded = model.load_weights(iter(weights)) + + params = dict(model.named_parameters()) + assert loaded == set(params) + + idx = _rope_weight_perm(head_dim, rope_head_dim) + + qkv = params[f"{prefix}.self_attn.qkv_proj.weight"] + q_part = qkv[:q_size] + k_part = qkv[q_size : q_size + kv_size] + v_part = qkv[q_size + kv_size :] + + q_norm = params[f"{prefix}.self_attn.q_norm.weight"] + k_norm = params[f"{prefix}.self_attn.k_norm.weight"] + + assert torch.equal(q_part, _fold_rows(raw["q_proj.weight"], head_dim, idx)) + assert torch.equal(k_part, _fold_rows(raw["k_proj.weight"], head_dim, idx)) + assert torch.equal(v_part, raw["v_proj.weight"]) # v never folded + assert torch.equal(q_norm, _fold_channels(raw["q_norm.weight"], head_dim, idx)) + assert torch.equal(k_norm, _fold_channels(raw["k_norm.weight"], head_dim, idx)) + assert qkv.dtype == dtype + + if attention_bias: + qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"] + q_bias = qkv_bias[:q_size] + k_bias = qkv_bias[q_size : q_size + kv_size] + v_bias = qkv_bias[q_size + kv_size :] + assert torch.equal(q_bias, _fold_channels(raw["q_proj.bias"], head_dim, idx)) + assert torch.equal(k_bias, _fold_channels(raw["k_proj.bias"], head_dim, idx)) + assert torch.equal(v_bias, raw["v_proj.bias"]) # v bias never folded + + +@pytest.mark.skipif( + not is_quant_method_supported("fp8"), + reason="fp8 is not supported.", +) +@pytest.mark.usefixtures("dist_init") +@pytest.mark.parametrize("head_dim,rope_head_dim", SHAPES) +@pytest.mark.parametrize("attention_bias", [False, True]) +def test_load_weights_fp8_qk_no_fold( + head_dim, + rope_head_dim, + attention_bias, + monkeypatch, +): + set_random_seed(0) + num_heads = 8 + num_kv_heads = 2 + hidden_size = 128 + intermediate_size = 2 * hidden_size + q_size = num_heads * head_dim + kv_size = num_kv_heads * head_dim + prefix = "layers.0" + + def rand_fp8(*shape): + return (torch.randn(*shape, device=DEVICE) * 32.0).to(torch.float8_e4m3fn) + + def rand_scale(): + return torch.rand(1, device=DEVICE, dtype=torch.float32) + 0.1 + + with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE): + hf_config = _make_hf_config( + hidden_size=hidden_size, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rope_head_dim=rope_head_dim, + attention_bias=attention_bias, + query_key_norm=True, + ) + quant_config = _Fp8QKVOnlyConfig( + is_checkpoint_fp8_serialized=True, activation_scheme="dynamic" + ) + model = _build_single_layer_model( + monkeypatch, hf_config, quant_config=quant_config + ) + + qkv_proj = model.layers[0].self_attn.qkv_proj + assert isinstance(qkv_proj.quant_method, Fp8LinearMethod) + assert model.layers[0].self_attn.fold_rope_weights is False + + raw_w = { + "q_proj.weight": rand_fp8(q_size, hidden_size), + "k_proj.weight": rand_fp8(kv_size, hidden_size), + "v_proj.weight": rand_fp8(kv_size, hidden_size), + } + raw_s = { + "q_proj.weight_scale": rand_scale(), + "k_proj.weight_scale": rand_scale(), + "v_proj.weight_scale": rand_scale(), + } + q_norm = torch.randn(q_size, device=DEVICE) + k_norm = torch.randn(kv_size, device=DEVICE) + + weights: list[tuple[str, torch.Tensor]] = [ + (f"{prefix}.self_attn.{name}", tensor) + for name, tensor in {**raw_w, **raw_s}.items() + ] + weights += [ + (f"{prefix}.self_attn.q_norm.weight", q_norm), + (f"{prefix}.self_attn.k_norm.weight", k_norm), + ] + biases = {} + if attention_bias: + biases = { + "q_proj.bias": torch.randn(q_size, device=DEVICE), + "k_proj.bias": torch.randn(kv_size, device=DEVICE), + "v_proj.bias": torch.randn(kv_size, device=DEVICE), + "o_proj.bias": torch.randn(hidden_size, device=DEVICE), + } + weights += [ + (f"{prefix}.self_attn.{name}", tensor) + for name, tensor in biases.items() + ] + weights += _non_qk_weights( + prefix, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + q_size=q_size, + device=DEVICE, + ) + + loaded = model.load_weights(iter(weights)) + + params = dict(model.named_parameters()) + assert loaded == set(params) + + qkv = params[f"{prefix}.self_attn.qkv_proj.weight"] + weight_scale = params[f"{prefix}.self_attn.qkv_proj.weight_scale"] + + assert qkv.dtype == torch.float8_e4m3fn + assert torch.equal(qkv[:q_size], raw_w["q_proj.weight"]) + assert torch.equal(qkv[q_size : q_size + kv_size], raw_w["k_proj.weight"]) + assert torch.equal(qkv[q_size + kv_size :], raw_w["v_proj.weight"]) + + assert torch.equal( + weight_scale, + torch.cat( + [ + raw_s["q_proj.weight_scale"], + raw_s["k_proj.weight_scale"], + raw_s["v_proj.weight_scale"], + ] + ), + ) + + assert torch.equal(params[f"{prefix}.self_attn.q_norm.weight"], q_norm) + assert torch.equal(params[f"{prefix}.self_attn.k_norm.weight"], k_norm) + + if attention_bias: + qkv_bias = params[f"{prefix}.self_attn.qkv_proj.bias"] + assert torch.equal(qkv_bias[:q_size], biases["q_proj.bias"]) + assert torch.equal(qkv_bias[q_size : q_size + kv_size], biases["k_proj.bias"]) + assert torch.equal(qkv_bias[q_size + kv_size :], biases["v_proj.bias"]) + + +def test_is_rope_weights_folding_supported(default_vllm_config): + dense = SimpleNamespace(quant_method=UnquantizedLinearMethod()) + quantized = SimpleNamespace(quant_method=object()) + + assert is_rope_weights_folding_supported(dense, None) is True + assert is_rope_weights_folding_supported(quantized, None) is False + assert is_rope_weights_folding_supported(dense, {"chunk_size": 8}) is False + assert is_rope_weights_folding_supported(SimpleNamespace(), None) is False diff --git a/vllm/model_executor/models/k2_horizon.py b/vllm/model_executor/models/k2_horizon.py index db435eaf05..8ef517ac32 100644 --- a/vllm/model_executor/models/k2_horizon.py +++ b/vllm/model_executor/models/k2_horizon.py @@ -71,6 +71,7 @@ from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, RowParallelLinear, + UnquantizedLinearMethod, ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -119,6 +120,96 @@ def interleaved_to_split(x): return x.reshape(*x.shape[:-1], -1, 2).transpose(-1, -2).reshape(*x.shape[:-1], -1) +def is_rope_weights_folding_supported( + qk_proj: nn.Module, + dual_chunk_attention_config: dict[str, Any] | None, +) -> bool: + """Whether the partial-RoPE weight-fold path is safe for this attention. + + Folding permutes the q/k projection's output channels at load time, which + only produces correct numerics when: + + * the q/k projection is dense/unquantized and + * dual-chunk attention is disabled -- the P.RoPE equivalence has not been + validated with dual-chunk attention. + + When either condition fails we fall back to the runtime channel-permutation + RoPE path instead of folding. + """ + is_dense_qk_proj = isinstance( + getattr(qk_proj, "quant_method", None), UnquantizedLinearMethod + ) + return is_dense_qk_proj and dual_chunk_attention_config is None + + +def apply_partial_rope( + rotary_emb, + positions: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + num_heads: int, + num_kv_heads: int, + head_dim: int, + rope_head_dim: int, + fold_rope_weights: bool, +) -> tuple[torch.Tensor, torch.Tensor]: + if fold_rope_weights or rope_head_dim == head_dim: + return rotary_emb(positions, q, k) + + q = q.reshape(*q.shape[:-1], num_heads, head_dim) + k = k.reshape(*k.shape[:-1], num_kv_heads, head_dim) + + q_rope, q_nope = torch.split( + split_to_interleaved(q), + split_size_or_sections=[rope_head_dim, head_dim - rope_head_dim], + dim=-1, + ) + k_rope, k_nope = torch.split( + split_to_interleaved(k), + split_size_or_sections=[rope_head_dim, head_dim - rope_head_dim], + dim=-1, + ) + + q_rope, k_rope = rotary_emb( + positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope) + ) + + q = interleaved_to_split( + torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1) + ).reshape(*q.shape[:-2], -1) + k = interleaved_to_split( + torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1) + ).reshape(*k.shape[:-2], -1) + + return q, k + + +def _rope_weight_perm(head_dim: int, rope_head_dim: int) -> torch.Tensor: + """Per-head gather index that folds K2Horizon's partial-RoPE channel reordering. + + The K2Horizon checkpoint stores each head's channels in a GPT-J-interleaved + convention with rope/nope channels interleaved. Applying this fixed + (position-independent) permutation to the q/k weight rows lets the runtime + use vLLM's *native* NeoX partial-RoPE path (``head_size = head_dim``, + ``rotary_dim = rope_head_dim``) with no per-forward permutes/all-gather. + """ + D, R = head_dim, rope_head_dim + assert R % 2 == 0 and R <= D, ( + f"partial NeoX RoPE requires an even rope_head_dim <= head_dim, " + f"got R={R}, D={D}" + ) + h = D // 2 + idx = torch.cat( + [ + torch.arange(0, R // 2), # rope first-half channels + torch.arange(h, h + R // 2), # rope second-half channels + torch.arange(R // 2, h), # nope remainder (first half) + torch.arange(h + R // 2, D), # nope remainder (second half) + ] + ) + return idx + + def calc_router_weights( router_logits: torch.Tensor, e_score_correction_bias: torch.Tensor | None, @@ -529,12 +620,30 @@ class K2HorizonAttention(nn.Module): ) self.rope_head_dim = rope_head_dim or self.head_dim - self.rotary_emb = get_rope( - self.rope_head_dim, - max_position=max_position_embeddings, - rope_parameters=rope_parameters, - dual_chunk_attention_config=dual_chunk_attention_config, + # Fold the partial-RoPE channel permutation into the q/k weights only + # when supported (dense projections + no dual-chunk attention); + # otherwise keep the runtime channel-permutation path + # (see ``is_rope_weights_folding_supported``). + self.fold_rope_weights = is_rope_weights_folding_supported( + self.qkv_proj, dual_chunk_attention_config ) + if self.fold_rope_weights: + rope_parameters = dict(rope_parameters) + rope_parameters["rope_dim"] = self.rope_head_dim + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + is_neox_style=True, + rope_parameters=rope_parameters, + dual_chunk_attention_config=dual_chunk_attention_config, + ) + else: + self.rotary_emb = get_rope( + self.rope_head_dim, + max_position=max_position_embeddings, + rope_parameters=rope_parameters, + dual_chunk_attention_config=dual_chunk_attention_config, + ) self.attn = Attention( self.num_heads, self.head_dim, @@ -584,40 +693,17 @@ class K2HorizonAttention(nn.Module): q = self.q_norm(q) k = self.k_norm(k) - if self.rope_head_dim == self.head_dim: - q, k = self.rotary_emb(positions, q, k) - else: - q = q.reshape(*q.shape[:-1], self.num_heads, self.head_dim) - k = k.reshape(*k.shape[:-1], self.num_kv_heads, self.head_dim) - - q_rope, q_nope = torch.split( - split_to_interleaved(q), - split_size_or_sections=[ - self.rope_head_dim, - self.head_dim - self.rope_head_dim, - ], - dim=-1, - ) - k_rope, k_nope = torch.split( - split_to_interleaved(k), - split_size_or_sections=[ - self.rope_head_dim, - self.head_dim - self.rope_head_dim, - ], - dim=-1, - ) - - q_rope, k_rope = self.rotary_emb( - positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope) - ) - - q = interleaved_to_split( - torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1) - ).reshape(*q.shape[:-2], -1) - - k = interleaved_to_split( - torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1) - ).reshape(*k.shape[:-2], -1) + q, k = apply_partial_rope( + self.rotary_emb, + positions, + q, + k, + self.num_heads, + self.num_kv_heads, + self.head_dim, + self.rope_head_dim, + self.fold_rope_weights, + ) attn_output = self.attn(q, k, v) @@ -732,12 +818,30 @@ class K2HorizonMoVAAttention(nn.Module): ) self.rope_head_dim = rope_head_dim or self.head_dim - self.rotary_emb = get_rope( - self.rope_head_dim, - max_position=max_position_embeddings, - rope_parameters=rope_parameters, - dual_chunk_attention_config=dual_chunk_attention_config, + # Fold the partial-RoPE channel permutation into the q/k weights only + # when supported (dense projections + no dual-chunk attention); + # otherwise keep the runtime channel-permutation path + # (see ``is_rope_weights_folding_supported``). + self.fold_rope_weights = is_rope_weights_folding_supported( + self.qk_proj, dual_chunk_attention_config ) + if self.fold_rope_weights: + rope_parameters = dict(rope_parameters) + rope_parameters["rope_dim"] = self.rope_head_dim + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + is_neox_style=True, + rope_parameters=rope_parameters, + dual_chunk_attention_config=dual_chunk_attention_config, + ) + else: + self.rotary_emb = get_rope( + self.rope_head_dim, + max_position=max_position_embeddings, + rope_parameters=rope_parameters, + dual_chunk_attention_config=dual_chunk_attention_config, + ) self.attn = Attention( self.num_heads, self.head_dim, @@ -838,40 +942,17 @@ class K2HorizonMoVAAttention(nn.Module): q = self.q_norm(q) k = self.k_norm(k) - if self.rope_head_dim == self.head_dim: - q, k = self.rotary_emb(positions, q, k) - else: - q = q.reshape(*q.shape[:-1], self.num_heads, self.head_dim) - k = k.reshape(*k.shape[:-1], self.num_kv_heads, self.head_dim) - - q_rope, q_nope = torch.split( - split_to_interleaved(q), - split_size_or_sections=[ - self.rope_head_dim, - self.head_dim - self.rope_head_dim, - ], - dim=-1, - ) - k_rope, k_nope = torch.split( - split_to_interleaved(k), - split_size_or_sections=[ - self.rope_head_dim, - self.head_dim - self.rope_head_dim, - ], - dim=-1, - ) - - q_rope, k_rope = self.rotary_emb( - positions, interleaved_to_split(q_rope), interleaved_to_split(k_rope) - ) - - q = interleaved_to_split( - torch.cat([split_to_interleaved(q_rope), q_nope], dim=-1) - ).reshape(*q.shape[:-2], -1) - - k = interleaved_to_split( - torch.cat([split_to_interleaved(k_rope), k_nope], dim=-1) - ).reshape(*k.shape[:-2], -1) + q, k = apply_partial_rope( + self.rotary_emb, + positions, + q, + k, + self.num_heads, + self.num_kv_heads, + self.head_dim, + self.rope_head_dim, + self.fold_rope_weights, + ) attn_output = self.attn(q, k, v) @@ -1109,6 +1190,27 @@ class K2HorizonModel(nn.Module, EagleModelMixin): ("gate_up_proj", "up_proj", 1), ] + # Per-head gather index used to fold K2Horizon's partial-RoPE channel + # reordering into the q_proj/k_proj rows and the q_norm/k_norm scales, + # so the runtime can use vLLM's native partial NeoX RoPE. + head_dim = self.config.head_dim or ( + self.config.hidden_size // self.config.num_attention_heads + ) + rope_head_dim = self.config.rope_head_dim or head_dim + rope_perm_idx = _rope_weight_perm(head_dim, rope_head_dim) + + # Attention modules whose q/k projections are dense: only these fold the + # RoPE channel permutation into their q/k weights, biases and q/k norm + # scales at load time. Attentions with packed/quantized q/k projections + # keep the runtime channel-permutation path, so their weights must be + # loaded verbatim (folding would misalign per-channel scales). + rope_fold_prefixes = tuple( + f"{module_name}." + for module_name, module in self.named_modules() + if isinstance(module, (K2HorizonAttention, K2HorizonMoVAAttention)) + and getattr(module, "fold_rope_weights", False) + ) + # Skip loading extra parameters for GPTQ/modelopt models. # ignore_suffixes = ( # ".bias", @@ -1139,6 +1241,14 @@ class K2HorizonModel(nn.Module, EagleModelMixin): if name not in params_dict: continue + # Fold the per-head RoPE channel permutation into the norm scales. + if self.config.query_key_norm and name.startswith(rope_fold_prefixes): + loaded_weight = ( + loaded_weight.view(-1, head_dim)[:, rope_perm_idx] + .reshape(-1) + .contiguous() + ) + param = params_dict[name] tp_rank = get_tensor_model_parallel_rank() tp_size = get_tensor_model_parallel_world_size() @@ -1203,6 +1313,27 @@ class K2HorizonModel(nn.Module, EagleModelMixin): if ".self_attn." in name and param_name == "gate_up_proj": assert weight_name == "gate_proj" continue + + # Fold the per-head RoPE channel permutation into the dense + # q_proj / k_proj rows (W' = P W), and biases, before TP + # sharding. + if shard_id in ("q", "k") and name.startswith(rope_fold_prefixes): + if name.endswith(".weight"): + hidden = loaded_weight.shape[-1] + loaded_weight = ( + loaded_weight.view(-1, head_dim, hidden)[ + :, rope_perm_idx, : + ] + .reshape(-1, hidden) + .contiguous() + ) + elif self.config.attention_bias and name.endswith(".bias"): + loaded_weight = ( + loaded_weight.view(-1, head_dim)[:, rope_perm_idx] + .reshape(-1) + .contiguous() + ) + if ( ".self_attn." in name and param_name == "qkv_proj" From 3627a6a124896edbc9f802e7c7e120633007e29d Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:16:29 +0200 Subject: [PATCH 097/248] [ROCm][Perf][GLM-5.3-Flash] Stride-aware decode KDA (#57979) Signed-off-by: simondanielsson --- tests/models/glm5next/test_kda_recurrent.py | 7 ++- .../ops/third_party/kda/fused_recurrent.py | 44 +++++++++++++++---- .../amd/ops/third_party/kda/kernels.py | 22 +++++++--- 3 files changed, 54 insertions(+), 19 deletions(-) diff --git a/tests/models/glm5next/test_kda_recurrent.py b/tests/models/glm5next/test_kda_recurrent.py index ddda0e0c34..fe017e4131 100644 --- a/tests/models/glm5next/test_kda_recurrent.py +++ b/tests/models/glm5next/test_kda_recurrent.py @@ -5,8 +5,7 @@ The decode path hands the kernel column slices of the merged ``q|k|v`` conv output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are token-strided rather than contiguous. Both backends must match a pure-PyTorch -recurrence. CUDA reads these slices in place and rejects unsupported layouts; -the ROCm wrapper makes contiguous copies before launching its kernel. +recurrence, read these slices in place and reject unsupported layouts. """ import pytest @@ -166,8 +165,8 @@ def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous( @pytest.mark.skipif( - not current_platform.is_cuda(), - reason="Only the CUDA implementation requires zero-copy token-strided layouts", + not current_platform.is_cuda_alike(), + reason="Both implementations require zero-copy token-strided layouts", ) @torch.inference_mode() def test_fused_recurrent_kda_rejects_unaddressable_layouts(): diff --git a/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py b/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py index d8b25211e0..a9d0fba476 100644 --- a/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py +++ b/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py @@ -15,6 +15,23 @@ from vllm.third_party.flash_linear_attention.ops.op import exp from vllm.triton_utils import tl, triton +def token_stride(x: torch.Tensor) -> int: + """Token stride (elements) of a ``[B, T, H, D]`` or ``[B, T, H]`` tensor. + + The recurrent kernel walks tokens with this stride and addresses heads + densely inside a token, so each token's ``[H, D]`` (or ``[H]``) block must + be contiguous, tokens must not overlap, and with ``B > 1`` sequence ``n`` + must start at token ``n * T`` (dense batch). Column slices of a wider + per-token projection buffer satisfy this and are consumed in place. + """ + st = x.stride() + assert x.dim() in (3, 4) and st[-1] == 1, (x.shape, st) + assert x.dim() == 3 or st[2] == x.shape[3], (x.shape, st) + assert st[1] >= x.shape[2] * (x.shape[3] if x.dim() == 4 else 1), (x.shape, st) + assert x.shape[0] == 1 or st[0] == x.shape[1] * st[1], (x.shape, st) + return st[1] + + @triton.heuristics( { "USE_INITIAL_STATE": lambda args: args["h0"] is not None, @@ -52,6 +69,11 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( stride_final_state_token: tl.constexpr, stride_indices_seq: tl.constexpr, stride_indices_tok: tl.constexpr, + # Token strides of q/k/v/beta (elements), see `token_stride`. + stride_q_t, + stride_k_t, + stride_v_t, + stride_beta_t, USE_INITIAL_STATE: tl.constexpr, # whether to use initial state INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar, @@ -86,13 +108,13 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( o_k = i_k * BK + tl.arange(0, BK) o_v = i_v * BV + tl.arange(0, BV) - p_q = q + (bos * H + i_h) * K + o_k - p_k = k + (bos * H + i_h) * K + o_k - p_v = v + (bos * HV + i_hv) * V + o_v + p_q = q + bos * stride_q_t + i_h * K + o_k + p_k = k + bos * stride_k_t + i_h * K + o_k + p_v = v + bos * stride_v_t + i_hv * V + o_v if IS_BETA_HEADWISE: - p_beta = beta + (bos * HV + i_hv) * V + o_v + p_beta = beta + bos * stride_beta_t + i_hv * V + o_v else: - p_beta = beta + bos * HV + i_hv + p_beta = beta + bos * stride_beta_t + i_hv if not IS_KDA: p_g = g + bos * HV + i_hv @@ -188,15 +210,15 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :] tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) - p_q += H * K - p_k += H * K + p_q += stride_q_t + p_k += stride_k_t p_o += HV * V - p_v += HV * V + p_v += stride_v_t if not IS_KDA: p_g += HV else: p_gk += HV * K - p_beta += HV * (V if IS_BETA_HEADWISE else 1) + p_beta += stride_beta_t def fused_recurrent_gated_delta_rule_fwd( @@ -265,6 +287,10 @@ def fused_recurrent_gated_delta_rule_fwd( stride_final_state_token=stride_final_state_token, stride_indices_seq=stride_indices_seq, stride_indices_tok=stride_indices_tok, + stride_q_t=token_stride(q), + stride_k_t=token_stride(k), + stride_v_t=token_stride(v), + stride_beta_t=token_stride(beta), IS_BETA_HEADWISE=beta.ndim == v.ndim, USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, INPLACE_FINAL_STATE=inplace_final_state, diff --git a/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py b/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py index ea86750e7d..9805b37471 100644 --- a/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py +++ b/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py @@ -24,7 +24,10 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is from vllm.triton_utils import tl, triton from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2 -from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel +from .fused_recurrent import ( + fused_recurrent_gated_delta_rule_fwd_kernel, + token_stride, +) BT_LIST_AUTOTUNE = [32, 64, 128] NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32] @@ -70,7 +73,7 @@ def fused_recurrent_kda_fwd( g_bias = g_bias.reshape(-1).contiguous() if out is None: - o = torch.empty_like(k) + o = torch.empty(k.shape, dtype=k.dtype, device=k.device) else: # Caller-provided output buffer; must be layout-compatible with the # tensor the kernel indexes (contiguous, same shape/dtype as k). @@ -119,6 +122,10 @@ def fused_recurrent_kda_fwd( stride_final_state_token=stride_final_state_token, stride_indices_seq=stride_indices_seq, stride_indices_tok=stride_indices_tok, + stride_q_t=token_stride(q), + stride_k_t=token_stride(k), + stride_v_t=token_stride(v), + stride_beta_t=token_stride(beta), IS_BETA_HEADWISE=beta.ndim == v.ndim, USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, INPLACE_FINAL_STATE=inplace_final_state, @@ -165,12 +172,15 @@ def fused_recurrent_kda( if scale is None: scale = k.shape[-1] ** -0.5 + # q/k/v/beta are consumed in place with an explicit token stride, so + # column slices of the fused projection buffer need no copy; layouts the + # kernel cannot address fail loudly in `token_stride`. o, final_state = fused_recurrent_kda_fwd( - q=q.contiguous(), - k=k.contiguous(), - v=v.contiguous(), + q=q, + k=k, + v=v, g=g.contiguous(), - beta=beta.contiguous(), + beta=beta, scale=scale, initial_state=initial_state, inplace_final_state=inplace_final_state, From fb91712b38a68bb751526ad53552b91226290edd Mon Sep 17 00:00:00 2001 From: Avishek Goswami <86944690+GOavi101@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:27:21 +0530 Subject: [PATCH 098/248] [Model] Make the BERT/RoBERTa embedding class a class attribute (#59348) Signed-off-by: Avishek Goswami Signed-off-by: Avishek Goswami <86944690+GOavi101@users.noreply.github.com> Co-authored-by: Avishek Goswami --- vllm/model_executor/models/bert.py | 13 +++++++++---- vllm/model_executor/models/roberta.py | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/vllm/model_executor/models/bert.py b/vllm/model_executor/models/bert.py index 7348ec9383..8f061b7284 100644 --- a/vllm/model_executor/models/bert.py +++ b/vllm/model_executor/models/bert.py @@ -461,6 +461,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant): """ is_pooling_model = True + embedding_class: type[nn.Module] = BertEmbedding def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -504,7 +505,7 @@ class BertEmbeddingModel(nn.Module, SupportsQuant): def _build_model(self, vllm_config: VllmConfig, prefix: str = "") -> BertModel: return BertModel( - vllm_config=vllm_config, prefix=prefix, embedding_class=BertEmbedding + vllm_config=vllm_config, prefix=prefix, embedding_class=self.embedding_class ) def _build_pooler(self, pooler_config: PoolerConfig) -> Pooler: @@ -781,6 +782,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu """ is_pooling_model = True + embedding_class: type[nn.Module] = BertEmbedding def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -790,7 +792,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu self.bert = BertPoolingModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "bert"), - embedding_class=BertEmbedding, + embedding_class=self.embedding_class, ) self.classifier = nn.Linear( config.hidden_size, @@ -840,6 +842,7 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu @default_pooling_type(tok_pooling_type="ALL") class BertForTokenClassification(nn.Module): is_pooling_model = True + embedding_class: type[nn.Module] = BertEmbedding def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -849,7 +852,7 @@ class BertForTokenClassification(nn.Module): self.bert = BertModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "bert"), - embedding_class=BertEmbedding, + embedding_class=self.embedding_class, ) self.classifier = nn.Linear( config.hidden_size, config.num_labels, dtype=self.head_dtype @@ -927,13 +930,15 @@ class BertForMaskedLM(nn.Module): } ) + embedding_class: type[nn.Module] = BertEmbedding + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config self.bert = BertModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "bert"), - embedding_class=BertEmbedding, + embedding_class=self.embedding_class, ) self.mlm_head = BertMLMHead( hidden_size=config.hidden_size, diff --git a/vllm/model_executor/models/roberta.py b/vllm/model_executor/models/roberta.py index 8e141229f0..a694e7c3ce 100644 --- a/vllm/model_executor/models/roberta.py +++ b/vllm/model_executor/models/roberta.py @@ -117,6 +117,8 @@ class RobertaClassificationHead(nn.Module): class RobertaEmbeddingModel(BertEmbeddingModel): """A model that uses Roberta to provide embedding functionalities.""" + embedding_class: type[nn.Module] = RobertaEmbedding + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__(vllm_config=vllm_config, prefix=prefix) self.padding_idx: int = vllm_config.model_config.hf_config.pad_token_id @@ -143,7 +145,7 @@ class RobertaEmbeddingModel(BertEmbeddingModel): return BertModel( vllm_config=vllm_config, prefix=prefix, - embedding_class=RobertaEmbedding, + embedding_class=self.embedding_class, ) else: return JinaRobertaModel(vllm_config=vllm_config, prefix=prefix) @@ -293,6 +295,8 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, Support } ) + embedding_class: type[nn.Module] = RobertaEmbedding + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config @@ -303,7 +307,7 @@ class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding, Support self.roberta = BertModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "bert"), - embedding_class=RobertaEmbedding, + embedding_class=self.embedding_class, ) self.classifier = RobertaClassificationHead(vllm_config.model_config) @@ -355,6 +359,7 @@ class RobertaForTokenClassification(nn.Module): """ is_pooling_model = True + embedding_class: type[nn.Module] = RobertaEmbedding def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -364,7 +369,7 @@ class RobertaForTokenClassification(nn.Module): self.roberta = BertModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "roberta"), - embedding_class=RobertaEmbedding, + embedding_class=self.embedding_class, ) self.classifier = nn.Linear( config.hidden_size, config.num_labels, dtype=self.head_dtype From ae66f9a97c6db39bb7225d7b0569acf6d2fdc300 Mon Sep 17 00:00:00 2001 From: Akash kaothalkar <61960177+Akashcodes732@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:44:17 +0530 Subject: [PATCH 099/248] [Hardware][PowerPC] Prioritize bfloat16 for auto dtype on PowerPC (#58528) Signed-off-by: Akash kaothalkar Co-authored-by: Akash kaothalkar Co-authored-by: Antigravity Co-authored-by: Li, Jiang --- vllm/config/model.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index 15d046de76..4c9ed9e13f 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -26,7 +26,7 @@ from vllm.config.quantization import QuantizationConfigArgs from vllm.config.scheduler import RunnerType from vllm.config.utils import config, getattr_iter from vllm.logger import init_logger -from vllm.platforms import current_platform +from vllm.platforms import CpuArchEnum, current_platform from vllm.tasks import PoolingTask, ScoreType, SupportedTask from vllm.transformers_utils.config import ( ConfigFormat, @@ -2419,6 +2419,12 @@ def _get_and_verify_dtype( config_dtype, is_pooling_model=is_pooling_model, ) + if ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + and torch_dtype in (torch.float16, torch.float32) + ): + torch_dtype = torch.bfloat16 else: if dtype not in _STR_DTYPE_TO_TORCH_DTYPE: raise ValueError(f"Unknown dtype: {dtype!r}") From 713ec07655a3d39daeeff0fd2fcb9dd251c73979 Mon Sep 17 00:00:00 2001 From: Ronald Date: Wed, 30 Sep 2026 17:19:43 +0800 Subject: [PATCH 100/248] [Frontend][RL] Track HTTP weight operation outcomes and concurrency (#55781) Signed-off-by: Ronald1995 Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 Co-authored-by: aoshen02 --- rust/src/metrics/src/api_server.rs | 77 +++++++ rust/src/server/src/routes/tests.rs | 34 ++++ rust/src/server/src/routes/weight_transfer.rs | 6 + .../dev/rlhf/test_weight_operation_metrics.py | 87 ++++++++ .../serve/lora/test_serving_models.py | 18 +- .../test_rl_weight_operation_metrics.py | 188 ++++++++++++++++++ vllm/entrypoints/serve/dev/rlhf/api_router.py | 24 ++- vllm/entrypoints/serve/dev/rlhf/metrics.py | 56 ++++++ 8 files changed, 476 insertions(+), 14 deletions(-) create mode 100644 tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py create mode 100644 tests/entrypoints/unit_tests/test_rl_weight_operation_metrics.py create mode 100644 vllm/entrypoints/serve/dev/rlhf/metrics.py diff --git a/rust/src/metrics/src/api_server.rs b/rust/src/metrics/src/api_server.rs index 1dc7154998..51599ee51e 100644 --- a/rust/src/metrics/src/api_server.rs +++ b/rust/src/metrics/src/api_server.rs @@ -23,6 +23,10 @@ fn http_request_duration_highr_histogram() -> Histogram { Histogram::new(HTTP_REQUEST_DURATION_HIGHR_BUCKETS.iter().copied()) } +fn weight_operation_duration_histogram() -> Histogram { + Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0]) +} + fn sleep_mode_operation_duration_histogram() -> Histogram { Histogram::new([0.01, 0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0]) } @@ -40,6 +44,11 @@ pub struct HttpHandlerLabels { pub handler: String, } +#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] +pub struct WeightOperationLabels { + pub operation: &'static str, +} + #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] pub struct SleepModeOperationLabels { pub operation: &'static str, @@ -48,6 +57,9 @@ pub struct SleepModeOperationLabels { pub(crate) type HttpRequestCounterFamily = Family; pub(crate) type HttpHandlerHistogramFamily = Family Histogram>; +pub(crate) type WeightOperationHistogramFamily = + Family Histogram>; +pub(crate) type WeightOperationGaugeFamily = Family; pub(crate) type SleepModeOperationHistogramFamily = Family Histogram>; pub(crate) type SleepModeOperationGaugeFamily = Family; @@ -57,6 +69,8 @@ pub struct ApiServerMetrics { pub http_requests: HttpRequestCounterFamily, pub http_request_duration_seconds: HttpHandlerHistogramFamily, pub http_request_duration_highr_seconds: Histogram, + pub weight_operation_duration_seconds: WeightOperationHistogramFamily, + pub weight_operations_in_flight: WeightOperationGaugeFamily, pub sleep_mode_operation_duration_seconds: SleepModeOperationHistogramFamily, pub sleep_mode_operations_in_flight: SleepModeOperationGaugeFamily, } @@ -87,6 +101,21 @@ impl ApiServerMetrics { http_request_duration_highr_seconds.clone(), ); + let weight_operation_duration_seconds = + Family::new_with_constructor(weight_operation_duration_histogram as fn() -> Histogram); + registry.register( + "vllm:rl_weight_update_operation_duration_seconds", + "Duration of one weight-transfer operation.", + weight_operation_duration_seconds.clone(), + ); + + let weight_operations_in_flight = WeightOperationGaugeFamily::default(); + registry.register( + "vllm:rl_weight_update_operations_in_flight", + "Weight-transfer operations currently awaited.", + weight_operations_in_flight.clone(), + ); + let sleep_mode_operation_duration_seconds = Family::new_with_constructor( sleep_mode_operation_duration_histogram as fn() -> Histogram, ); @@ -107,11 +136,25 @@ impl ApiServerMetrics { http_requests, http_request_duration_seconds, http_request_duration_highr_seconds, + weight_operation_duration_seconds, + weight_operations_in_flight, sleep_mode_operation_duration_seconds, sleep_mode_operations_in_flight, } } + /// Track one weight-transfer operation until the returned guard is dropped. + pub fn record_weight_operation(&self, operation: &'static str) -> WeightOperationRecorder { + let labels = WeightOperationLabels { operation }; + let in_flight = self.weight_operations_in_flight.get_or_create_owned(&labels); + in_flight.inc(); + WeightOperationRecorder { + started_at: Instant::now(), + duration: self.weight_operation_duration_seconds.get_or_create_owned(&labels), + in_flight, + } + } + /// Record one dispatched sleep-mode engine operation. pub fn record_sleep_mode_operation( &self, @@ -128,6 +171,19 @@ impl ApiServerMetrics { } } +pub struct WeightOperationRecorder { + started_at: Instant, + duration: Histogram, + in_flight: U64Gauge, +} + +impl Drop for WeightOperationRecorder { + fn drop(&mut self) { + self.in_flight.dec(); + self.duration.observe(self.started_at.elapsed().as_secs_f64()); + } +} + /// Records duration and clears in-flight tracking on completion or cancellation. pub struct SleepModeOperationRecorder { started_at: Instant, @@ -194,4 +250,25 @@ mod tests { ))); } } + + #[test] + fn weight_operation_recorder_tracks_in_flight_and_duration() { + let mut registry = Registry::default(); + let metrics = ApiServerMetrics::register(&mut registry); + + let recorder = metrics.record_weight_operation("update"); + assert!( + rendered_metrics(®istry) + .contains("vllm:rl_weight_update_operations_in_flight{operation=\"update\"} 1") + ); + drop(recorder); + + let after = rendered_metrics(®istry); + assert!( + after.contains("vllm:rl_weight_update_operations_in_flight{operation=\"update\"} 0") + ); + assert!(after.contains( + "vllm:rl_weight_update_operation_duration_seconds_count{operation=\"update\"} 1" + )); + } } diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index d229be25fa..7225f2bebf 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -6723,6 +6723,7 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() { }) }) .await; + let metrics_before = METRICS.render().unwrap(); let mut responses = Vec::new(); for (method, path, body) in [ @@ -6785,6 +6786,26 @@ async fn weight_transfer_routes_support_the_http_training_lifecycle() { ); } engine_task.await.expect("mock engine task"); + let metrics_after = METRICS.render().unwrap(); + for operation in ["update", "finish"] { + assert_eq!( + metric_delta( + &metrics_before, + &metrics_after, + "vllm:rl_weight_update_operation_duration_seconds_count", + Some(&format!("operation=\"{operation}\"")), + ), + 2.0 + ); + assert_eq!( + metric_value( + &metrics_after, + "vllm:rl_weight_update_operations_in_flight", + Some(&format!("operation=\"{operation}\"")), + ), + Some(0.0) + ); + } expect_test::expect![[r#" [ "{\"message\":\"Weight transfer initialized\"}", @@ -6811,6 +6832,7 @@ async fn weight_transfer_routes_reject_invalid_payloads_before_engine_calls() { }) }) .await; + let metrics_before = METRICS.render().unwrap(); for (path, body) in [ ("/init_weight_transfer_engine", "{"), @@ -6840,6 +6862,18 @@ async fn weight_transfer_routes_reject_invalid_payloads_before_engine_calls() { .expect("call app"); assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}: {body}"); } + let metrics_after = METRICS.render().unwrap(); + for operation in ["init", "update", "finish"] { + assert_eq!( + metric_delta( + &metrics_before, + &metrics_after, + "vllm:rl_weight_update_operation_duration_seconds_count", + Some(&format!("operation=\"{operation}\"")), + ), + 0.0 + ); + } engine_task.abort_and_join().await; } diff --git a/rust/src/server/src/routes/weight_transfer.rs b/rust/src/server/src/routes/weight_transfer.rs index df55e25593..5ccd66e39b 100644 --- a/rust/src/server/src/routes/weight_transfer.rs +++ b/rust/src/server/src/routes/weight_transfer.rs @@ -8,6 +8,7 @@ use axum::extract::State; use axum::extract::rejection::JsonRejection; use serde::{Deserialize, Serialize}; use serde_json::Value as JsonValue; +use vllm_metrics::METRICS; use crate::error::{ApiError, invalid_request}; use crate::state::AppState; @@ -62,6 +63,7 @@ pub async fn init_weight_transfer_engine( ) })?; + let _recorder = METRICS.api_server.record_weight_operation("init"); state .engine_core_client() .init_weight_transfer_engine(init_info) @@ -77,6 +79,7 @@ pub async fn init_weight_transfer_engine( pub async fn start_weight_update( State(state): State>, ) -> Result, ApiError> { + let _recorder = METRICS.api_server.record_weight_operation("start"); state .engine_core_client() .start_weight_update() @@ -92,6 +95,7 @@ pub async fn start_weight_update( pub async fn start_draft_weight_update( State(state): State>, ) -> Result, ApiError> { + let _recorder = METRICS.api_server.record_weight_operation("start_draft"); state .engine_core_client() .start_draft_weight_update() @@ -122,6 +126,7 @@ pub async fn update_weights( ) })?; + let _recorder = METRICS.api_server.record_weight_operation("update"); state .engine_core_client() .update_weights(update_info) @@ -142,6 +147,7 @@ pub async fn finish_weight_update( let request = body?.map(|Json(request)| request).unwrap_or_default(); let client = state.engine_core_client(); + let _recorder = METRICS.api_server.record_weight_operation("finish"); client .finish_weight_update() .await diff --git a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py b/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py new file mode 100644 index 0000000000..6c7cb60bbb --- /dev/null +++ b/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Weight-transfer metrics scraped from a real two-API-server deployment.""" + +from pathlib import Path + +import requests +import torch +from prometheus_client.multiprocess import MultiProcessCollector +from prometheus_client.parser import text_string_to_metric_families +from transformers import AutoModelForCausalLM + +from tests.entrypoints.serve.dev.rlhf.conftest import MODEL_NAME +from tests.utils import RemoteOpenAIServer +from vllm.distributed.weight_transfer import ( + HTTPVLLMWeightSyncClient, + ModuleSource, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo + +DURATION = "vllm:rl_weight_update_operation_duration_seconds" +IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight" +# Enough requests that the kernel spreads them over both API servers. +SYNCS = 8 + + +def scrape(url: str) -> dict[tuple[str, str], float]: + text = requests.get(f"{url}/metrics", timeout=30).text + return { + (sample.name, sample.labels["operation"]): sample.value + for family in text_string_to_metric_families(text) + for sample in family.samples + if sample.name in (f"{DURATION}_count", IN_FLIGHT) + } + + +def pids_that_recorded(multiproc_dir: Path) -> set[str]: + return { + path.stem.rsplit("_", 1)[1] + for path in multiproc_dir.glob("histogram_*.db") + if any( + metric.name == DURATION + for metric in MultiProcessCollector.merge([str(path)]) + ) + } + + +def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path): + args = [ + "--load-format", + "dummy", + "--enforce-eager", + "--max-model-len", + "512", + "--gpu-memory-utilization", + "0.5", + "--api-server-count", + "2", + "--weight-transfer-config", + '{"backend": "ipc"}', + ] + env = { + "VLLM_SERVER_DEV_MODE": "1", + "VLLM_ALLOW_INSECURE_SERIALIZATION": "1", + "PROMETHEUS_MULTIPROC_DIR": str(tmp_path), + } + with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env) as server: + url = server.url_root + trainer = AutoModelForCausalLM.from_pretrained( + MODEL_NAME, torch_dtype=torch.bfloat16 + ).cuda() + engine = WeightTransferTrainerFactory.trainer_init( + IPCTrainerInitInfo(rank=0), + client=HTTPVLLMWeightSyncClient(url), + source=ModuleSource(trainer), + ) + for _ in range(SYNCS): + engine.send_weights() + metrics = scrape(url) + + calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS} + for operation, count in calls.items(): + assert metrics[(f"{DURATION}_count", operation)] == count, operation + assert metrics[(IN_FLIGHT, operation)] == 0, operation + # The counts above are sums over processes only if both servers recorded. + assert len(pids_that_recorded(tmp_path)) == 2 diff --git a/tests/entrypoints/serve/lora/test_serving_models.py b/tests/entrypoints/serve/lora/test_serving_models.py index b2aefff219..2fb39b53ab 100644 --- a/tests/entrypoints/serve/lora/test_serving_models.py +++ b/tests/entrypoints/serve/lora/test_serving_models.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from http import HTTPStatus -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest @@ -29,7 +29,9 @@ LORA_UNLOADING_SUCCESS_MESSAGE = ( ) -async def _async_serving_models_init() -> OpenAIServingModels: +async def _async_serving_models_init_with_mock() -> tuple[ + OpenAIServingModels, MagicMock +]: mock_engine_client = MagicMock(spec=EngineClient) # Set the max_model_len attribute to avoid missing attribute mock_model_config = MagicMock(spec=ModelConfig) @@ -45,6 +47,13 @@ async def _async_serving_models_init() -> OpenAIServingModels: ) await serving_models.init_static_loras() + # The mock is returned separately: `engine_client` is typed as the protocol, + # so assertion helpers are not visible through it. + return serving_models, mock_engine_client + + +async def _async_serving_models_init() -> OpenAIServingModels: + serving_models, _ = await _async_serving_models_init_with_mock() return serving_models @@ -103,7 +112,7 @@ async def test_load_lora_adapter_duplicate(): @pytest.mark.parametrize("load_inplace", [False, True]) async def test_load_lora_adapter_base_model_name(load_inplace: bool): """A LoRA named after a served model would shadow it for all requests.""" - serving_models = await _async_serving_models_init() + serving_models, mock_engine_client = await _async_serving_models_init_with_mock() request = LoadLoRAAdapterRequest( lora_name=MODEL_NAME, lora_path="/path/to/adapter", load_inplace=load_inplace ) @@ -112,8 +121,7 @@ async def test_load_lora_adapter_base_model_name(load_inplace: bool): assert response.error.type == "InvalidUserInput" assert response.error.code == HTTPStatus.BAD_REQUEST assert len(serving_models.lora_requests) == 0 - assert isinstance(serving_models.engine_client.add_lora, AsyncMock) - serving_models.engine_client.add_lora.assert_not_called() + mock_engine_client.add_lora.assert_not_called() @pytest.mark.asyncio diff --git a/tests/entrypoints/unit_tests/test_rl_weight_operation_metrics.py b/tests/entrypoints/unit_tests/test_rl_weight_operation_metrics.py new file mode 100644 index 0000000000..bdddf3d247 --- /dev/null +++ b/tests/entrypoints/unit_tests/test_rl_weight_operation_metrics.py @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +import os +import subprocess +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from prometheus_client import CollectorRegistry + +from vllm.entrypoints.serve.dev.rlhf import api_router +from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics +from vllm.entrypoints.serve.dev.rlhf.metrics import WeightOperationMetrics +from vllm.entrypoints.serve.exception_handling.register import init_exception_handler +from vllm.v1.metrics.prometheus import get_prometheus_registry + +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] + +DURATION = "vllm:rl_weight_update_operation_duration_seconds_count" +IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight" + + +def sample(registry, name, operation): + return registry.get_sample_value(name, {"operation": operation}) + + +class Request: + def __init__(self, engine): + self.app = SimpleNamespace(state=SimpleNamespace(engine_client=engine)) + + +class Engine: + """Holds every engine call open until the test releases it.""" + + def __init__(self): + self.started = asyncio.Event() + self.release = asyncio.Event() + + def __getattr__(self, name): + async def call(*args, **kwargs): + self.started.set() + await self.release.wait() + + return call + + +@pytest.fixture +def registry(monkeypatch): + registry = CollectorRegistry() + monkeypatch.setattr(rlhf_metrics, "_metrics", WeightOperationMetrics(registry)) + return registry + + +@pytest.mark.parametrize("fail", [False, True]) +def test_record_releases_in_flight_and_observes_duration(registry, fail): + metrics = rlhf_metrics.weight_operation_metrics() + with ( + pytest.raises(RuntimeError) if fail else nullcontext(), + metrics.record("update"), + ): + assert sample(registry, IN_FLIGHT, "update") == 1 + if fail: + raise RuntimeError + assert sample(registry, IN_FLIGHT, "update") == 0 + assert sample(registry, DURATION, "update") == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "operation,route,body", + [ + ("init", api_router.init_weight_transfer_engine, {"init_info": {}}), + ("start", api_router.start_weight_update, None), + ("start_draft", api_router.start_draft_weight_update, None), + ("update", api_router.update_weights, {"update_info": {}}), + ("finish", api_router.finish_weight_update, None), + ], +) +async def test_route_records_its_operation(registry, operation, route, body): + engine = Engine() + task = asyncio.create_task(route(Request(engine), **(body or {}))) + await engine.started.wait() + assert sample(registry, IN_FLIGHT, operation) == 1 + engine.release.set() + await task + assert sample(registry, IN_FLIGHT, operation) == 0 + assert sample(registry, DURATION, operation) == 1 + + +@pytest.mark.asyncio +async def test_cancelled_route_releases_in_flight(registry): + engine = Engine() + task = asyncio.create_task( + api_router.update_weights(Request(engine), update_info={}) + ) + await engine.started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert sample(registry, IN_FLIGHT, "update") == 0 + assert sample(registry, DURATION, "update") == 1 + + +HOLD_OPERATION = """ +import sys +from vllm.entrypoints.serve.dev.rlhf.metrics import weight_operation_metrics +with weight_operation_metrics().record("update"): + print("entered", flush=True) + sys.stdin.readline() +""" + + +def test_in_flight_is_summed_across_api_server_processes(tmp_path, monkeypatch): + """Two API-server processes each hold an operation; /metrics sums them.""" + monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path)) + servers = [ + subprocess.Popen( + [sys.executable, "-c", HOLD_OPERATION], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + env=os.environ.copy(), + ) + for _ in range(2) + ] + try: + for server in servers: + assert server.stdout is not None + assert server.stdout.readline().strip() == "entered" + assert sample(get_prometheus_registry(), IN_FLIGHT, "update") == 2 + finally: + for server in servers: + server.communicate("\n", timeout=60) + + scraped = get_prometheus_registry() + assert sample(scraped, IN_FLIGHT, "update") == 0 + assert sample(scraped, DURATION, "update") == 2 + + +def test_rejected_request_is_not_recorded(registry): + app = FastAPI() + app.include_router(api_router.router) + init_exception_handler(app) + app.state.engine_client = Engine() + app.state.args = SimpleNamespace(log_error_stack=False) + response = TestClient(app).post("/update_weights", json={}) + assert response.status_code == 400 + assert sample(registry, DURATION, "update") is None + + +def test_default_recorder_is_created_lazily_on_the_default_registry(): + """The default recorder registers on the process registry only when first used. + + Runs in a fresh interpreter: the registry is process-global, so a route test + elsewhere in the session may already have registered these collectors, and + "not registered yet" would then be unobservable. + """ + probe = """ +from prometheus_client import REGISTRY + +from vllm.entrypoints.serve.dev.rlhf import metrics as rlhf_metrics + +NAMES = [ + "vllm:rl_weight_update_operation_duration_seconds", + "vllm:rl_weight_update_operations_in_flight", +] +assert rlhf_metrics._metrics is None +assert not set(NAMES) & set(REGISTRY._names_to_collectors) + +metrics = rlhf_metrics.weight_operation_metrics() +assert REGISTRY._names_to_collectors[NAMES[0]] is metrics.duration +assert REGISTRY._names_to_collectors[NAMES[1]] is metrics.in_flight +assert rlhf_metrics.weight_operation_metrics() is metrics +print("lazy registration ok") +""" + result = subprocess.run( + [sys.executable, "-c", probe], + cwd=Path(__file__).parents[3], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert "lazy registration ok" in result.stdout diff --git a/vllm/entrypoints/serve/dev/rlhf/api_router.py b/vllm/entrypoints/serve/dev/rlhf/api_router.py index 9c464e6a11..7e52e51217 100644 --- a/vllm/entrypoints/serve/dev/rlhf/api_router.py +++ b/vllm/entrypoints/serve/dev/rlhf/api_router.py @@ -14,6 +14,7 @@ from vllm.distributed.weight_transfer.base import ( WeightTransferUpdateRequest, ) from vllm.engine.protocol import EngineClient +from vllm.entrypoints.serve.dev.rlhf.metrics import weight_operation_metrics from vllm.logger import init_logger from vllm.v1.engine import PauseMode @@ -163,21 +164,24 @@ async def init_weight_transfer_engine( raw_request: Request, init_info: Annotated[dict[str, Any], Body(embed=True)], ): - await engine_client(raw_request).init_weight_transfer_engine( - WeightTransferInitRequest(init_info=init_info) - ) + with weight_operation_metrics().record("init"): + await engine_client(raw_request).init_weight_transfer_engine( + WeightTransferInitRequest(init_info=init_info) + ) return JSONResponse(content={"message": "Weight transfer initialized"}) @router.post("/start_weight_update") async def start_weight_update(raw_request: Request): - await engine_client(raw_request).start_weight_update() + with weight_operation_metrics().record("start"): + await engine_client(raw_request).start_weight_update() return JSONResponse(content={"message": "Weight update started"}) @router.post("/start_draft_weight_update") async def start_draft_weight_update(raw_request: Request): - await engine_client(raw_request).start_draft_weight_update() + with weight_operation_metrics().record("start_draft"): + await engine_client(raw_request).start_draft_weight_update() return JSONResponse(content={"message": "Draft weight update started"}) @@ -186,9 +190,10 @@ async def update_weights( raw_request: Request, update_info: Annotated[WeightTransferUpdatePayload, Body(embed=True)], ): - await engine_client(raw_request).update_weights( - request=WeightTransferUpdateRequest(update_info=update_info) - ) + with weight_operation_metrics().record("update"): + await engine_client(raw_request).update_weights( + request=WeightTransferUpdateRequest(update_info=update_info) + ) return JSONResponse(content={"message": "Weights updated"}) @@ -197,7 +202,8 @@ async def finish_weight_update( raw_request: Request, weight_version: Annotated[str | None, Body(embed=True)] = None, ): - await engine_client(raw_request).finish_weight_update(weight_version) + with weight_operation_metrics().record("finish"): + await engine_client(raw_request).finish_weight_update(weight_version) return JSONResponse(content={"message": "Weight update finished"}) diff --git a/vllm/entrypoints/serve/dev/rlhf/metrics.py b/vllm/entrypoints/serve/dev/rlhf/metrics.py new file mode 100644 index 0000000000..ed16f9a820 --- /dev/null +++ b/vllm/entrypoints/serve/dev/rlhf/metrics.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Duration and in-flight metrics for RL weight-transfer operations.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from threading import Lock +from time import perf_counter +from typing import Literal + +from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram + +Operation = Literal["init", "start", "start_draft", "update", "finish"] + + +class WeightOperationMetrics: + def __init__(self, registry: CollectorRegistry = REGISTRY): + self.duration = Histogram( + "vllm:rl_weight_update_operation_duration_seconds", + "Duration of one weight-transfer operation.", + ["operation"], + buckets=(0.01, 0.1, 1, 10, 30, 60, 120, 300, 600), + registry=registry, + ) + self.in_flight = Gauge( + "vllm:rl_weight_update_operations_in_flight", + "Weight-transfer operations currently awaited.", + ["operation"], + multiprocess_mode="livesum", + registry=registry, + ) + + @contextmanager + def record(self, operation: Operation) -> Iterator[None]: + in_flight = self.in_flight.labels(operation) + in_flight.inc() + started = perf_counter() + try: + yield + finally: + in_flight.dec() + self.duration.labels(operation).observe(perf_counter() - started) + + +_metrics: WeightOperationMetrics | None = None +_metrics_lock = Lock() + + +def weight_operation_metrics() -> WeightOperationMetrics: + # Lazy, so the collectors exist only after multiprocess Prometheus setup. + global _metrics + if _metrics is None: + with _metrics_lock: + if _metrics is None: + _metrics = WeightOperationMetrics() + return _metrics From 749192ff41f4fddd6e5d3c4ef1eb5851175e8fc2 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 30 Sep 2026 17:20:32 +0800 Subject: [PATCH 101/248] [CI] Skip the IPC weight-checker test on non-CUDA platforms (#59398) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- .../serve/dev/rlhf/state_transitions/test_weight_checker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py index 262b022ef5..d2fb97174e 100644 --- a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py +++ b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py @@ -23,6 +23,7 @@ from vllm.distributed.weight_transfer import ( WeightTransferTrainerFactory, ) from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo +from vllm.platforms import current_platform def weight_checker(url: str, action: str, baseline=None) -> requests.Response: @@ -96,6 +97,9 @@ class TestSingleGPU: assert ok(gen(server_url)) +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="IPC weight transfer uses CUDA IPC." +) def test_ipc_weight_transfer_restores_reset_weights(): """The documented RL flow: reset, then transfer the checkpoint back.""" args = ["--weight-transfer-config", '{"backend": "ipc"}'] From 16d4ac4ba3a0a7d0b6a5eb11e203cd171a2d64a1 Mon Sep 17 00:00:00 2001 From: "yhcheong (vllmellm)" Date: Wed, 30 Sep 2026 17:23:00 +0800 Subject: [PATCH 102/248] [ROCm][MoE] Support MiMo-V2.6 MXFP4 on gfx942 (#58262) Signed-off-by: vllmellm --- .../test_mimo_v2_w4a16_routing.py | 176 ++++++++++++++++++ .../experts/aiter_mxfp4_w4a16_moe.py | 21 ++- 2 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 tests/models/quantization/test_mimo_v2_w4a16_routing.py diff --git a/tests/models/quantization/test_mimo_v2_w4a16_routing.py b/tests/models/quantization/test_mimo_v2_w4a16_routing.py new file mode 100644 index 0000000000..9dc22a77ca --- /dev/null +++ b/tests/models/quantization/test_mimo_v2_w4a16_routing.py @@ -0,0 +1,176 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend. + +MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias +(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which +``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``. +``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not +support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose +device gate accepts the card, so the checkpoint could not be served natively at +all. + +aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus +a correction bias), so the backend only has to ask for it. This module covers +both halves: the gate accepts the router family, and the routing the wrapper +requests reproduces the model's own math. +""" + +import pytest +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + RoutingMethodType, + get_routing_method_type, +) +from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import ( + AiterW4A16ExpertsMonolithic, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static + +MIMO_EXPERTS = 256 # MiMo-V2.6-Flash +MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro +MIMO_TOP_K = 8 + + +def mimo_routing_method() -> RoutingMethodType: + """MiMo-V2.6: sigmoid + correction bias, a single expert group.""" + return get_routing_method_type( + scoring_func="sigmoid", + top_k=MIMO_TOP_K, + renormalize=True, + num_expert_group=1, + has_e_score_bias=True, + ) + + +def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig: + return FusedMoEConfig( + num_experts=num_experts, + experts_per_token=MIMO_TOP_K, + hidden_dim=4096, + intermediate_size=2048, + num_local_experts=num_experts, + num_logical_experts=num_experts, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + activation=MoEActivation.SILU, + in_dtype=torch.bfloat16, + device=torch.device("cuda"), + routing_method=mimo_routing_method(), + router_logits_dtype=torch.float32, + ) + + +def test_ungrouped_sigmoid_router_is_deepseekv3() -> None: + """Guard the premise of the fix: MiMo's router is DeepSeekV3-classified.""" + assert mimo_routing_method() == RoutingMethodType.DeepSeekV3 + + +def test_aiter_w4a16_supports_mimo_routing() -> None: + """The gate accepts MiMo's router family — the reported construction abort. + + Only the predicate the fix touches is exercised, so this runs on any host; + the full config path is covered by the test below. + """ + assert AiterW4A16ExpertsMonolithic._supports_routing_method( + RoutingMethodType.DeepSeekV3, kMxfp4Static, None + ) + + +@pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="only runs on ROCm with a supported AITER install", +) +def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None: + """The same gate through the real config path, device check included.""" + monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + + supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config( + AiterW4A16ExpertsMonolithic, + mimo_moe_config(), + kMxfp4Static, + None, + mk.FusedMoEActivationFormat.Standard, + ) + + assert supported, reason + + +@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS]) +@pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="only runs on ROCm with a supported AITER install", +) +def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None: + """The routing the wrapper requests must equal MiMo's own router math. + + MiMo's router is ungrouped: sigmoid scores, selection on + ``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then + renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro. + + The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20). + """ + aiter_routing = pytest.importorskip( + "aiter.ops.triton.moe.moe_routing.routing" + ).routing + + torch.manual_seed(0) + top_k, num_tokens = MIMO_TOP_K, 16 + logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0 + bias = torch.randn(num_experts, device="cuda") * 0.5 + + # Reference: select on the biased score, return the unbiased score, renorm. + scores = torch.sigmoid(logits.float()) + ref_ids = (scores + bias).topk(top_k, dim=-1).indices + ref_weights = scores.gather(1, ref_ids) + ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True) + + def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]: + routing_data, topk_indx, _ = aiter_routing( + logits, + top_k, + score_mode=score_mode, + bias=bias, + renorm=True, + routed_scaling_factor=1.0, + use_grouped_topk=False, + ) + # `topk_indx[i]` is the token-slot index (token * top_k + slot) of the + # i-th entry of the expert-sorted array whose weights are gate_scal. + hist = routing_data.expt_hist.to(torch.long).cpu() + expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist) + slot = topk_indx.reshape(-1).to(torch.long).cpu() + weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu() + ids = torch.full((num_tokens, top_k), -1, dtype=torch.long) + out = torch.zeros(num_tokens, top_k) + for i in range(slot.numel()): + token, position = int(slot[i]) // top_k, int(slot[i]) % top_k + ids[token, position] = int(expert_of_slot[i]) + out[token, position] = float(weights[i]) + return ids, out + + got_ids, got_weights = routed("sigmoid") + + # aiter's triton top-k emits each token's selection sorted by *ascending + # expert id*, while the reference is in descending ``sigmoid(logits)+bias`` + # rank order — align both sides on the expert id so the weight check + # becomes elementwise and still validates the expert<->weight pairing. + def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + order = torch.argsort(ids, dim=-1) + return ids.gather(1, order), w.gather(1, order) + + got_ids, got_weights = align(got_ids, got_weights) + ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu()) + assert torch.equal(got_ids, ref_ids) + assert torch.allclose(got_weights, ref_weights, atol=1e-3) + + # Control: the correction bias must steer selection only. Ignoring it would + # pick from the plain sigmoid scores instead. + plain_ids = scores.topk(top_k, dim=-1).indices + assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids) diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py index 13c34698c8..4776a908fb 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py @@ -295,6 +295,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): RoutingMethodType.Renormalize, RoutingMethodType.RenormalizeNaive, RoutingMethodType.DeepseekV4, + RoutingMethodType.DeepSeekV3, ) @staticmethod @@ -344,6 +345,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): RoutingMethodType.Renormalize, RoutingMethodType.RenormalizeNaive, RoutingMethodType.DeepseekV4, + RoutingMethodType.DeepSeekV3, ] @staticmethod @@ -373,11 +375,20 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): routed_scaling_factor: float | None = None, topk_group: int | None = None, ) -> torch.Tensor: - score_mode = ( - "sqrtsoftplus" - if self.moe_config.routing_method == RoutingMethodType.DeepseekV4 - else None - ) + routing_method = self.moe_config.routing_method + if routing_method == RoutingMethodType.DeepseekV4: + score_mode = "sqrtsoftplus" + elif routing_method == RoutingMethodType.DeepSeekV3: + # MiMo-V2.6: an ungrouped sigmoid router with a per-expert + # correction bias. aiter's flat top-k implements this natively + # (select on sigmoid(logits) + bias, return the unbiased sigmoid + # scores). + assert e_score_correction_bias is not None, ( + "DeepSeekV3 routing requires e_score_correction_bias" + ) + score_mode = "sigmoid" + else: + score_mode = None return aiter_triton_kernel_w4a16_moe_forward( hidden_states=hidden_states, w1=w1, From 50319916a3c72ec1358f9e29c7c282d316d86f78 Mon Sep 17 00:00:00 2001 From: djramic Date: Wed, 30 Sep 2026 11:42:25 +0200 Subject: [PATCH 103/248] [ROCm][CI] Move Basic Models (Other) to MI355 (#59400) Signed-off-by: Djordje Ramic Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 4 ++-- .buildkite/test_areas/models_basic.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 61f33d74ee..fd2398e759 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -5108,11 +5108,11 @@ steps: #------------------------------------------------------ mi355 · models / basic -------------------------------------------------------# -- label: ":amd: (MI355 DPX) Basic Models (Other)" # TBD +- label: ":amd: (MI355) Basic Models (Other)" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false - agent_pool: mi355_dpx + agent_pool: mi355_1 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 54130d0d02..7c03db5cb4 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -72,9 +72,9 @@ steps: - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py mirror: amd: - label: ":amd: (MI355 DPX) Basic Models (Other)" + label: ":amd: (MI355) Basic Models (Other)" dind: false - device: mi355_dpx + device: mi355_1 timeout_in_minutes: 65 depends_on: - image-build-amd From 91dab0eb7f42efd8580086935a7c0768a594f575 Mon Sep 17 00:00:00 2001 From: Mathew Odden <1471252+mrodden@users.noreply.github.com> Date: Wed, 30 Sep 2026 04:43:16 -0500 Subject: [PATCH 104/248] [ROCm][Perf] Allocate the pinned PLE prefetch buffer lazily (#58797) Signed-off-by: Mathew Odden Co-authored-by: opencode+glm-5.3+vllm --- tests/models/qwen4_exp/test_ple.py | 106 ++++++++++++++++++ .../qwen4_exp/common/ngram_embedding.py | 64 ++++++++--- 2 files changed, 155 insertions(+), 15 deletions(-) diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index b3038524eb..d271877dfc 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -609,6 +609,32 @@ def test_pinned_embedding_forward_finalizes_prefetched_output( assert torch.equal(output, expected) +def test_pinned_embedding_forward_requires_prior_start_prefetch() -> None: + """Forward before any start_prefetch must fail loudly, not read garbage.""" + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding._prefetch_buffer = None + embedding._prefetch_stream = None + embedding._output_dim = 6 + hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16) + + with pytest.raises(RuntimeError, match="prior start_prefetch"): + embedding(hidden_states) + + +def test_pinned_embedding_finalize_requires_prior_start_prefetch() -> None: + """_finalize_prefetch before any start_prefetch must fail loudly.""" + embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) + nn.Module.__init__(embedding) + embedding._prefetch_buffer = None + embedding._prefetch_stream = None + embedding.tp_size = 1 + output = torch.zeros(2, 6, dtype=torch.bfloat16) + + with pytest.raises(RuntimeError, match="prior start_prefetch"): + embedding._finalize_prefetch(torch.empty(4, 2, 3), output) + + def test_pinned_fp8_embedding_uses_int8_for_parallel_reduce() -> None: embedding = Qwen4ExpPLEPinnedHostEmbedding.__new__(Qwen4ExpPLEPinnedHostEmbedding) nn.Module.__init__(embedding) @@ -723,6 +749,67 @@ def test_ple_pinned_embedding_loads_on_cpu_and_looks_up_through_uva( ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_pinned_embedding_start_prefetch_allocates_lazily_and_feeds_forward( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """start_prefetch lazily allocates the stream/buffer, then forward consumes it. + + Exercises the full pinned-prefetch contract end to end: first call + allocates the side stream and buffer with the constructor geometry, + and forward finalizes the lookup into the returned output. + """ + _mock_etp_group(monkeypatch) + monkeypatch.setattr(embedding_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + embedding_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + monkeypatch.setattr(parameter_module, "get_tensor_model_parallel_rank", lambda: 0) + monkeypatch.setattr( + parameter_module, "get_tensor_model_parallel_world_size", lambda: 1 + ) + + with torch.device("cuda:0"): + embedding = Qwen4ExpPLEPinnedHostEmbedding( + 4, + 3, + params_dtype=torch.bfloat16, + padding_size=1, + prefix="test.ple_embedding", + embedding_method=Qwen4ExpPLEUnquantizedEmbeddingMethod(), + num_ngram_heads=2, + max_total_tokens=4, + ) + assert embedding._prefetch_buffer is None + assert embedding._prefetch_stream is None + + loaded_weight = torch.arange(12, dtype=torch.float32).reshape(4, 3) + loaded_weight = loaded_weight.to(torch.bfloat16) + copy_ple_embedding_shard_( + embedding.weight, + loaded_weight, + checkpoint_start=0, + tp_start=0, + tp_end=4, + ) + + ngram_ids = torch.tensor([[3, 0], [1, 2]], device="cuda:0") + hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16, device="cuda:0") + embedding.start_prefetch(hidden_states, ngram_ids) + + buffer = embedding._prefetch_buffer + assert buffer is not None + assert buffer.shape == (4, 2, 3) + assert buffer.dtype == torch.bfloat16 + assert buffer.device.type == "cuda" + assert embedding._prefetch_stream is not None + + output = embedding(hidden_states) + + expected = loaded_weight[ngram_ids.cpu()].to(device="cuda:0").flatten(-2) + torch.testing.assert_close(output, expected, rtol=0, atol=0) + + def test_ple_fp8_embedding_supports_mixed_precision_config() -> None: prefix = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding" quant_config = ModelOptMixedPrecisionConfig.from_config( @@ -2004,6 +2091,25 @@ def test_amd_fp8_embedding_loads_checkpoint_shards_and_global_scale( ) +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") +def test_amd_pinned_embedding_prefetch_machinery_is_lazy( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The pinned embedding must not allocate the prefetch stream/buffer in init. + + Paths that never prefetch (AMD uses the synchronous pinned-lookup op) + keep the stream/buffer unallocated for the process lifetime. NVIDIA's + first ``start_prefetch`` — the eager profile run, always before cudagraph + capture — allocates them lazily. + """ + module, _ = _build_amd_ngram_embedding( + monkeypatch, device="cuda:0", cpu_offload=True, fp8_checkpoint=False + ) + embedding = module.ngram_embedding + assert embedding._prefetch_buffer is None + assert embedding._prefetch_stream is None + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA or ROCm") @pytest.mark.parametrize("fp8_checkpoint", [False, True], ids=["bf16", "fp8"]) def test_amd_pinned_embedding_output_written_under_compile( diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py index 4d1d358958..e7371770f8 100644 --- a/vllm/models/qwen4_exp/common/ngram_embedding.py +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -7,6 +7,7 @@ n-gram embedding table can be kept in pinned host memory and looked up through Unified Virtual Addressing on any CUDA-alike platform. """ +import threading from abc import ABC, abstractmethod from typing import ClassVar @@ -415,14 +416,11 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): ) self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight) self._block_d = triton.next_power_of_2(self.embedding_dim) - self._prefetch_stream = torch.cuda.Stream(device=self._uva_weight.device) - self._prefetch_buffer = torch.empty( - max_total_tokens * self.etp_data_parallel_size, - num_ngram_heads, - self.embedding_dim, - dtype=self.weight.dtype, - device=self._uva_weight.device, - ) + self._prefetch_stream: torch.cuda.Stream | None = None + self._prefetch_buffer: torch.Tensor | None = None + self._prefetch_alloc_lock = threading.Lock() + self._prefetch_rows = max_total_tokens * self.etp_data_parallel_size + self._num_ngram_heads = num_ngram_heads self._output_dim = num_ngram_heads * self.embedding_dim def allocate_embedding_weight( @@ -506,10 +504,42 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): ngram_ids: torch.Tensor, ) -> None: """Gather ETP IDs and launch their UVA lookup on the side stream.""" + buffer = self._prefetch_buffer + if buffer is None: + # First use allocates. The eager profile run always precedes + # cudagraph capture, so allocation never happens mid-capture; + # the lock keeps concurrent first callers from tearing the + # stream/buffer pair. + with self._prefetch_alloc_lock: + buffer = self._prefetch_buffer + if buffer is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "pinned PLE prefetch buffer must be allocated " + "eagerly, before cudagraph capture" + ) + self._prefetch_stream = torch.cuda.Stream( + device=self._uva_weight.device + ) + buffer = torch.empty( + self._prefetch_rows, + self._num_ngram_heads, + self.embedding_dim, + dtype=self.weight.dtype, + device=self._uva_weight.device, + ) + self._prefetch_buffer = buffer + prefetch_stream = self._prefetch_stream + if prefetch_stream is None: + raise RuntimeError("pinned PLE prefetch stream was not allocated") slot_size, _ = self._get_dp_gather_slot(ngram_ids.shape[0]) gathered_ids = self._gather_dp_ids(ngram_ids, slot_size) - active_output = self._prefetch_buffer[: gathered_ids.shape[0]] - prefetch_stream = self._prefetch_stream + if gathered_ids.shape[0] > buffer.shape[0]: + raise ValueError( + f"pinned PLE prefetch buffer holds {buffer.shape[0]} rows, " + f"but the batch needs {gathered_ids.shape[0]}" + ) + active_output = buffer[: gathered_ids.shape[0]] prefetch_stream.wait_stream(torch.cuda.current_stream()) gathered_ids.record_stream(prefetch_stream) with torch.cuda.stream(prefetch_stream): @@ -522,7 +552,10 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): output: torch.Tensor, ) -> None: """Join the side stream, reduce ETP shards, and select local rows.""" - torch.cuda.current_stream().wait_stream(self._prefetch_stream) + prefetch_stream = self._prefetch_stream + if prefetch_stream is None: + raise RuntimeError("pinned PLE finalize requires a prior start_prefetch") + torch.cuda.current_stream().wait_stream(prefetch_stream) slot_size, slot_offset = self._get_dp_gather_slot(output.shape[0]) active_output = prefetch_output[: slot_size * self.etp_data_parallel_size] embeddings = self._reduce_etp_embeddings(active_output) @@ -535,8 +568,9 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding): def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: """Finish the pinned lookup into graph-owned output storage.""" - output = self._prefetch_buffer.new_empty( - (hidden_states.shape[0], self._output_dim) - ) - self._finalize_prefetch(self._prefetch_buffer, output) + buffer = self._prefetch_buffer + if buffer is None: + raise RuntimeError("pinned PLE lookup requires a prior start_prefetch") + output = buffer.new_empty((hidden_states.shape[0], self._output_dim)) + self._finalize_prefetch(buffer, output) return output From d71f6626061af38dad1c3f27830366cf7f279fe4 Mon Sep 17 00:00:00 2001 From: Akash kaothalkar <61960177+Akashcodes732@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:29:02 +0530 Subject: [PATCH 105/248] [Bugfix][Model][CPU] Fix Mamba2 quantized in_proj weight and scale loading for TP >1 (#58083) Signed-off-by: Akash kaothalkar Co-authored-by: Akash kaothalkar Co-authored-by: Antigravity Co-authored-by: Li, Jiang --- .../layers/mamba/mamba_mixer2.py | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index b567d188eb..812cab0212 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -232,13 +232,14 @@ def mamba_v2_sharded_weight_loader( # - the ignore is for a mundane mypy error as it does not # seem to handle slices well. # https://github.com/python/mypy/issues/2410 + target_slice = param.data[boundary : (boundary + take), ...] param.data[ boundary : (boundary + take), ... # type: ignore[misc] ] = loaded_weight[ loaded_start_idx : ( loaded_start_idx + take ) # type: ignore[misc] - ] # type: ignore[misc] + ].view_as(target_slice) # move indexing boundaries boundary += shard_size @@ -440,17 +441,18 @@ class MambaMixer2(MambaBase, PluggableLayer): tp_rank, ) - # Apply the custom weight loader to in_proj.weight + # Apply the custom weight loader to in_proj.weight and its scales. # Works for both non-quantized (Parameter) and quantized # (ModelWeightParameter which extends BasevLLMParameter) - if isinstance(self.in_proj.weight, BasevLLMParameter): - # For BasevLLMParameter subclasses (quantized layers like FP8) - # These have a weight_loader property that can be directly set - self.in_proj.weight.weight_loader = mamba_loader - else: - # For standard Parameter (non-quantized layers) - delattr(self.in_proj.weight, "weight_loader") - set_weight_attrs(self.in_proj.weight, {"weight_loader": mamba_loader}) + for attr_name in ("weight", "weight_scale", "input_scale"): + param = getattr(self.in_proj, attr_name, None) + if param is not None: + if isinstance(param, BasevLLMParameter): + param.weight_loader = mamba_loader + else: + if hasattr(param, "weight_loader"): + delattr(param, "weight_loader") + set_weight_attrs(param, {"weight_loader": mamba_loader}) # unsqueeze to fit conv1d weights shape into the linear weights shape. # Can't do this in `weight_loader` since it already exists in From d5ee309314c78a52a0e154cb15d53e34844d623f Mon Sep 17 00:00:00 2001 From: Francesco Fusco Date: Wed, 30 Sep 2026 12:34:31 +0200 Subject: [PATCH 106/248] [UX] Tag torch.compile log lines with the component being compiled (#48133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Francesco Fusco Co-authored-by: Claude Co-authored-by: Luka Govedič --- vllm/compilation/backends.py | 37 +++++++++++++++++--------- vllm/compilation/decorators.py | 48 +++++++++++++++++++++++++++------- vllm/compilation/monitor.py | 21 +++++++++++---- 3 files changed, 79 insertions(+), 27 deletions(-) diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index 4ac6949316..43252ce3d8 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -142,6 +142,10 @@ class CompilerManager: self.compilation_config = compilation_config self.compiler = make_compiler(compilation_config) self.loaded_artifacts: dict[str, Any] = {} + self.prefix: str = "" + + def _log_prefix(self) -> str: + return f"[{self.prefix}] " if self.prefix else "" def compute_hash(self, vllm_config: VllmConfig) -> str: return self.compiler.compute_hash(vllm_config) @@ -179,6 +183,7 @@ class CompilerManager: self.disable_cache = disable_cache self.cache_dir = cache_dir self.cache_file_path = os.path.join(cache_dir, "vllm_compile_cache.py") + self.prefix = prefix if not disable_cache and os.path.exists(self.cache_file_path): # load the cache from the file @@ -287,9 +292,10 @@ class CompilerManager: # after loading the last graph for this shape, record the time. # there can be multiple graphs due to piecewise compilation. elapsed = time.perf_counter() - compilation_start_time - logger.info_once( - "Directly load the compiled graph(s) for compile range %s " - "from the cache, took %.3f s", + logger.info( + "%sDirectly load the compiled graph(s) for compile " + "range %s from the cache, took %.3f s", + self._log_prefix(), str(compile_range), elapsed, ) @@ -373,8 +379,9 @@ class CompilerManager: self.is_cache_updated = True if graph_index == 0: # adds some info logging for the first graph - logger.info_once( - "Cache the graph of compile range %s for later use", + logger.info( + "%sCache the graph of compile range %s for later use", + self._log_prefix(), str(compile_range), ) logger.debug_once( @@ -388,8 +395,9 @@ class CompilerManager: # after compiling the last graph, record the end time if graph_index == num_graphs - 1: elapsed = time.perf_counter() - compilation_start_time - logger.info_once( - "Compiling a graph for compile range %s takes %.2f s", + logger.info( + "%sCompiling a graph for compile range %s takes %.2f s", + self._log_prefix(), str(compile_range), elapsed, ) @@ -865,6 +873,9 @@ class VllmBackend: # `torch.compile` is JIT compiled, so we don't need to # do anything here + def _log_prefix(self) -> str: + return f"[{self.prefix}] " if self.prefix else "" + def collect_standalone_compile_artifacts( self, ) -> tuple[Any, dict[str, list[int]] | None, dict[str, bool] | None]: @@ -1085,10 +1096,11 @@ class VllmBackend: disable_cache = disable_cache or is_ngram_gpu_enabled if disable_cache: - logger.info_once("vLLM's torch.compile cache is disabled.") + logger.info("%svLLM's torch.compile cache is disabled.", self._log_prefix()) else: - logger.info_once( - "Using cache directory: %s for vLLM's torch.compile", + logger.info( + "%sUsing cache directory: %s for vLLM's torch.compile", + self._log_prefix(), local_cache_dir, ) @@ -1148,8 +1160,9 @@ class VllmBackend: current_perf = time.perf_counter() current_epoch = time.time() dynamo_time = current_perf - torch_compile_start_time - logger.info_once( - "Dynamo bytecode transform time: %.2f s", + logger.info( + "%sDynamo bytecode transform time: %.2f s", + self._log_prefix(), dynamo_time, ) diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 35c0e6dee5..3aad32cc24 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -292,7 +292,12 @@ def _try_load_aot_compiled_fn( Re-raises on failure when ``VLLM_FORCE_AOT_LOAD`` is set. """ try: - with monitor_torch_compile(model.vllm_config, is_encoder=model._is_encoder): + with monitor_torch_compile( + model.vllm_config, + is_encoder=model._is_encoder, + tag=getattr(model, "_compile_tag", ""), + announce_start=False, + ): with ( set_current_vllm_config(model.vllm_config), open(aot_compilation_path, "rb") as f, @@ -309,8 +314,12 @@ def _try_load_aot_compiled_fn( with maybe_use_cudagraph_partition_wrapper(model.vllm_config): loaded_fn._artifacts.compiled_fn.finalize_loading(model.vllm_config) compilation_counter.num_aot_artifacts_loaded += 1 + tag = getattr(model, "_compile_tag", "") + log_prefix = f"[{tag}] " if tag else "" logger.info( - "Directly load AOT compilation from path %s", aot_compilation_path + "%sDirectly load AOT compilation from path %s", + log_prefix, + aot_compilation_path, ) return loaded_fn except Exception as e: @@ -319,8 +328,11 @@ def _try_load_aot_compiled_fn( message = "Compile cache file corrupted." else: message = str(e) + tag = getattr(model, "_compile_tag", "") + log_prefix = f"[{tag}] " if tag else "" logger.warning( - "Compiling model again due to a load failure from %s, reason: %s", + "%sCompiling model again due to a load failure from %s, reason: %s", + log_prefix, aot_compilation_path, message, ) @@ -402,6 +414,16 @@ def _support_torch_compile( compilation_counter.num_models_seen += 1 self.compiled = False + # Capture the active model tag (e.g. "backbone", "eagle_head") so + # later compilation/warmup log lines can identify which component + # they belong to. Mirror VllmBackend's precedence rule + # (`compile_prefix or model_tag`) so log tags line up with the + # cache-dir tag: encoders use the class name, everything else + # falls through to `model_tag`. + from vllm.compilation.backends import model_tag as _current_model_tag + + self._compile_tag = (cls.__name__ if is_encoder else "") or _current_model_tag + # Handled by monkeypatching `TorchCompileWithNoGuardsWrapper` into base class TorchCompileWithNoGuardsWrapper.__init__( self, @@ -567,7 +589,7 @@ def _support_torch_compile( self.aot_compiled_fn = loaded_fn self.was_aot_compile_fn_loaded_from_disk = True with ( - monitor_profiling_run(), + monitor_profiling_run(tag=self._compile_tag), maybe_use_cudagraph_partition_wrapper(self.vllm_config), ): output = self.aot_compiled_fn(self, *args, **kwargs) @@ -667,7 +689,9 @@ def _support_torch_compile( self._aot_compilation_path = aot_compilation_path self._aot_cache_dir = cache_dir with monitor_torch_compile( - self.vllm_config, is_encoder=self._is_encoder + self.vllm_config, + is_encoder=self._is_encoder, + tag=self._compile_tag, ): self.aot_compiled_fn = self.aot_compile(*args, **kwargs) compilation_counter.num_aot_compiles += 1 @@ -675,14 +699,15 @@ def _support_torch_compile( # AOT artifact. self.save_aot_compiled_function() - with monitor_profiling_run(): + with monitor_profiling_run(tag=self._compile_tag): output = self.aot_compiled_fn(self, *args, **kwargs) else: with monitor_torch_compile( self.vllm_config, - "torch.compile and initial profiling/warmup " + "%storch.compile and initial profiling/warmup " "run together took %.2f s in total", is_encoder=self._is_encoder, + tag=self._compile_tag, ): output = TorchCompileWithNoGuardsWrapper.__call__( self, # type: ignore[arg-type] @@ -706,6 +731,7 @@ def _support_torch_compile( self.aot_compiled_fn and self._aot_compilation_path and self._aot_cache_dir ) + log_prefix = f"[{self._compile_tag}] " if self._compile_tag else "" try: os.makedirs(self._aot_cache_dir, exist_ok=True) # File saving should be atomic, so we will save to a temporary location @@ -714,13 +740,15 @@ def _support_torch_compile( self.aot_compiled_fn.save_compiled_function(tmp_file) os.replace(tmp_file, self._aot_compilation_path) compilation_counter.num_aot_artifacts_saved += 1 - logger.info_once( - "saved AOT compiled function to %s", + logger.info( + "%ssaved AOT compiled function to %s", + log_prefix, self._aot_compilation_path, ) except Exception as e: logger.warning( - "unable to save AOT compiled function to %s: %s", + "%sunable to save AOT compiled function to %s: %s", + log_prefix, self._aot_compilation_path, e, ) diff --git a/vllm/compilation/monitor.py b/vllm/compilation/monitor.py index c23a8f6722..e38192034b 100644 --- a/vllm/compilation/monitor.py +++ b/vllm/compilation/monitor.py @@ -17,18 +17,27 @@ torch_compile_start_time: float = 0.0 @contextlib.contextmanager def monitor_torch_compile( vllm_config: VllmConfig, - message: str = "torch.compile took %.2f s in total", + message: str = "%storch.compile took %.2f s in total", is_encoder: bool = False, + tag: str = "", + announce_start: bool = True, ) -> Generator[None, None, None]: """Context manager that times torch.compile and manages depyf debugging. On normal exit: logs the compile time and exits depyf. On exception: cleans up depyf without logging (compilation failed). + + ``announce_start=False`` suppresses the "Starting torch.compile" line; + used by the AOT-load probe so a failed load attempt doesn't announce a + compile it didn't perform. """ global torch_compile_start_time torch_compile_start_time = time.perf_counter() compilation_config = vllm_config.compilation_config + log_prefix = f"[{tag}] " if tag else "" + if announce_start and compilation_config.mode == CompilationMode.VLLM_COMPILE: + logger.info("%sStarting torch.compile", log_prefix) depyf_cm = None path = vllm_config.compile_debug_dump_path() if compilation_config.mode == CompilationMode.VLLM_COMPILE and path: @@ -50,7 +59,7 @@ def monitor_torch_compile( compilation_config.encoder_compilation_time += total_compile_time else: compilation_config.compilation_time += total_compile_time - logger.info_once(message, total_compile_time) + logger.info(message, log_prefix, total_compile_time) finally: if depyf_cm is not None: try: @@ -60,7 +69,7 @@ def monitor_torch_compile( @contextlib.contextmanager -def monitor_profiling_run() -> Generator[None, None, None]: +def monitor_profiling_run(tag: str = "") -> Generator[None, None, None]: """Context manager that times the initial profiling run. Asserts that no backend compilation occurs during the profiling run @@ -78,8 +87,10 @@ def monitor_profiling_run() -> Generator[None, None, None]: "backend compilation occurred during the initial profiling run; " "all compilation should be complete before the profiling run starts." ) - logger.info_once( - "Initial profiling/warmup run took %.2f s", + log_prefix = f"[{tag}] " if tag else "" + logger.info( + "%sInitial profiling/warmup run took %.2f s", + log_prefix, elapsed, ) From 0103a9b96fde27d26a464bf2c30b4d977b2a134f Mon Sep 17 00:00:00 2001 From: Matej Sirovatka <54212263+S1ro1@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:42:03 -0700 Subject: [PATCH 107/248] [Quantization] Add per-token NVFP4 CuTe-DSL MoE backend (#50030) Signed-off-by: aoshen02 Signed-off-by: S1ro1 Co-authored-by: aoshen02 Co-authored-by: OpenAI Codex Co-authored-by: aoshen02 --- .../moe/test_flashinfer_cutedsl_nvfp4_moe.py | 71 ++++++++++++++----- tests/quantization/test_online.py | 24 +++++++ .../test_trtllm_nvfp4_hidden_dim_padding.py | 6 +- .../experts/flashinfer_cutedsl_moe.py | 44 ++++++++++-- .../fused_moe/experts/trtllm_nvfp4_moe.py | 34 ++------- .../layers/fused_moe/oracle/nvfp4.py | 9 ++- .../layers/fused_moe/runner/moe_runner.py | 2 +- .../quantization/utils/flashinfer_fp4_moe.py | 42 ++++++++--- .../quantization/utils/flashinfer_utils.py | 20 +++++- 9 files changed, 186 insertions(+), 66 deletions(-) diff --git a/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py b/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py index 3d637258d3..e3a3478e37 100644 --- a/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py +++ b/tests/kernels/moe/test_flashinfer_cutedsl_nvfp4_moe.py @@ -29,14 +29,20 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, RoutingMethodType, - nvfp4_moe_quant_config, ) from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import ( FlashInferCuteDSLExperts, ) +from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import ( + NvFp4MoeBackend, + make_nvfp4_moe_quant_config, +) from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import ( prepare_nvfp4_moe_layer_for_flashinfer_cutedsl, ) +from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + quantize_nvfp4_per_token_input, +) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4 from vllm.utils.math_utils import next_power_of_2 @@ -159,8 +165,12 @@ def _dequantize_nvfp4_linear( return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype) -@pytest.mark.parametrize("m,n,k,e,topk", [(16, 128, 512, 4, 2)]) +@pytest.mark.parametrize( + "m,n,k,e,topk", + [(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)], +) @pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES) +@pytest.mark.parametrize("per_token_activation", [False, True]) @pytest.mark.parametrize("dtype", [torch.bfloat16]) @torch.inference_mode() def test_flashinfer_cutedsl_fp4_moe( @@ -173,6 +183,7 @@ def test_flashinfer_cutedsl_fp4_moe( alpha: float | None, beta: float | None, limit: float | None, + per_token_activation: bool, dtype: torch.dtype, workspace_init, ): @@ -181,6 +192,10 @@ def test_flashinfer_cutedsl_fp4_moe( VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1)) ): hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10 + # Keep GEMM1 activation variance stable as K changes, so the fixed + # tolerance against the BF16 GEMM2 reference measures comparable FP4 + # quantization error at each hidden width. + hidden_states *= (512 / k) ** 0.5 w1_rows = 2 * n if activation.is_gated else n w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15 @@ -193,7 +208,6 @@ def test_flashinfer_cutedsl_fp4_moe( hidden_states, score, topk, renormalize=False ) - activation = MoEActivation.RELU2_NO_MUL moe_config = FusedMoEConfig( num_experts=e, experts_per_token=topk, @@ -231,19 +245,30 @@ def test_flashinfer_cutedsl_fp4_moe( w2_scale_2=(1.0 / w2_global_scale), a2_scale=a2_scale, ) - quant_config = nvfp4_moe_quant_config( - g1_alphas=w1_alpha, - g2_alphas=w2_alpha, - a1_gscale=(1.0 / a1_scale), - a2_gscale=(1.0 / a2_scale), - w1_scale=w1_scale_cutedsl, + # I=192 must remain unpadded for gated experts, but become 256 for + # non-gated experts. The reference below uses the checkpoint shapes. + expected_n = n if activation.is_gated else (n + 127) // 128 * 128 + assert moe_config.intermediate_size_per_partition == expected_n + assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1) + assert w2_cutedsl.shape[2] * 2 == expected_n + expected_k = (k + 255) // 256 * 256 + assert moe_config.hidden_dim == expected_k + assert moe_config.hidden_dim_unpadded == k + assert w1_cutedsl.shape[2] * 2 == expected_k + assert w2_cutedsl.shape[1] == expected_k + quant_config = make_nvfp4_moe_quant_config( + backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL, + w13_scale_2=w1_alpha, + w2_scale_2=w2_alpha, + a13_scale=a1_scale, + a2_scale=a2_scale, + w13_scale=w1_scale_cutedsl, w2_scale=w2_scale_cutedsl, - is_scale_swizzled=False, # Unset params must be omitted rather than forwarded as None into # the kernel's float-typed SwiGLU arguments. - gemm1_alpha=alpha, - gemm1_beta=beta, - gemm1_clamp_limit=limit, + swiglu_alpha=alpha, + swiglu_beta=beta, + swiglu_limit=limit, ) cutedsl_experts = mk.FusedMoEKernel( @@ -256,11 +281,13 @@ def test_flashinfer_cutedsl_fp4_moe( FlashInferCuteDSLExperts( moe_config=moe_config, quant_config=quant_config, + per_token_activation=per_token_activation, ), ) cutedsl_output = cutedsl_experts.apply( - hidden_states=hidden_states, + # The model runner pads inputs and trims outputs around this kernel. + hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)), w1=w1_cutedsl, w2=w2_cutedsl, topk_weights=topk_weights, @@ -270,19 +297,25 @@ def test_flashinfer_cutedsl_fp4_moe( expert_map=None, apply_router_weight_on_input=False, ) + cutedsl_output = cutedsl_output[:, :k] a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32) - a_q, a_scale = ops.scaled_fp4_quant( - hidden_states, - a_global_scale, - is_sf_swizzled_layout=False, - ) + if per_token_activation: + a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states) + else: + a_q, a_scale = ops.scaled_fp4_quant( + hidden_states, + a_global_scale, + is_sf_swizzled_layout=False, + ) a_in_dtype = _dequantize_nvfp4_linear( a_q, a_scale, a_global_scale, dtype=dtype, ) + if per_token_activation: + a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype) w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype) w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 645ad67062..c7495326e2 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -605,6 +605,30 @@ def test_checkpoint_quantization_rejects_online_shorthand(tmp_path) -> None: ModelConfig(model=str(tmp_path), quantization="fp8_per_channel") +def test_nvfp4_per_token_backend_contract() -> None: + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import ( + FlashInferCuteDSLExperts, + ) + from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import ( + FlashInferExperts, + ) + from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import ( + TrtLlmNvFp4ExpertsModular, + TrtLlmNvFp4ExpertsMonolithic, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kNvfp4DynamicToken, + kNvfp4Static, + ) + + scheme = (kNvfp4Static, kNvfp4DynamicToken) + assert TrtLlmNvFp4ExpertsMonolithic._supports_quant_scheme(*scheme) + assert TrtLlmNvFp4ExpertsModular._supports_quant_scheme(*scheme) + assert FlashInferCuteDSLExperts._supports_quant_scheme(*scheme) + assert FlashInferCuteDSLExperts._supports_no_act_and_mul() + assert not FlashInferExperts._supports_quant_scheme(*scheme) + + @pytest.mark.parametrize("per_token_activation", [False, True]) @pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float16]) def test_nvfp4_one_sided_sizes_dispatched_activations( diff --git a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py index be8f984853..9effb4f7fc 100644 --- a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py +++ b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py @@ -11,8 +11,8 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import ( prepare_nvfp4_moe_layer_for_fi_or_cutlass, ) from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + align_fp4_moe_hidden_dim_for_fi, align_fp4_moe_weights_for_fi, - align_trtllm_fp4_moe_hidden_dim_for_fi, ) @@ -63,7 +63,7 @@ def test_align_trtllm_fp4_moe_hidden_dim_noop(): w2_scale = torch.arange(2 * 512 * 1, dtype=torch.uint8).reshape(2, 512, 1) out_w13, out_w13_scale, out_w2, out_w2_scale, padded_hidden = ( - align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale) + align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale) ) assert padded_hidden == 512 @@ -90,7 +90,7 @@ def test_align_trtllm_fp4_moe_hidden_dim_pads_to_256_multiple(): ) out_w13, out_w13_scale, out_w2, out_w2_scale, out_hidden_dim = ( - align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale) + align_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale) ) assert out_hidden_dim == padded_hidden_dim diff --git a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py index d525822b91..15fa18b13c 100644 --- a/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/flashinfer_cutedsl_moe.py @@ -14,11 +14,14 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceNoOP, ) from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE, activation_to_flashinfer_int, + quantize_nvfp4_per_token_input, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kNvfp4Dynamic, + kNvfp4DynamicToken, kNvfp4Static, ) from vllm.platforms import current_platform @@ -40,6 +43,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): self, moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig, + per_token_activation: bool = False, ): super().__init__( moe_config=moe_config, @@ -63,6 +67,14 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): self.gemm1_clamp_limit = quant_config.gemm1_clamp_limit self.situ_beta = moe_config.activation_situ_beta self.situ_linear_beta = moe_config.activation_situ_linear_beta + self.per_token_activation = per_token_activation + self.per_token_global_scale = None + if per_token_activation: + assert quant_config.a2_gscale is not None + self.per_token_global_scale = quant_config.a2_gscale.new_full( + (1,), + NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE, + ) def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale) @@ -92,6 +104,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): ) -> bool: SUPPORTED_W_A = [ (kNvfp4Static, kNvfp4Dynamic), + (kNvfp4Static, kNvfp4DynamicToken), ] return (weight_key, activation_key) in SUPPORTED_W_A @@ -114,6 +127,10 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: return TopKWeightAndReduceNoOP() + @property + def expects_unquantized_inputs(self) -> bool: + return self.per_token_activation + def workspace_shapes( self, M: int, @@ -127,8 +144,8 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: workspace1 = (0,) workspace2 = (0,) - # K is packed (K//2 for uint8), so output uses hidden_dim. - assert self.hidden_dim == K * 2 + expected_hidden_dim = K if self.expects_unquantized_inputs else K * 2 + assert self.hidden_dim == expected_hidden_dim output = (M, self.hidden_dim) return (workspace1, workspace2, output) @@ -151,13 +168,26 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): apply_router_weight_on_input: bool | None, ): assert self.quant_dtype == "nvfp4" - assert a1q_scale is not None assert self.w1_scale is not None assert self.w2_scale is not None + if self.expects_unquantized_inputs: + hidden_states, block_scale, per_token_scale = ( + quantize_nvfp4_per_token_input(hidden_states) + ) + fc2_input_scale = self.per_token_global_scale + else: + assert a1q_scale is not None + block_scale = a1q_scale + per_token_scale = None + fc2_input_scale = self.a2_gscale + + assert block_scale is not None + assert fc2_input_scale is not None + # a1q_scale is (M, K//16) float8_e4m3fn from fp4_quantize. # The functional API expects x_sf with trailing dim: (M, K//16, 1). - x_sf = a1q_scale.unsqueeze(-1) + x_sf = block_scale.unsqueeze(-1) # The kernel defaults swiglu_{alpha,beta,limit} to the plain-SwiGLU # values, so only forward the ones the model actually sets. @@ -188,6 +218,9 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): } swiglu_kwargs = {k: v for k, v in swiglu_params.items() if v is not None} + per_token_kwargs = ( + {"per_token_scale": per_token_scale} if self.per_token_activation else {} + ) flashinfer_cute_dsl_fused_moe_nvfp4( x=hidden_states, x_sf=x_sf, @@ -196,7 +229,7 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): w1_weight=w1, w1_weight_sf=self.w1_scale, w1_alpha=self.g1_alphas, - fc2_input_scale=self.a2_gscale, + fc2_input_scale=fc2_input_scale, w2_weight=w2, w2_weight_sf=self.w2_scale, w2_alpha=self.g2_alphas, @@ -209,4 +242,5 @@ class FlashInferCuteDSLExperts(mk.FusedMoEExpertsModular): MoEActivation.SILU if activation == MoEActivation.SITU else activation ), **swiglu_kwargs, + **per_token_kwargs, ) diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py index f9cca18a9c..9dcfc6af3f 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.utils import fi_moe_largest_bucket from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( activation_to_flashinfer_int, has_flashinfer_situ_activation, + quantize_nvfp4_per_token_input, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -37,10 +38,6 @@ from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe logger = init_logger(__name__) -# Base scale for per-token NVFP4 activation quant; the kernel folds the -# per-token global scale (from the activation amax) on top of it. -_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0) - class TrtLlmNvFp4ExpertsBase: """NvFp4 TRTLLM-Gen MoE kernels. Supports modular and monolithic interface.""" @@ -259,23 +256,6 @@ class TrtLlmNvFp4ExpertsBase: def expects_unquantized_inputs(self) -> bool: return self.per_token_activation - def _quantize_per_token_input( - self, hidden_states: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """NVFP4-quantize activations with a per-token global scale. - - Returns ``(packed_fp4, block_scale, per_token_scale)``. - """ - from flashinfer import SfLayout, nvfp4_quantize - - hs_fp4, hs_block_scale, per_token_scale = nvfp4_quantize( - hidden_states, - _PER_TOKEN_BASE_GLOBAL_SCALE, - sfLayout=SfLayout.layout_linear, - per_token_activation=True, - ) - return hs_fp4, hs_block_scale, per_token_scale - def _get_chunk_size(self) -> int: MAX_GRID_Y = 65535 MAX_TILE_TOKENS_DIM = 128 @@ -361,9 +341,9 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula # Per-token: input is unquantized, quantize it here. Otherwise it was # already quantized in prepare() with the static global scale. - if self.per_token_activation: + if self.expects_unquantized_inputs: hidden_states, block_scale, per_token_scale = ( - self._quantize_per_token_input(hidden_states) + quantize_nvfp4_per_token_input(hidden_states) ) else: block_scale, per_token_scale = a1q_scale, None @@ -437,7 +417,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula ) -> UnfinalizedMoEOutput | None: assert self._supports_activation(activation) # Per-token defers input quant to _invoke_kernel, so a1q_scale is None. - assert a1q_scale is not None or self.per_token_activation + assert a1q_scale is not None or self.expects_unquantized_inputs # DeepEP produces int64 indexes. topk_ids = topk_ids.to(dtype=torch.int32) @@ -539,7 +519,7 @@ class TrtLlmNvFp4ExpertsMonolithic( import flashinfer assert self._supports_activation(activation) - assert a1q_scale is not None or self.per_token_activation + assert a1q_scale is not None or self.expects_unquantized_inputs assert self.quant_config.w1_scale is not None assert self.quant_config.w2_scale is not None assert ( @@ -551,9 +531,9 @@ class TrtLlmNvFp4ExpertsMonolithic( ) # Per-token: input is unquantized, quantize it here (see modular apply). - if self.per_token_activation: + if self.expects_unquantized_inputs: hidden_states, block_scale, per_token_scale = ( - self._quantize_per_token_input(hidden_states) + quantize_nvfp4_per_token_input(hidden_states) ) else: block_scale, per_token_scale = a1q_scale, None diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 0fecb65e4b..e1952c0032 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -612,7 +612,14 @@ def make_nvfp4_moe_kernel( logger.info_once("Using %s", prepare_finalize.__class__.__name__) extra_kwargs = {} - if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM and per_token_activation: + if ( + backend + in ( + NvFp4MoeBackend.FLASHINFER_TRTLLM, + NvFp4MoeBackend.FLASHINFER_CUTEDSL, + ) + and per_token_activation + ): extra_kwargs["per_token_activation"] = True # Create Experts. diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 08279d1c89..9cb7cfbb0f 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -720,7 +720,7 @@ class MoERunner(MoERunnerInterface): ) # Record before `_maybe_pad_hidden_states` pads activations to match - # `moe_config.hidden_dim`, e.g. after `align_trtllm_fp4_moe_hidden_dim_for_fi` + # `moe_config.hidden_dim`, e.g. after `align_fp4_moe_hidden_dim_for_fi` # so routed output can be trimmed before # shared+routed add / latent up proj if needed. diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py b/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py index c5c01e59ca..24b95ef89a 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py @@ -9,8 +9,8 @@ import torch from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + align_fp4_moe_hidden_dim_for_fi, align_fp4_moe_weights_for_fi, - align_trtllm_fp4_moe_hidden_dim_for_fi, ) from vllm.model_executor.layers.quantization.utils.nvfp4_utils import ( swizzle_blockscale, @@ -128,9 +128,9 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl( ]: """Prepare weights for the CuteDSL wrapper-based NvFP4 MoE backend. - Converts weight scale factors to MMA layout expected by CuteDslMoEWrapper, - and interleaves w13 gate/linear rows for gated activations. Non-gated - activations use a single w13 projection and keep its row order unchanged. + Pads the runtime expert tensors to the kernel's GEMM alignment, converts + weight scale factors to the MMA layout expected by CuteDslMoEWrapper, and + interleaves w13 gate/linear rows for gated activations. """ # Global scaling factors (same as other FlashInfer backends). num_experts = w13.shape[0] @@ -140,12 +140,38 @@ def prepare_nvfp4_moe_layer_for_flashinfer_cutedsl( ) a2_scale = amax_for_moe_activation_quant(a2_scale, enable_eplb).repeat(num_experts) - if layer.activation.is_gated: + gated = layer.activation.is_gated + if gated: w13, w13_scale = reorder_w13_to_w31_for_flashinfer_cutedsl( layer.activation, w13, w13_scale ) - # Interleave up/gate rows for w13 weights and scales. + # GEMM1's output dimension must be a multiple of 128: 2I for gated + # activations (also required by interleaving), but only I for non-gated. + # Keep the checkpoint tensors unchanged and pad only the kernel's runtime + # representation. Zero rows also make the padded GEMM2 contraction a no-op. + w13, w13_scale, w2, w2_scale, padded_intermediate = align_fp4_moe_weights_for_fi( + w13, + w13_scale, + w2, + w2_scale, + is_act_and_mul=gated, + min_alignment=64 if gated else 128, + ) + layer.moe_config.intermediate_size_per_partition = padded_intermediate + + # GEMM1 gathers full 256-element K tiles of activations and their scales + # without a K-tail predicate. Pad runtime weights to match; the MoE runner + # pads activations and trims the output using the original hidden size. + w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi( + w13, w13_scale, w2, w2_scale + ) + if layer.moe_config.hidden_dim_unpadded is None: + layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim + layer.moe_config.hidden_dim = padded_hidden + + if gated: + # Interleave up/gate rows for the fused gated activation. w13 = interleave_linear_and_gate(w13, group_size=64, dim=1) w13_scale = interleave_linear_and_gate(w13_scale, group_size=64, dim=1) @@ -371,8 +397,8 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass( # Shuffle weights and scales for FI TRTLLM NVFP4 MoE kernels. if backend == NvFp4MoeBackend.FLASHINFER_TRTLLM: - w13, w13_scale, w2, w2_scale, padded_hidden = ( - align_trtllm_fp4_moe_hidden_dim_for_fi(w13, w13_scale, w2, w2_scale) + w13, w13_scale, w2, w2_scale, padded_hidden = align_fp4_moe_hidden_dim_for_fi( + w13, w13_scale, w2, w2_scale ) if layer.moe_config.hidden_dim_unpadded is None: layer.moe_config.hidden_dim_unpadded = layer.moe_config.hidden_dim diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 1bfd7e61f9..be28de9b66 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -13,6 +13,8 @@ if TYPE_CHECKING: logger = init_logger(__name__) +NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE = 1.0 / (448.0 * 6.0) + def activation_to_flashinfer_int(activation: MoEActivation) -> int: return activation_to_flashinfer_type(activation).value @@ -50,6 +52,20 @@ def activation_to_flashinfer_type(activation: MoEActivation) -> "ActivationType" return ACTIVATION_TO_FI_ACTIVATION[activation] +def quantize_nvfp4_per_token_input( + hidden_states: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Quantize NVFP4 activations with one FP32 decode scale per token.""" + from flashinfer import SfLayout, nvfp4_quantize + + return nvfp4_quantize( + hidden_states, + NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE, + sfLayout=SfLayout.layout_linear, + per_token_activation=True, + ) + + def swap_w13_to_w31(x: torch.Tensor) -> torch.Tensor: return ( x.reshape(-1, 2, x.shape[-2] // 2, x.shape[-1]).flip(dims=[1]).reshape(x.shape) @@ -264,7 +280,7 @@ def align_fp4_moe_weights_for_fi( return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate -def align_trtllm_fp4_moe_hidden_dim_for_fi( +def align_fp4_moe_hidden_dim_for_fi( w13: torch.Tensor, w13_scale: torch.Tensor, w2: torch.Tensor, @@ -279,7 +295,7 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi( return w13, w13_scale, w2, w2_scale, hidden_size logger.warning_once( - "Padding hidden size from %d to %d for TRTLLM NVFP4 MoE weights. " + "Padding hidden size from %d to %d for FlashInfer NVFP4 MoE weights. " "This requires activation slicing at runtime and may cause " "performance degradation.", hidden_size, From c4fa6f36a7ba6fc9bb46c5934dc1e6ed88c00862 Mon Sep 17 00:00:00 2001 From: apex-mochen <2756823972@qq.com> Date: Wed, 30 Sep 2026 18:54:32 +0800 Subject: [PATCH 108/248] [Tests][Multimodal] Cover scoped processor kwargs precedence (#59399) Signed-off-by: apex-mochen <2756823972@qq.com> --- tests/config/test_multimodal_config.py | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index bd9343bd15..6c8bead2d5 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -240,6 +240,43 @@ def test_merge_mm_processor_kwargs_scoped_precedence(): } +def test_merge_mm_processor_kwargs_preserves_scoped_siblings_and_precedence(): + config = MultiModalConfig( + mm_processor_kwargs={ + "size": { + "shortest_edge": 100, + "longest_edge": 1000, + }, + "videos_kwargs": { + "fps": 2, + "size": {"longest_edge": 1200}, + }, + }, + mm_device_do_normalize=False, + ) + + assert config.merge_mm_processor_kwargs( + { + "size": {"longest_edge": 1800}, + "videos_kwargs": { + "size": {"shortest_edge": 200}, + }, + } + ) == { + "size": { + "shortest_edge": 100, + "longest_edge": 1800, + }, + "videos_kwargs": { + "fps": 2, + "size": { + "shortest_edge": 200, + "longest_edge": 1800, + }, + }, + } + + def test_merge_mm_processor_kwargs_mapping_vs_replacement(): config = MultiModalConfig( mm_processor_kwargs={ From 5225f1bacb907e3d7ec76e7f356c598468d229ec Mon Sep 17 00:00:00 2001 From: Taimys <96967844+Taimys@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:06:07 +0800 Subject: [PATCH 109/248] [MyPy][3/N] Fix MyPy errors in test groups (part 3) (#55939) Signed-off-by: Taimys Signed-off-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Codex Co-authored-by: Luyi Xiao <81631911+luyixiao95@users.noreply.github.com> Co-authored-by: Taimys Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- tests/benchmarks/sweep/test_param_sweep.py | 16 +-- tests/benchmarks/test_audio_dataset.py | 15 ++- tests/benchmarks/test_bfcl_dataset.py | 4 +- ...est_custom_dataset_chat_template_kwargs.py | 4 +- tests/benchmarks/test_custom_image_dataset.py | 44 +++--- tests/benchmarks/test_random_dataset.py | 2 + tests/benchmarks/test_sampling_params.py | 8 +- tests/benchmarks/test_throughput_cli.py | 5 +- .../test_anthropic_messages_conversion.py | 20 ++- .../llm/test_mm_processor_kwargs.py | 110 +++++++++------ .../openai/chat_completion/test_audio.py | 18 +-- .../pooling/classify/test_offline.py | 4 +- .../pooling/embed/test_io_processor.py | 19 +-- .../entrypoints/pooling/embed/test_offline.py | 4 +- .../entrypoints/pooling/embed/test_online.py | 22 +-- .../pooling/embed/test_online_dimensions.py | 7 +- .../pooling/embed/test_online_long_text.py | 4 + .../reward/test_token_reward_online.py | 5 +- .../pooling/scoring/test_io_processor_unit.py | 10 +- .../test_jina_ranking_io_processor_unit.py | 22 ++- .../scoring/test_late_interaction_serving.py | 29 ++-- .../entrypoints/pooling/test_io_processor.py | 4 +- .../pooling/token_classify/test_online.py | 1 + .../pooling/token_embed/test_online.py | 1 + .../attention/test_attention_selector.py | 20 +-- .../attention/test_cascade_flash_attn.py | 2 +- tests/kernels/attention/test_cpu_attn.py | 2 +- tests/kernels/attention/test_flash_attn.py | 4 +- .../test_kimi_k3_mla_key_concat_kv_cache.py | 9 +- tests/kernels/attention/test_mha_attn.py | 8 +- tests/kernels/attention/test_minimax_m3.py | 29 ++-- ...st_minimax_m3_msa_cutlass_sparse_decode.py | 8 +- tests/kernels/attention/test_rocm_aiter_fa.py | 3 + .../test_rocm_aiter_mla_causal_verify_mask.py | 6 +- .../test_rocm_aiter_mla_fp8_prefill.py | 4 +- .../attention/test_rocm_aiter_mla_sink.py | 15 ++- ...est_rocm_aiter_mla_sparse_metadata_sync.py | 4 +- .../test_rocm_paged_mxfp4_indexer.py | 6 +- .../attention/test_rocm_triton_attn_dsv4.py | 4 +- .../test_triton_unified_attention_diffkv.py | 1 + tests/kernels/helion/test_config_manager.py | 2 +- ...test_dynamic_per_token_scaled_fp8_quant.py | 2 +- .../kernels/helion/test_fused_qk_norm_rope.py | 2 +- tests/kernels/helion/test_pattern_matching.py | 7 +- .../helion/test_per_token_group_fp8_quant.py | 2 +- tests/kernels/helion/test_register.py | 1 + .../test_rms_norm_dynamic_per_token_quant.py | 2 +- .../helion/test_rms_norm_per_block_quant.py | 2 +- .../test_silu_and_mul_per_block_quant.py | 2 +- tests/kernels/helion/test_silu_mul_fp8.py | 3 +- tests/models/language/pooling/embed_utils.py | 9 ++ tests/multimodal/media/test_video.py | 18 +-- tests/multimodal/test_audio.py | 14 +- tests/multimodal/test_cache.py | 19 ++- tests/multimodal/test_hasher.py | 2 + tests/multimodal/test_processing.py | 35 +++-- tests/multimodal/test_registry.py | 2 +- tests/multimodal/test_video.py | 43 +++--- tests/quantization/test_auto_gptq.py | 8 +- tests/quantization/test_auto_round.py | 18 +-- tests/quantization/test_cutlass_w4a16.py | 1 + tests/quantization/test_fp8.py | 41 ++++-- tests/quantization/test_gfx950_moe.py | 3 +- tests/quantization/test_int8_moe_oracle.py | 3 +- tests/quantization/test_lm_head.py | 2 + tests/quantization/test_modelopt.py | 1 + tests/quantization/test_moe_wna16.py | 12 +- tests/quantization/test_online.py | 29 ++-- tests/quantization/test_online_mxfp4.py | 3 +- .../test_quantization_config_args.py | 27 ++-- tests/quantization/test_quark.py | 13 +- .../test_register_quantization_config.py | 2 +- tests/quantization/test_torchao.py | 2 + .../test_trtllm_nvfp4_hidden_dim_padding.py | 2 +- tests/quantization/test_turboquant.py | 2 +- tests/quantization/utils.py | 17 ++- .../test_base_thinking_reasoning_parser.py | 10 +- .../test_deepseekr1_reasoning_parser.py | 12 +- .../test_deepseekv3_reasoning_parser.py | 4 +- .../test_ernie45_reasoning_parser.py | 4 +- .../test_glm4_moe_reasoning_parser.py | 17 ++- .../test_granite_thinking_reasoning_parser.py | 43 +++--- .../reasoning/test_holo2_reasoning_parser.py | 4 +- .../test_k2_horizon_reasoning_parser.py | 20 ++- .../test_kimi_k2_reasoning_parser.py | 17 ++- .../test_kimi_k3_reasoning_parser.py | 19 ++- ...test_minimax_m2_append_reasoning_parser.py | 4 +- .../test_minimax_m2_reasoning_parser.py | 12 +- .../test_nemotron_v3_reasoning_parser.py | 23 ++-- .../reasoning/test_qwen3_reasoning_parser.py | 4 +- .../test_step3p5_reasoning_parser.py | 16 +-- tests/reasoning/utils.py | 9 +- .../v1/logits_processors/test_correctness.py | 125 ++++++++---------- tests/v1/logits_processors/utils.py | 16 ++- tests/v1/sample/test_diffusion_gemma_reads.py | 1 + .../v1/sample/test_diffusion_sampler_stats.py | 3 + tests/v1/sample/test_dry.py | 9 +- tests/v1/sample/test_head_dtype.py | 28 +++- tests/v1/sample/test_logprobs.py | 50 ++++--- tests/v1/sample/test_sampler.py | 1 + tests/v1/sample/test_thinking_budget_state.py | 8 +- tests/v1/sample/test_trace_replay_params.py | 3 +- tests/v1/sample/utils.py | 18 ++- .../spec_decode/test_adaptive_verification.py | 24 ++-- tests/v1/spec_decode/test_dflash2.py | 18 +-- tests/v1/spec_decode/test_dflash_lookahead.py | 8 +- .../spec_decode/test_dsv4_mtp_local_argmax.py | 5 +- tests/v1/spec_decode/test_dynamic_sd_cug.py | 8 +- tests/v1/spec_decode/test_eagle.py | 28 ++-- .../test_eagle_draft_attn_metadata.py | 2 +- .../spec_decode/test_extract_hidden_states.py | 10 +- tests/v1/spec_decode/test_lilicorr.py | 58 +++++--- .../v1/spec_decode/test_llm_base_proposer.py | 11 +- .../test_llm_base_proposer_sampling.py | 2 +- tests/v1/spec_decode/test_mtp.py | 1 - .../spec_decode/test_mtp_structured_output.py | 8 +- .../test_rejection_sampler_utils.py | 10 +- tools/pre_commit/mypy.py | 24 ++-- 118 files changed, 948 insertions(+), 605 deletions(-) diff --git a/tests/benchmarks/sweep/test_param_sweep.py b/tests/benchmarks/sweep/test_param_sweep.py index 483eb13378..749d5a995b 100644 --- a/tests/benchmarks/sweep/test_param_sweep.py +++ b/tests/benchmarks/sweep/test_param_sweep.py @@ -116,7 +116,7 @@ class TestParameterSweep: def test_from_records_list(self): """Test creating ParameterSweep from a list of records.""" - records = [ + records: list[dict[str, object]] = [ {"max_tokens": 100, "temperature": 0.7}, {"max_tokens": 200, "temperature": 0.9}, ] @@ -127,7 +127,7 @@ class TestParameterSweep: def test_read_from_dict(self): """Test creating ParameterSweep from a dict format.""" - data = { + data: dict[str, dict[str, object]] = { "experiment1": {"max_tokens": 100, "temperature": 0.7}, "experiment2": {"max_tokens": 200, "temperature": 0.9}, } @@ -149,7 +149,7 @@ class TestParameterSweep: def test_read_json_list_format(self): """Test reading JSON file with list format.""" - records = [ + records: list[dict[str, object]] = [ {"max_tokens": 100, "temperature": 0.7}, {"max_tokens": 200, "temperature": 0.9}, ] @@ -168,7 +168,7 @@ class TestParameterSweep: def test_read_json_dict_format(self): """Test reading JSON file with dict format.""" - data = { + data: dict[str, dict[str, object]] = { "experiment1": {"max_tokens": 100, "temperature": 0.7}, "experiment2": {"max_tokens": 200, "temperature": 0.9}, } @@ -190,7 +190,7 @@ class TestParameterSweep: def test_unique_benchmark_names_validation(self): """Test that duplicate _benchmark_name values raise an error.""" # Test with duplicate names in list format - records = [ + records: list[dict[str, object]] = [ {"_benchmark_name": "exp1", "max_tokens": 100}, {"_benchmark_name": "exp1", "max_tokens": 200}, ] @@ -200,7 +200,7 @@ class TestParameterSweep: def test_unique_benchmark_names_multiple_duplicates(self): """Test validation with multiple duplicate names.""" - records = [ + records: list[dict[str, object]] = [ {"_benchmark_name": "exp1", "max_tokens": 100}, {"_benchmark_name": "exp1", "max_tokens": 200}, {"_benchmark_name": "exp2", "max_tokens": 300}, @@ -212,7 +212,7 @@ class TestParameterSweep: def test_no_benchmark_names_allowed(self): """Test that records without _benchmark_name are allowed.""" - records = [ + records: list[dict[str, object]] = [ {"max_tokens": 100, "temperature": 0.7}, {"max_tokens": 200, "temperature": 0.9}, ] @@ -221,7 +221,7 @@ class TestParameterSweep: def test_mixed_benchmark_names_allowed(self): """Test that mixing records with and without _benchmark_name is allowed.""" - records = [ + records: list[dict[str, object]] = [ {"_benchmark_name": "exp1", "max_tokens": 100}, {"max_tokens": 200, "temperature": 0.9}, ] diff --git a/tests/benchmarks/test_audio_dataset.py b/tests/benchmarks/test_audio_dataset.py index d0cb3c4d99..e81c0cd173 100644 --- a/tests/benchmarks/test_audio_dataset.py +++ b/tests/benchmarks/test_audio_dataset.py @@ -13,6 +13,7 @@ import soundfile as sf import vllm.benchmarks.datasets.datasets as datasets_module import vllm.benchmarks.lib.endpoint_request_func as request_func_module from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput +from vllm.tokenizers import TokenizerLike pytestmark = pytest.mark.skip_global_cleanup @@ -44,6 +45,10 @@ class _CohereNameOnlyTokenizer(_Tokenizer): super().__init__("cohere/some-local-checkpoint") +def _as_tokenizer(tokenizer: _Tokenizer) -> TokenizerLike: + return cast(TokenizerLike, tokenizer) + + def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None: num_samples = int(duration_s * sample_rate) sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate) @@ -109,7 +114,7 @@ def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None: ] samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_as_tokenizer(_Tokenizer()), num_requests=1, output_len=32, asr_min_audio_len_sec=0.0, @@ -146,7 +151,7 @@ def test_asr_dataset_sample_handles_embedded_audio_bytes( ] samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_as_tokenizer(_Tokenizer()), num_requests=1, output_len=32, asr_min_audio_len_sec=0.0, @@ -240,7 +245,7 @@ def _make_asr_dataset(tmp_path: Path) -> datasets_module.ASRDataset: def test_asr_dataset_cohere_class_name_gets_decoder_prompt(tmp_path: Path) -> None: dataset = _make_asr_dataset(tmp_path) samples = dataset.sample( - tokenizer=CohereAsrTokenizer(), + tokenizer=_as_tokenizer(CohereAsrTokenizer()), num_requests=1, output_len=32, asr_min_audio_len_sec=0.0, @@ -255,7 +260,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt( ) -> None: dataset = _make_asr_dataset(tmp_path) samples = dataset.sample( - tokenizer=_CohereNameOnlyTokenizer(), + tokenizer=_as_tokenizer(_CohereNameOnlyTokenizer()), num_requests=1, output_len=32, asr_min_audio_len_sec=0.0, @@ -268,7 +273,7 @@ def test_asr_dataset_cohere_name_or_path_fallback_gets_decoder_prompt( def test_asr_dataset_unknown_tokenizer_gets_empty_prompt(tmp_path: Path) -> None: dataset = _make_asr_dataset(tmp_path) samples = dataset.sample( - tokenizer=_Tokenizer(name_or_path="some-other/asr-model"), + tokenizer=_as_tokenizer(_Tokenizer(name_or_path="some-other/asr-model")), num_requests=1, output_len=32, asr_min_audio_len_sec=0.0, diff --git a/tests/benchmarks/test_bfcl_dataset.py b/tests/benchmarks/test_bfcl_dataset.py index d191922319..222a57a33f 100644 --- a/tests/benchmarks/test_bfcl_dataset.py +++ b/tests/benchmarks/test_bfcl_dataset.py @@ -271,7 +271,7 @@ def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None: args.num_prompts = 1 with _patch_hf_api(fake_download): - samples = get_samples(args, fake) + samples = get_samples(args, fake) # type: ignore[arg-type] assert len(samples) == 1 assert captured["tools"] is not None, ( @@ -311,7 +311,7 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools( args.num_prompts = 1 with _patch_hf_api(fake_download): - samples = get_samples(args, _LegacyTokenizer()) + samples = get_samples(args, _LegacyTokenizer()) # type: ignore[arg-type] assert len(samples) == 1 assert samples[0].prompt_len > 0 diff --git a/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py b/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py index 149ea6625a..3817fbbb86 100644 --- a/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py +++ b/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py @@ -58,7 +58,7 @@ def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None: _write_one(jsonl) tok = _RecordingTokenizer() - get_samples(_args(str(jsonl), {"thinking": True}), tok) + get_samples(_args(str(jsonl), {"thinking": True}), tok) # type: ignore[arg-type] assert tok.captured_kwargs == {"thinking": True} @@ -70,6 +70,6 @@ def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None: _write_one(jsonl) tok = _RecordingTokenizer() - get_samples(_args(str(jsonl), None), tok) + get_samples(_args(str(jsonl), None), tok) # type: ignore[arg-type] assert tok.captured_kwargs == {} diff --git a/tests/benchmarks/test_custom_image_dataset.py b/tests/benchmarks/test_custom_image_dataset.py index f2a48abe60..3cbf733125 100644 --- a/tests/benchmarks/test_custom_image_dataset.py +++ b/tests/benchmarks/test_custom_image_dataset.py @@ -4,7 +4,7 @@ import json from argparse import Namespace from io import BytesIO from pathlib import Path -from typing import Any +from typing import Any, cast import pybase64 as base64 import pytest @@ -17,6 +17,7 @@ from vllm.benchmarks.lib.endpoint_request_func import ( _get_chat_content, _get_chat_messages, ) +from vllm.tokenizers import TokenizerLike pytestmark = pytest.mark.skip_global_cleanup @@ -31,6 +32,10 @@ class _Tokenizer: return _TokenizedPrompt(prompt) +def _tokenizer() -> TokenizerLike: + return cast(TokenizerLike, _Tokenizer()) + + def _write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None: with path.open("w") as f: for row in rows: @@ -92,7 +97,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content( ], ) - samples = get_samples(_args_for_custom_image(jsonl), _Tokenizer()) + samples = get_samples(_args_for_custom_image(jsonl), _tokenizer()) assert len(samples) == 2 assert samples[0].request_id == "req-0" @@ -106,6 +111,7 @@ def test_get_samples_custom_image_cli_path_supports_multi_image_and_content( assert samples[1].multi_modal_data is None assert isinstance(samples[1].prompt, list) assert samples[1].prompt[0] == {"type": "text", "text": "Now compare "} + assert isinstance(samples[1].prompt[1], dict) assert samples[1].prompt[1]["image_url"]["url"] == f"file://{image_c}" @@ -126,7 +132,7 @@ def test_custom_image_dataset_uses_all_image_files(tmp_path: Path) -> None: dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, ) @@ -171,7 +177,7 @@ def test_custom_image_dataset_preserves_interleaved_content_order( dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, ) @@ -181,14 +187,18 @@ def test_custom_image_dataset_preserves_interleaved_content_order( assert sample.multi_modal_data is None assert sample.prompt_len == 2 assert isinstance(sample.prompt, list) - assert [part["type"] for part in sample.prompt] == [ + parts = [] + for part in sample.prompt: + assert isinstance(part, dict) + parts.append(part) + assert [part["type"] for part in parts] == [ "text", "image_url", "text", "image_url", ] - assert sample.prompt[1]["image_url"]["url"] == f"file://{image_a}" - assert sample.prompt[3]["image_url"] == { + assert parts[1]["image_url"]["url"] == f"file://{image_a}" + assert parts[3]["image_url"] == { "url": f"file://{image_b}", "detail": "low", } @@ -223,7 +233,7 @@ def test_custom_image_dataset_wraps_interleaved_content_for_multimodal_chat( dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, enable_multimodal_chat=True, @@ -289,7 +299,7 @@ def test_custom_image_dataset_encodes_image_media_when_requested( dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, ensure_client_side_data=True, @@ -335,7 +345,7 @@ def test_custom_image_dataset_encodes_interleaved_image_media( dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) samples = dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, ensure_client_side_data=True, @@ -343,9 +353,13 @@ def test_custom_image_dataset_encodes_interleaved_image_media( sample = samples[0] assert isinstance(sample.prompt, list) - _assert_png_data_url(sample.prompt[1]["image_url"]["url"]) - _assert_png_data_url(sample.prompt[2]["image_url"]["url"]) - assert sample.prompt[2]["image_url"]["detail"] == "low" + parts = [] + for part in sample.prompt: + assert isinstance(part, dict) + parts.append(part) + _assert_png_data_url(parts[1]["image_url"]["url"]) + _assert_png_data_url(parts[2]["image_url"]["url"]) + assert parts[2]["image_url"]["detail"] == "low" @pytest.mark.benchmark @@ -363,7 +377,7 @@ def test_custom_image_dataset_rejects_invalid_image_media( dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) with pytest.raises(ValueError, match="Invalid image URL"): dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, ensure_client_side_data=True, @@ -380,7 +394,7 @@ def test_custom_image_dataset_rejects_invalid_content_part( dataset = CustomImageDataset(dataset_path=str(jsonl), disable_shuffle=True) with pytest.raises(ValueError, match="type 'text', 'image', or 'image_url'"): dataset.sample( - tokenizer=_Tokenizer(), + tokenizer=_tokenizer(), num_requests=1, output_len=32, ) diff --git a/tests/benchmarks/test_random_dataset.py b/tests/benchmarks/test_random_dataset.py index ff691ae15d..d57ab45d85 100644 --- a/tests/benchmarks/test_random_dataset.py +++ b/tests/benchmarks/test_random_dataset.py @@ -37,6 +37,7 @@ def random_dataset_params() -> Params: def _fingerprint_sample(req: SampleRequest) -> tuple[str, int, int]: """Project a SampleRequest into a comparable tuple.""" + assert isinstance(req.prompt, str) return (req.prompt, req.prompt_len, req.expected_output_len) @@ -153,6 +154,7 @@ def _mm_fingerprint_sample( - count of multimodal items - per-item type and URL prefix (e.g., 'data:image/jpeg;base64,') """ + assert isinstance(req.prompt, str) items = req.multi_modal_data or [] item_prefixes: list[str] = [] for it in items: diff --git a/tests/benchmarks/test_sampling_params.py b/tests/benchmarks/test_sampling_params.py index 3bc34a84b3..d009b4db6c 100644 --- a/tests/benchmarks/test_sampling_params.py +++ b/tests/benchmarks/test_sampling_params.py @@ -19,7 +19,7 @@ class _FakeTokenizer(TokenizerLike): # -- Properties required by TokenizerLike -- @classmethod - def from_pretrained(cls, path_or_repo_id, *a, **kw): # type: ignore[override] + def from_pretrained(cls, path_or_repo_id, *a, **kw): return cls() @property @@ -80,16 +80,16 @@ class _FakeTokenizer(TokenizerLike): def apply_chat_template(self, messages, **kw): # type: ignore[override] raise NotImplementedError - def convert_tokens_to_ids(self, tokens): # type: ignore[override] + def convert_tokens_to_ids(self, tokens): raise NotImplementedError def convert_tokens_to_string(self, tokens: list[str]) -> str: raise NotImplementedError - def decode(self, ids, skip_special_tokens: bool = False) -> str: # type: ignore[override] + def decode(self, ids, skip_special_tokens: bool = False) -> str: raise NotImplementedError - def convert_ids_to_tokens( # type: ignore[override] + def convert_ids_to_tokens( self, ids, skip_special_tokens: bool = False ) -> list[str]: raise NotImplementedError diff --git a/tests/benchmarks/test_throughput_cli.py b/tests/benchmarks/test_throughput_cli.py index c4a39f24d9..37ced49838 100644 --- a/tests/benchmarks/test_throughput_cli.py +++ b/tests/benchmarks/test_throughput_cli.py @@ -9,6 +9,7 @@ serve-side dataset coverage. """ import subprocess +from argparse import Namespace from types import SimpleNamespace import pytest @@ -289,7 +290,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None: explicit 0 instead of coercing it to None (which later crashed RandomMultiModalDataset.get_prefix on ``None <= 0``). """ - args = SimpleNamespace( + args = Namespace( random_input_len=32, random_output_len=1, random_prefix_len=0, @@ -306,7 +307,7 @@ def test_to_serve_args_supports_mm_processor_flags() -> None: def test_to_serve_args_keeps_legacy_flag_fallback() -> None: """Callers that only set the legacy flags still get them mapped.""" - args = SimpleNamespace( + args = Namespace( input_len=256, output_len=8, prefix_len=16, diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index ccd88c462b..23246d4380 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -355,9 +355,11 @@ class TestToolResultContent: if m["role"] == "user" and isinstance(m.get("content"), list) ] assert len(follow_up) == 1 - assert follow_up[0]["content"][0]["image_url"]["url"] == ( - "data:image/jpeg;base64,QUFB" - ) + content = follow_up[0]["content"] + assert isinstance(content, list) + image = content[0] + assert image["type"] == "image_url" + assert image["image_url"]["url"] == "data:image/jpeg;base64,QUFB" def test_tool_result_with_multiple_images(self): request = self._make_tool_result_request( @@ -387,7 +389,12 @@ class TestToolResultContent: if m["role"] == "user" and isinstance(m.get("content"), list) ] assert len(follow_up) == 1 - urls = [p["image_url"]["url"] for p in follow_up[0]["content"]] + content = follow_up[0]["content"] + assert isinstance(content, list) + urls = [] + for part in content: + assert part["type"] == "image_url" + urls.append(part["image_url"]["url"]) assert urls == [ "data:image/png;base64,IMG1", "https://example.com/img2.jpg", @@ -584,7 +591,8 @@ class TestThinkingBlockConversion: asst = asst_msgs[0] assert asst.get("reasoning") == "I need to call the calculator." - tool_calls = list(asst.get("tool_calls", [])) + tool_calls = asst.get("tool_calls") + assert tool_calls is not None assert len(tool_calls) == 1 assert tool_calls[0]["function"]["name"] == "calculator" # No text content alongside reasoning + tool_use. @@ -1439,7 +1447,7 @@ class TestDetectMergeInlineSystem: model_config=None, ) assert ( - AnthropicServingMessages._should_merge_inline_system(online_renderer) + AnthropicServingMessages._should_merge_inline_system(online_renderer) # type: ignore[arg-type] is expected ) assert mock_logger.warning_once.called is expected diff --git a/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py b/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py index 738cea9fd0..9e170c61fc 100644 --- a/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py +++ b/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py @@ -7,22 +7,26 @@ from unittest.mock import Mock import pytest from vllm import LLM, SamplingParams +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam +from vllm.inputs import TextPrompt +from vllm.outputs import RequestOutput def _make_mock_llm() -> LLM: llm = object.__new__(LLM) - llm.model_config = SimpleNamespace( + llm.model_config = SimpleNamespace( # type: ignore[assignment] runner_type="generate", enable_prompt_embeds=False ) return llm -def test_generate_forwards_mm_processor_kwargs() -> None: +def test_generate_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"num_crops": 4} sampling_params = SamplingParams(max_tokens=1) - llm._run_completion = Mock(return_value=["ok"]) + _run_completion = Mock(return_value=["ok"]) + monkeypatch.setattr(llm, "_run_completion", _run_completion) outputs = llm.generate( "prompt", @@ -31,17 +35,18 @@ def test_generate_forwards_mm_processor_kwargs() -> None: ) assert outputs == ["ok"] - assert llm._run_completion.call_args.kwargs["mm_processor_kwargs"] == ( + assert _run_completion.call_args.kwargs["mm_processor_kwargs"] == ( mm_processor_kwargs ) -def test_enqueue_forwards_mm_processor_kwargs() -> None: +def test_enqueue_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"do_resize": False} sampling_params = SamplingParams(max_tokens=1) - llm._add_completion_requests = Mock(return_value=["req-0"]) + _add_completion_requests = Mock(return_value=["req-0"]) + monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests) request_ids = llm.enqueue( "prompt", @@ -51,18 +56,19 @@ def test_enqueue_forwards_mm_processor_kwargs() -> None: ) assert request_ids == ["req-0"] - assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == ( + assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == ( mm_processor_kwargs ) -def test_chat_forwards_mm_processor_kwargs() -> None: +def test_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"do_pan_and_scan": True} sampling_params = SamplingParams(max_tokens=1) - messages = [{"role": "user", "content": "hello"}] + messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}] - llm._run_chat = Mock(return_value=["ok"]) + _run_chat = Mock(return_value=["ok"]) + monkeypatch.setattr(llm, "_run_chat", _run_chat) outputs = llm.chat( messages, @@ -71,18 +77,19 @@ def test_chat_forwards_mm_processor_kwargs() -> None: ) assert outputs == ["ok"] - assert llm._run_chat.call_args.kwargs["mm_processor_kwargs"] == ( - mm_processor_kwargs - ) + assert _run_chat.call_args.kwargs["mm_processor_kwargs"] == mm_processor_kwargs -def test_enqueue_chat_forwards_mm_processor_kwargs() -> None: +def test_enqueue_chat_forwards_mm_processor_kwargs( + monkeypatch: pytest.MonkeyPatch, +) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"do_pan_and_scan": True} sampling_params = SamplingParams(max_tokens=1) - messages = [{"role": "user", "content": "hello"}] + messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}] - llm._add_chat_requests = Mock(return_value=["req-0"]) + _add_chat_requests = Mock(return_value=["req-0"]) + monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests) request_ids = llm.enqueue_chat( messages, @@ -92,67 +99,79 @@ def test_enqueue_chat_forwards_mm_processor_kwargs() -> None: ) assert request_ids == ["req-0"] - assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == ( + assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == ( mm_processor_kwargs ) -def test_run_chat_forwards_mm_processor_kwargs() -> None: +def test_run_chat_forwards_mm_processor_kwargs(monkeypatch: pytest.MonkeyPatch) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"num_crops": 8} sampling_params = SamplingParams(max_tokens=1) - messages = [{"role": "user", "content": "hello"}] + messages: list[ChatCompletionMessageParam] = [{"role": "user", "content": "hello"}] sentinel_output = ["done"] - llm._add_chat_requests = Mock() - llm._run_engine = Mock(return_value=sentinel_output) + _add_chat_requests = Mock() + monkeypatch.setattr(llm, "_add_chat_requests", _add_chat_requests) + _run_engine = Mock(return_value=sentinel_output) + monkeypatch.setattr(llm, "_run_engine", _run_engine) outputs = llm._run_chat( messages=messages, params=sampling_params, - output_type=object, + output_type=RequestOutput, use_tqdm=False, mm_processor_kwargs=mm_processor_kwargs, ) assert outputs == sentinel_output - assert llm._add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == ( + assert _add_chat_requests.call_args.kwargs["mm_processor_kwargs"] == ( mm_processor_kwargs ) -def test_run_completion_forwards_mm_processor_kwargs() -> None: +def test_run_completion_forwards_mm_processor_kwargs( + monkeypatch: pytest.MonkeyPatch, +) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"min_pixels": 4 * 28 * 28} sampling_params = SamplingParams(max_tokens=1) sentinel_output = ["done"] - llm._add_completion_requests = Mock() - llm._run_engine = Mock(return_value=sentinel_output) + _add_completion_requests = Mock() + monkeypatch.setattr(llm, "_add_completion_requests", _add_completion_requests) + _run_engine = Mock(return_value=sentinel_output) + monkeypatch.setattr(llm, "_run_engine", _run_engine) outputs = llm._run_completion( prompts=["prompt"], params=sampling_params, - output_type=object, + output_type=RequestOutput, use_tqdm=False, mm_processor_kwargs=mm_processor_kwargs, ) assert outputs == sentinel_output - assert llm._add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == ( + assert _add_completion_requests.call_args.kwargs["mm_processor_kwargs"] == ( mm_processor_kwargs ) -def test_add_completion_requests_forwards_mm_processor_kwargs() -> None: +def test_add_completion_requests_forwards_mm_processor_kwargs( + monkeypatch: pytest.MonkeyPatch, +) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"max_dynamic_patch": 4} sampling_params = SamplingParams(max_tokens=1) - llm._params_to_seq = Mock(return_value=[sampling_params]) - llm._lora_request_to_seq = Mock(return_value=[None]) - llm._priority_to_seq = Mock(return_value=[0]) - llm._preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]}) + _params_to_seq = Mock(return_value=[sampling_params]) + monkeypatch.setattr(llm, "_params_to_seq", _params_to_seq) + _lora_request_to_seq = Mock(return_value=[None]) + monkeypatch.setattr(llm, "_lora_request_to_seq", _lora_request_to_seq) + _priority_to_seq = Mock(return_value=[0]) + monkeypatch.setattr(llm, "_priority_to_seq", _priority_to_seq) + _preprocess_cmpl_one = Mock(return_value={"prompt_token_ids": [1]}) + monkeypatch.setattr(llm, "_preprocess_cmpl_one", _preprocess_cmpl_one) captured_prompts = [] @@ -160,7 +179,8 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None: captured_prompts.extend(prompts) return ["req-0"] - llm._render_and_add_requests = Mock(side_effect=fake_render_and_add_requests) + _render_and_add_requests = Mock(side_effect=fake_render_and_add_requests) + monkeypatch.setattr(llm, "_render_and_add_requests", _render_and_add_requests) request_ids = llm._add_completion_requests( prompts=["prompt"], @@ -170,7 +190,7 @@ def test_add_completion_requests_forwards_mm_processor_kwargs() -> None: ) assert request_ids == ["req-0"] - llm._preprocess_cmpl_one.assert_called_once_with( + _preprocess_cmpl_one.assert_called_once_with( "prompt", None, mm_processor_kwargs=mm_processor_kwargs, @@ -183,7 +203,7 @@ def test_preprocess_cmpl_applies_mm_processor_kwargs_to_renderer( ) -> None: llm = _make_mock_llm() mm_processor_kwargs = {"num_crops": 8} - prompt = {"prompt": "", "multi_modal_data": {"image": object()}} + prompt: TextPrompt = {"prompt": "", "multi_modal_data": {"image": object()}} renderer = Mock() renderer.default_cmpl_tok_params = Mock() @@ -213,7 +233,7 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override( monkeypatch: pytest.MonkeyPatch, ) -> None: llm = _make_mock_llm() - prompt = { + prompt: TextPrompt = { "prompt": "", "multi_modal_data": {"image": object()}, "mm_processor_kwargs": {"num_crops": 2}, @@ -243,7 +263,9 @@ def test_preprocess_cmpl_keeps_prompt_mm_processor_kwargs_when_no_override( def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None: llm = _make_mock_llm() mm_processor_kwargs = {"num_crops": 8} - messages = [[{"role": "user", "content": "Describe this image."}]] + messages: list[list[ChatCompletionMessageParam]] = [ + [{"role": "user", "content": "Describe this image."}] + ] renderer = Mock() renderer.tokenizer = object() @@ -269,7 +291,9 @@ def test_preprocess_chat_applies_mm_processor_kwargs_to_renderer() -> None: def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None: llm = _make_mock_llm() - messages = [[{"role": "user", "content": "Describe this image."}]] + messages: list[list[ChatCompletionMessageParam]] = [ + [{"role": "user", "content": "Describe this image."}] + ] renderer = Mock() renderer.tokenizer = object() @@ -291,7 +315,9 @@ def test_preprocess_chat_omits_mm_processor_kwargs_when_no_override() -> None: def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None: # Matches `ChatCompletionRequest.add_special_tokens` on the server. llm = _make_mock_llm() - messages = [[{"role": "user", "content": "hi"}]] + messages: list[list[ChatCompletionMessageParam]] = [ + [{"role": "user", "content": "hi"}] + ] renderer = Mock() renderer.tokenizer = object() @@ -309,7 +335,9 @@ def test_preprocess_chat_defaults_add_special_tokens_to_false() -> None: def test_preprocess_chat_tokenization_kwargs_override_add_special_tokens() -> None: llm = _make_mock_llm() - messages = [[{"role": "user", "content": "hi"}]] + messages: list[list[ChatCompletionMessageParam]] = [ + [{"role": "user", "content": "hi"}] + ] renderer = Mock() renderer.tokenizer = object() diff --git a/tests/entrypoints/multimodal/openai/chat_completion/test_audio.py b/tests/entrypoints/multimodal/openai/chat_completion/test_audio.py index d806100e8e..5c0ca3194a 100644 --- a/tests/entrypoints/multimodal/openai/chat_completion/test_audio.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_audio.py @@ -47,18 +47,20 @@ async def client(server): @pytest.fixture(scope="session") def base64_encoded_audio() -> dict[str, str]: - return { - audio_url: encode_audio_base64(*fetch_audio(audio_url)) - for audio_url in TEST_AUDIO_URLS - } + encoded = {} + for audio_url in TEST_AUDIO_URLS: + audio, sample_rate = fetch_audio(audio_url) + encoded[audio_url] = encode_audio_base64(audio, int(sample_rate)) + return encoded @pytest.fixture(scope="session") def url_encoded_audio() -> dict[str, str]: - return { - audio_url: encode_audio_url(*fetch_audio(audio_url)) - for audio_url in TEST_AUDIO_URLS - } + encoded = {} + for audio_url in TEST_AUDIO_URLS: + audio, sample_rate = fetch_audio(audio_url) + encoded[audio_url] = encode_audio_url(audio, int(sample_rate)) + return encoded def dummy_messages_from_audio_url( diff --git a/tests/entrypoints/pooling/classify/test_offline.py b/tests/entrypoints/pooling/classify/test_offline.py index 9db7b48642..a372eb5fc2 100644 --- a/tests/entrypoints/pooling/classify/test_offline.py +++ b/tests/entrypoints/pooling/classify/test_offline.py @@ -7,6 +7,7 @@ import torch from tests.models.utils import softmax from vllm import LLM, ClassificationRequestOutput, PoolingParams +from vllm.inputs import PromptType from vllm.tasks import PoolingTask MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach" @@ -53,7 +54,8 @@ def test_token_ids_prompts(llm: LLM): @pytest.mark.skip_global_cleanup def test_list_prompts(llm: LLM): - outputs = llm.classify([prompt, prompt_token_ids], use_tqdm=False) + prompts: list[PromptType] = [prompt, prompt_token_ids] + outputs = llm.classify(prompts, use_tqdm=False) assert len(outputs) == 2 for i in range(len(outputs)): assert isinstance(outputs[i], ClassificationRequestOutput) diff --git a/tests/entrypoints/pooling/embed/test_io_processor.py b/tests/entrypoints/pooling/embed/test_io_processor.py index f06e43d143..15bdcf1d59 100644 --- a/tests/entrypoints/pooling/embed/test_io_processor.py +++ b/tests/entrypoints/pooling/embed/test_io_processor.py @@ -21,6 +21,7 @@ from vllm.entrypoints.pooling.typing import ( PoolingEngineInput, PoolingServeContext, ) +from vllm.inputs import tokens_input from vllm.outputs import PoolingOutput, PoolingRequestOutput @@ -28,7 +29,7 @@ class TestEmbeddingRequestParsing: """Unit tests for OpenAI embedding request parsing.""" def test_input_messages_parses_as_chat_request(self): - request = TypeAdapter(EmbeddingRequest).validate_python( + request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python( { "model": "test", "input": [{"role": "user", "content": "hello"}], @@ -42,7 +43,7 @@ class TestEmbeddingRequestParsing: assert request.chat_template_kwargs == {"instruction": "Represent the query: "} def test_batched_input_messages_parses_as_batch_chat_input_request(self): - request = TypeAdapter(EmbeddingRequest).validate_python( + request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python( { "model": "test", "input": [ @@ -65,7 +66,7 @@ class TestEmbeddingRequestParsing: assert request.chat_template_kwargs == {"instruction": "Represent the query: "} def test_token_ids_still_parse_as_completion_request(self): - request = TypeAdapter(EmbeddingRequest).validate_python( + request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python( { "model": "test", "input": [[1, 2, 3], [4, 5]], @@ -76,7 +77,7 @@ class TestEmbeddingRequestParsing: assert request.input == [[1, 2, 3], [4, 5]] def test_messages_still_parses_as_chat_request(self): - request = TypeAdapter(EmbeddingRequest).validate_python( + request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python( { "model": "test", "messages": [{"role": "user", "content": "hello"}], @@ -89,7 +90,7 @@ class TestEmbeddingRequestParsing: assert request.chat_template_kwargs == {"instruction": "Represent the query: "} def test_batched_messages_parses_as_batch_chat_request(self): - request = TypeAdapter(EmbeddingRequest).validate_python( + request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python( { "model": "test", "messages": [ @@ -410,13 +411,13 @@ class TestChunkedEmbeddingProcessing: @classmethod def _make_handler(cls): handler = object.__new__(EmbedIOProcessor) - handler.model_config = cls._FakeModelConfig() + handler.model_config = cls._FakeModelConfig() # type: ignore[assignment] handler.enable_chunked_processing = True return handler @staticmethod def _make_context() -> PoolingServeContext[EmbeddingCompletionRequest]: - request = TypeAdapter(EmbeddingRequest).validate_python( + request: EmbeddingRequest = TypeAdapter(EmbeddingRequest).validate_python( { "model": "test", "input": [[0, 1, 2, 3, 4], [10, 11]], @@ -431,13 +432,13 @@ class TestChunkedEmbeddingProcessing: request_id="embd-client-prompt-999-chunk-888", engine_inputs=[ PoolingEngineInput( - prompts={"prompt_token_ids": [0, 1, 2, 3, 4]}, + prompts=tokens_input(prompt_token_ids=[0, 1, 2, 3, 4]), params=pooling_params, lora_requests=None, priorities=0, ), PoolingEngineInput( - prompts={"prompt_token_ids": [10, 11]}, + prompts=tokens_input(prompt_token_ids=[10, 11]), params=pooling_params, lora_requests=None, priorities=0, diff --git a/tests/entrypoints/pooling/embed/test_offline.py b/tests/entrypoints/pooling/embed/test_offline.py index 641f8b714e..717a78d022 100644 --- a/tests/entrypoints/pooling/embed/test_offline.py +++ b/tests/entrypoints/pooling/embed/test_offline.py @@ -7,6 +7,7 @@ import torch import torch.nn.functional as F from vllm import LLM, EmbeddingRequestOutput, PoolingParams +from vllm.inputs import PromptType from vllm.tasks import PoolingTask MODEL_NAME = "intfloat/multilingual-e5-small" @@ -54,7 +55,8 @@ def test_token_ids_prompts(llm: LLM): @pytest.mark.skip_global_cleanup def test_list_prompts(llm: LLM): - outputs = llm.embed([prompt, prompt_token_ids], use_tqdm=False) + prompts: list[PromptType] = [prompt, prompt_token_ids] + outputs = llm.embed(prompts, use_tqdm=False) assert len(outputs) == 2 for i in range(len(outputs)): assert isinstance(outputs[i], EmbeddingRequestOutput) diff --git a/tests/entrypoints/pooling/embed/test_online.py b/tests/entrypoints/pooling/embed/test_online.py index 6cc0320d0b..8ac70876f4 100644 --- a/tests/entrypoints/pooling/embed/test_online.py +++ b/tests/entrypoints/pooling/embed/test_online.py @@ -12,9 +12,13 @@ import requests import torch import torch.nn.functional as F -from tests.models.language.pooling.embed_utils import run_embedding_correctness_test +from tests.models.language.pooling.embed_utils import ( + float_embeddings, + run_embedding_correctness_test, +) from tests.models.utils import check_embeddings_close from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse from vllm.entrypoints.pooling.utils import ( @@ -145,7 +149,7 @@ async def test_completion_request( assert embeddings.usage.prompt_tokens == len(input_tokens) assert embeddings.usage.total_tokens == len(input_tokens) - vllm_outputs = [d.embedding for d in embeddings.data] + vllm_outputs = float_embeddings(embeddings) run_embedding_correctness_test(hf_model, [input_text], vllm_outputs) # test input: list[int] @@ -165,7 +169,7 @@ async def test_completion_request( assert embeddings.usage.prompt_tokens == len(input_tokens) assert embeddings.usage.total_tokens == len(input_tokens) - vllm_outputs = [d.embedding for d in embeddings.data] + vllm_outputs = float_embeddings(embeddings) run_embedding_correctness_test(hf_model, [input_text], vllm_outputs) @@ -194,7 +198,7 @@ async def test_completion_request_batched( assert embeddings.usage.prompt_tokens == len(input_tokens) * N assert embeddings.usage.total_tokens == len(input_tokens) * N - vllm_outputs = [d.embedding for d in embeddings.data] + vllm_outputs = float_embeddings(embeddings) run_embedding_correctness_test(hf_model, input_texts, vllm_outputs) # test list[list[int]] @@ -214,7 +218,7 @@ async def test_completion_request_batched( assert embeddings.usage.prompt_tokens == len(input_tokens) * N assert embeddings.usage.total_tokens == len(input_tokens) * N - vllm_outputs = [d.embedding for d in embeddings.data] + vllm_outputs = float_embeddings(embeddings) run_embedding_correctness_test(hf_model, input_texts, vllm_outputs) @@ -331,7 +335,7 @@ async def test_padding(client: openai.AsyncOpenAI, model_name: str): async def test_chat_request( server: RemoteOpenAIServer, client: openai.AsyncOpenAI, model_name: str ): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": "The cat sat on the mat.", @@ -382,8 +386,8 @@ async def test_chat_request( assert chat_embeddings.created <= completion_embeddings.created # Use tolerance-based comparison for embeddings check_embeddings_close( - embeddings_0_lst=[d.embedding for d in chat_embeddings.data], - embeddings_1_lst=[d.embedding for d in completion_embeddings.data], + embeddings_0_lst=float_embeddings(chat_embeddings), + embeddings_1_lst=float_embeddings(completion_embeddings), name_0="chat", name_1="completion", ) @@ -488,7 +492,7 @@ async def test_invocations_completion_request( @pytest.mark.asyncio async def test_invocations_chat_request(server: RemoteOpenAIServer): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": "The cat sat on the mat.", diff --git a/tests/entrypoints/pooling/embed/test_online_dimensions.py b/tests/entrypoints/pooling/embed/test_online_dimensions.py index 67c6d7b134..79c0f8a340 100644 --- a/tests/entrypoints/pooling/embed/test_online_dimensions.py +++ b/tests/entrypoints/pooling/embed/test_online_dimensions.py @@ -6,7 +6,10 @@ import openai import pytest from tests.conftest import HfRunner -from tests.models.language.pooling.embed_utils import run_embedding_correctness_test +from tests.models.language.pooling.embed_utils import ( + float_embeddings, + run_embedding_correctness_test, +) from tests.models.utils import EmbedModelInfo from tests.utils import ROCM_EXTRA_ARGS, RemoteOpenAIServer from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse @@ -95,7 +98,7 @@ async def test_matryoshka( if dimensions is not None: assert len(embeddings.data[0].embedding) == dimensions - vllm_outputs = [d.embedding for d in embeddings.data] + vllm_outputs = float_embeddings(embeddings) run_embedding_correctness_test(hf_model, prompts, vllm_outputs, dimensions) if model_info.is_matryoshka: diff --git a/tests/entrypoints/pooling/embed/test_online_long_text.py b/tests/entrypoints/pooling/embed/test_online_long_text.py index ab3633cd3d..bc2e032657 100644 --- a/tests/entrypoints/pooling/embed/test_online_long_text.py +++ b/tests/entrypoints/pooling/embed/test_online_long_text.py @@ -274,6 +274,7 @@ async def test_long_text_embedding_1500_chars( # Verify embedding vector validity embedding_vector = embeddings.data[0].embedding + assert isinstance(embedding_vector, list) assert all(isinstance(x, float) for x in embedding_vector), ( "Embedding vector should contain floats" ) @@ -322,6 +323,7 @@ async def test_long_text_embedding_2500_chars( # Verify embedding vector validity embedding_vector = embeddings.data[0].embedding + assert isinstance(embedding_vector, list) assert all(isinstance(x, float) for x in embedding_vector), ( "Embedding vector should contain floats" ) @@ -409,6 +411,7 @@ async def test_chunked_vs_normal_consistency( # 验证embedding向量的有效性 embedding_vector = embeddings.data[0].embedding + assert isinstance(embedding_vector, list) assert all(isinstance(x, float) for x in embedding_vector) assert not all(x == 0 for x in embedding_vector) @@ -438,6 +441,7 @@ async def test_chunked_processing_response_format( # Verify embedding vector properties embedding_vector = embeddings.data[0].embedding + assert isinstance(embedding_vector, list) import math vector_norm = math.sqrt(sum(x * x for x in embedding_vector)) diff --git a/tests/entrypoints/pooling/reward/test_token_reward_online.py b/tests/entrypoints/pooling/reward/test_token_reward_online.py index 2878c8684e..9d89f8c423 100644 --- a/tests/entrypoints/pooling/reward/test_token_reward_online.py +++ b/tests/entrypoints/pooling/reward/test_token_reward_online.py @@ -11,6 +11,7 @@ import torch from tests.models.utils import check_embeddings_close from tests.utils import RemoteOpenAIServer +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse from vllm.entrypoints.pooling.utils import ( MetadataItem, @@ -137,7 +138,7 @@ def test_completion_request_batched(server: RemoteOpenAIServer, model_name: str) @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_chat_request(server: RemoteOpenAIServer, model_name: str): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": "The cat sat on the mat.", @@ -524,7 +525,7 @@ async def test_invocations_chat_request(server: RemoteOpenAIServer): @pytest.mark.asyncio async def test_invocations_conversation_chat_request(server: RemoteOpenAIServer): - messages = [ + messages: list[ChatCompletionMessageParam] = [ { "role": "user", "content": "The cat sat on the mat.", diff --git a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py index c83eaa544a..7d7b275032 100644 --- a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py @@ -47,7 +47,7 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor: ) processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor) - processor.model_config = SimpleNamespace(enable_prompt_embeds=False) + processor.model_config = SimpleNamespace(enable_prompt_embeds=False) # type: ignore[assignment] processor.tokenizer = TokenizersBackend( tokenizer_object=backend, unk_token="[UNK]", @@ -95,12 +95,16 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt(): truncation_side="left", ) - prompt = tok_params.apply_post_tokenization(tokenizer, prompt) + processed_prompt = tok_params.apply_post_tokenization( + tokenizer, # type: ignore[arg-type] + prompt, + ) + assert "prompt_token_ids" in processed_prompt token_type_ids = _apply_post_tokenization_to_token_type_ids( tokenizer, tok_params, token_type_ids ) - prompt_token_ids = prompt["prompt_token_ids"] + prompt_token_ids = processed_prompt["prompt_token_ids"] assert len(token_type_ids) == len(prompt_token_ids) # Keeping the last 40 tokens drops the first 10 query tokens, so 10 query diff --git a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py index ab5a21c085..ee535ced23 100644 --- a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py @@ -30,12 +30,18 @@ def test_online_forwards_truncate_prompt_tokens_to_proxy(monkeypatch): silently disables truncate_prompt_tokens for Jina rerank/score. """ proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor) - proc.valid_inputs_online = MagicMock( - return_value=ScoringData(data_1=["query"], data_2=["doc"]) + monkeypatch.setattr( + proc, + "valid_inputs_online", + MagicMock(return_value=ScoringData(data_1=["query"], data_2=["doc"])), + ) + monkeypatch.setattr(proc, "_get_token_limits", MagicMock(return_value=(0, 0))) + monkeypatch.setattr( + proc, "ensure_str", MagicMock(side_effect=lambda data: list(data)) + ) + monkeypatch.setattr( + proc, "format_docs_prompts_func", MagicMock(return_value="formatted prompt") ) - proc._get_token_limits = MagicMock(return_value=(0, 0)) - proc.ensure_str = MagicMock(side_effect=lambda data: list(data)) - proc.format_docs_prompts_func = MagicMock(return_value="formatted prompt") captured: dict[str, object] = {} @@ -71,8 +77,10 @@ def offline_processor_and_context(): backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit() proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor) proc.tokenizer = TokenizersBackend(tokenizer_object=backend) - proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False) - proc.renderer = SimpleNamespace( + proc.model_config = SimpleNamespace( # type: ignore[assignment] + max_model_len=1024, is_encoder_decoder=False + ) + proc.renderer = SimpleNamespace( # type: ignore[assignment] default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024) ) ctx = OfflineScoringInputsContext( diff --git a/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py b/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py index 464baf89e7..def1dc264f 100644 --- a/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py +++ b/tests/entrypoints/pooling/scoring/test_late_interaction_serving.py @@ -6,7 +6,7 @@ from unittest.mock import AsyncMock, Mock import pytest from vllm.entrypoints.pooling.scoring.serving import ServingScores -from vllm.entrypoints.pooling.typing import PoolingServeContext +from vllm.entrypoints.pooling.typing import PoolingEngineInput, PoolingServeContext from vllm.inputs import tokens_input from vllm.pooling_params import PoolingParams @@ -21,8 +21,18 @@ def _make_context() -> PoolingServeContext: priorities=None, prompt_extras=None, engine_inputs=[ - tokens_input(prompt_token_ids=[1]), - tokens_input(prompt_token_ids=[2]), + PoolingEngineInput( + prompts=tokens_input(prompt_token_ids=[1]), + params=PoolingParams(task="token_embed"), + lora_requests=None, + priorities=0, + ), + PoolingEngineInput( + prompts=tokens_input(prompt_token_ids=[2]), + params=PoolingParams(task="token_embed"), + lora_requests=None, + priorities=0, + ), ], n_queries=1, ) @@ -39,10 +49,13 @@ def _query_key(context: PoolingServeContext) -> str: @pytest.mark.asyncio -async def test_colliding_request_ids_use_distinct_query_cache_keys(): +async def test_colliding_request_ids_use_distinct_query_cache_keys( + monkeypatch: pytest.MonkeyPatch, +): serving = object.__new__(ServingScores) - serving._prepare_generators = AsyncMock() - serving._collect_batch = AsyncMock() + prepare_generators = AsyncMock() + monkeypatch.setattr(serving, "_prepare_generators", prepare_generators) + monkeypatch.setattr(serving, "_collect_batch", AsyncMock()) first = _make_context() second = _make_context() @@ -50,9 +63,7 @@ async def test_colliding_request_ids_use_distinct_query_cache_keys(): await serving._flash_late_interaction_encode_queries(context) await serving._flash_late_interaction_encode_docs(context) - prepared_contexts = [ - call.args[0] for call in serving._prepare_generators.await_args_list - ] + prepared_contexts = [call.args[0] for call in prepare_generators.await_args_list] first_query_key = _query_key(prepared_contexts[0]) first_doc_key = _query_key(prepared_contexts[1]) second_query_key = _query_key(prepared_contexts[2]) diff --git a/tests/entrypoints/pooling/test_io_processor.py b/tests/entrypoints/pooling/test_io_processor.py index 9d2207fdf1..d5b4909499 100644 --- a/tests/entrypoints/pooling/test_io_processor.py +++ b/tests/entrypoints/pooling/test_io_processor.py @@ -53,8 +53,8 @@ def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor): def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor): - processor.model_config = SimpleNamespace(is_encoder_decoder=False) - processor.renderer = SimpleNamespace( + processor.model_config = SimpleNamespace(is_encoder_decoder=False) # type: ignore[assignment] + processor.renderer = SimpleNamespace( # type: ignore[assignment] # Tokenization defaults only. default_cmpl_tok_params=TokenizeParams(max_total_tokens=None) ) ctx = OfflineEncodeInputsContext( diff --git a/tests/entrypoints/pooling/token_classify/test_online.py b/tests/entrypoints/pooling/token_classify/test_online.py index a8e240ab1a..91ae40deb5 100644 --- a/tests/entrypoints/pooling/token_classify/test_online.py +++ b/tests/entrypoints/pooling/token_classify/test_online.py @@ -45,6 +45,7 @@ async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: st poolings = PoolingResponse.model_validate(response.json()) assert len(poolings.data) == 1 assert len(poolings.data[0].data) == 8 + assert isinstance(poolings.data[0].data[0], list) assert len(poolings.data[0].data[0]) == 2 diff --git a/tests/entrypoints/pooling/token_embed/test_online.py b/tests/entrypoints/pooling/token_embed/test_online.py index 2c807a5b52..55a4ec0d6e 100644 --- a/tests/entrypoints/pooling/token_embed/test_online.py +++ b/tests/entrypoints/pooling/token_embed/test_online.py @@ -71,6 +71,7 @@ async def test_pooling_token_embed(server: RemoteOpenAIServer, model_name: str): assert poolings.id == f"pooling-{request_id}" assert len(poolings.data) == 1 assert len(poolings.data[0].data) == len(input_tokens) + assert isinstance(poolings.data[0].data[0], list) assert len(poolings.data[0].data[0]) == 384 diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index cb72c27904..9144928f91 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from contextlib import contextmanager +from typing import Literal from unittest.mock import MagicMock, patch import pytest @@ -14,6 +15,7 @@ from vllm.config import ( VllmConfig, set_current_vllm_config, ) +from vllm.config.cache import CacheDType from vllm.platforms import current_platform from vllm.platforms.cpu import CpuPlatform from vllm.platforms.interface import DeviceCapability @@ -21,12 +23,12 @@ from vllm.platforms.interface import DeviceCapability if current_platform.is_cuda(): from vllm.platforms.cuda import CudaPlatform else: - CudaPlatform = None + CudaPlatform = None # type: ignore[assignment] # Unavailable platform import. if current_platform.is_rocm(): from vllm.platforms.rocm import RocmPlatform else: - RocmPlatform = None + RocmPlatform = None # type: ignore[misc, assignment] # Unavailable platform import. from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -354,14 +356,14 @@ def test_invalid_backend(): def test_auto_backend_string(auto_value: str): """Test that 'auto' string value triggers automatic backend selection.""" # Using "auto" should result in backend=None (automatic selection) - attention_config = AttentionConfig(backend=auto_value) + attention_config = AttentionConfig(backend=auto_value) # type: ignore[arg-type] assert attention_config.backend is None def test_auto_backend_selection_behavior(): """Test that 'auto' backend behaves same as None (automatic selection).""" # Create config with explicit "auto" - auto_config = AttentionConfig(backend="auto") + auto_config = AttentionConfig(backend="auto") # type: ignore[arg-type] # Create config with None (default) none_config = AttentionConfig(backend=None) @@ -406,7 +408,7 @@ def test_auto_backend_selection_behavior(): reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.", ) def test_per_head_quant_scales_backend_selection( - backend_name: str, flash_attn_version: int | None, should_succeed: bool + backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool ): """Test backend selection when use_per_head_quant_scales=True.""" # Clear cache to ensure fresh backend selection @@ -559,7 +561,7 @@ def test_non_causal_autoselect_backend(): "int8_per_token_head", ], ) -def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str): +def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType): """FlashAttentionBackend must not claim support for kv_cache dtypes that it cannot handle.""" from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend @@ -569,7 +571,7 @@ def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str): @pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"]) def test_flash_attn_accepts_handled_fp8_variants( - kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch + kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch ): """FlashAttentionBackend must accept the two fp8 dtypes it can actually handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.""" @@ -876,7 +878,9 @@ def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection): config = EngineArgs( model="google/gemma-4-31B-it", dtype="bfloat16", - attention_config={"backend": "TRITON_FLASHINFER"}, + attention_config=AttentionConfig( + backend=AttentionBackendEnum.TRITON_FLASHINFER + ), ).create_engine_config() with set_current_vllm_config(config): assert ( diff --git a/tests/kernels/attention/test_cascade_flash_attn.py b/tests/kernels/attention/test_cascade_flash_attn.py index 80c5c853de..36c0c51de4 100755 --- a/tests/kernels/attention/test_cascade_flash_attn.py +++ b/tests/kernels/attention/test_cascade_flash_attn.py @@ -151,7 +151,7 @@ def test_cascade( max_seqlen_k=max_kv_len, softmax_scale=scale, causal=True, - window_size=window_size, + window_size=list(window_size), block_table=block_tables, softcap=soft_cap if soft_cap is not None else 0, ) diff --git a/tests/kernels/attention/test_cpu_attn.py b/tests/kernels/attention/test_cpu_attn.py index f79b1b15d2..1aa5b056c4 100644 --- a/tests/kernels/attention/test_cpu_attn.py +++ b/tests/kernels/attention/test_cpu_attn.py @@ -836,7 +836,7 @@ def varlen_encoder_zentorch_sdpa( key, value, output, - SimpleNamespace(query_start_loc=query_start_loc, causal=False), + SimpleNamespace(query_start_loc=query_start_loc, causal=False), # type: ignore[arg-type] scale, sliding_window if sliding_window is not None else -1, alibi_slopes, diff --git a/tests/kernels/attention/test_flash_attn.py b/tests/kernels/attention/test_flash_attn.py index c58156c685..ea1d6744c5 100644 --- a/tests/kernels/attention/test_flash_attn.py +++ b/tests/kernels/attention/test_flash_attn.py @@ -187,7 +187,7 @@ def test_varlen_with_paged_kv( max_seqlen_k=max_kv_len, softmax_scale=scale, causal=True, - window_size=window_size, + window_size=list(window_size), block_table=block_tables, softcap=soft_cap if soft_cap is not None else 0, fa_version=fa_version, @@ -274,7 +274,7 @@ def test_fa4_hd256_paged_call_shape( max_seqlen_k=num_pages * block_size, softmax_scale=scale, causal=True, - window_size=window_size, + window_size=list(window_size), block_table=block_tables[:, :num_pages], num_splits=1, fa_version=4, diff --git a/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py b/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py index af41722605..feaeb4144f 100644 --- a/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py +++ b/tests/kernels/attention/test_kimi_k3_mla_key_concat_kv_cache.py @@ -197,11 +197,6 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No mixed_cache = initial_cache.clone() reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots) scale_inv = torch.ones(1, device="cuda", dtype=torch.float32) - kwargs = { - "ds_mla": cache_format == "fp8_ds_mla", - "q_scale_inv": scale_inv if cache_format == "fp8" else None, - "cache_scale_inv": scale_inv if cache_format == "fp8" else None, - } output = fused_mla_decode_q_concat_kv_cache_insert( inputs["ql_nope"], @@ -210,7 +205,9 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No inputs["k_pe"], mixed_cache, slots, - **kwargs, + ds_mla=cache_format == "fp8_ds_mla", + q_scale_inv=scale_inv if cache_format == "fp8" else None, + cache_scale_inv=scale_inv if cache_format == "fp8" else None, ) expected = _latent_query(inputs["ql_nope"], inputs["q_pe"]) diff --git a/tests/kernels/attention/test_mha_attn.py b/tests/kernels/attention/test_mha_attn.py index 8fa4963a6a..c8106af9e6 100644 --- a/tests/kernels/attention/test_mha_attn.py +++ b/tests/kernels/attention/test_mha_attn.py @@ -277,7 +277,7 @@ def test_mha_attn_varlen_forward_flashinfer( # Override vllm config so get_vit_attn_backend returns FLASHINFER (simulates # --mm-encoder-attn-backend=FLASHINFER). vllm_config = get_current_vllm_config() - old_model_config = getattr(vllm_config, "model_config", None) + old_model_config = vllm_config.model_config minimal_model_config = type( "MinimalModelConfig", (), @@ -376,7 +376,7 @@ def test_mha_attn_varlen_forward_aiter_fp8( torch.set_default_dtype(dtype) vllm_config = get_current_vllm_config() - old_model_config = getattr(vllm_config, "model_config", None) + old_model_config = vllm_config.model_config minimal_model_config = type( "MinimalModelConfig", (), @@ -442,7 +442,7 @@ def test_mha_attn_varlen_forward_aiter_fp8( ) def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config): vllm_config = get_current_vllm_config() - old_model_config = getattr(vllm_config, "model_config", None) + old_model_config = vllm_config.model_config vllm_config.model_config = type( "MinimalModelConfig", (), @@ -468,7 +468,7 @@ def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config): ) def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config): vllm_config = get_current_vllm_config() - old_model_config = getattr(vllm_config, "model_config", None) + old_model_config = vllm_config.model_config vllm_config.model_config = type( "MinimalModelConfig", (), diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index 9d3b1500e1..62fcb0beba 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -1689,8 +1689,10 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch, layout): ) backend = sparse_attn_mod.MiniMaxM3SparseBackend - assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts() - assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts() + layouts = backend.supported_kv_cache_layouts() + assert layouts is not None + assert KVCacheLayout.LHBNC in layouts + assert KVCacheLayout.LBHNC in layouts nb, h = 3, 1 cache_dtype = current_platform.fp8_dtype() @@ -1772,19 +1774,22 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch): import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True) - kwargs = dict( - topk_blocks=TOPK, - sparse_block_size=BLOCK_SIZE, - num_index_heads=1, - index_head_dim=HEAD_DIM, - indexer_kv_dtype="fp8_e4m3", - max_model_len=8192, - ) + + def unsupported_reason() -> str | None: + return indexer_aiter_mod.aiter_indexer_unsupported_reason( + topk_blocks=TOPK, + sparse_block_size=BLOCK_SIZE, + num_index_heads=1, + index_head_dim=HEAD_DIM, + # The implementation also accepts this legacy fp8 alias. + indexer_kv_dtype="fp8_e4m3", # type: ignore[arg-type] + max_model_len=8192, + ) monkeypatch.setattr( indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: False ) - reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs) + reason = unsupported_reason() assert reason is not None and "AITER sparse PA attend" in reason # With the attend asked for, the gate moves on to the kernel-contract @@ -1792,7 +1797,7 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch): monkeypatch.setattr( indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True ) - reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs) + reason = unsupported_reason() assert reason is None or "AITER sparse PA attend" not in reason diff --git a/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py b/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py index f6ea80cca0..c3dee9b580 100644 --- a/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py +++ b/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py @@ -235,14 +235,14 @@ def test_msa_metadata_builder_prepares_cutlass_for_regular_decode( builder.num_q_heads = 64 builder.num_kv_heads = 4 builder.topk_blocks = TOPK - builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4) + builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4) # type: ignore[assignment] builder.kv_cache_dtype = "fp8_e4m3" builder.decode_backend = "cutlass" - builder.msa_cutlass_plan_cache = object() + builder.msa_cutlass_plan_cache = object() # type: ignore[assignment] # Mocked plan-cache sentinel. metadata = builder.build( 0, - SimpleNamespace( + SimpleNamespace( # type: ignore[arg-type] seq_lens_cpu_upper_bound=torch.full((batch,), 257, dtype=torch.int32) ), ) @@ -377,7 +377,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay( with torch.cuda.stream(stream), override_forward_context(forward_context): capture = BreakableCUDAGraphCapture() with capture: - query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv) + query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv) # type: ignore[arg-type] if query_fp8 is not None: query_fp8.zero_() run_attention(query_fp8) diff --git a/tests/kernels/attention/test_rocm_aiter_fa.py b/tests/kernels/attention/test_rocm_aiter_fa.py index dd2870ccf1..c3b2c0754f 100644 --- a/tests/kernels/attention/test_rocm_aiter_fa.py +++ b/tests/kernels/attention/test_rocm_aiter_fa.py @@ -783,6 +783,9 @@ def test_aiter_fa_shared_kv_matches_reference( 32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16 ) value = torch.randn_like(key) + from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionImpl + + assert isinstance(target.impl, AiterFlashAttentionImpl) target.impl.do_kv_cache_update( target, key, diff --git a/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py b/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py index ae66f98a88..9193c03608 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py @@ -82,6 +82,7 @@ def _run_verify_block(): ) from vllm.config import SpeculativeConfig from vllm.config.vllm import set_current_vllm_config + from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import MLAAttentionSpec from vllm.v1.worker.workspace import init_workspace_manager @@ -106,7 +107,7 @@ def _run_verify_block(): vllm_config.speculative_config = SpeculativeConfig( method="ngram", num_speculative_tokens=QLEN - 1 ) - vllm_config.model_config.get_num_attention_heads = types.MethodType( + vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign] lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS, vllm_config.model_config, ) @@ -202,7 +203,8 @@ def _run_verify_block(): lambda: spy, ), ): - impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None) + assert isinstance(impl, AiterMLAImpl) + impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None) # type: ignore[arg-type] # Spy decode path does not read layer. return metadata, captured diff --git a/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py b/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py index 23dd09128b..fe97013b25 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py @@ -121,7 +121,7 @@ def _build_prefill_metadata( builder = object.__new__(AiterMLAMetadataBuilder) builder.num_heads = num_heads - builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM) + builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM) # type: ignore[assignment] builder._init_fp8_prefill_ps_buffers( max_num_reqs=len(seq_lens), max_prefill_qlen=max_q, @@ -136,7 +136,7 @@ def _build_prefill_metadata( prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q) metadata = SimpleNamespace(prefill=prefill, num_decodes=0) common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu) - builder._build_fp8_prefill_ps_metadata(metadata, common) + builder._build_fp8_prefill_ps_metadata(metadata, common) # type: ignore[arg-type] return metadata, total_q, builder diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sink.py b/tests/kernels/attention/test_rocm_aiter_mla_sink.py index 6de46afe3e..354800d048 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sink.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sink.py @@ -167,7 +167,10 @@ def test_sparse_mla_sink_matches_ragged_reference( impl.sinks = sinks output, lse = impl._forward_mla( - SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), q, kv, metadata + SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), + q, + kv, + metadata, # type: ignore[arg-type] ) kv_flat = kv_ref[:, 0] references = [] @@ -241,7 +244,7 @@ def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes( SimpleNamespace(_q_scale=None, _k_scale=None), torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype), torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype), - metadata, + metadata, # type: ignore[arg-type] ) @@ -381,7 +384,7 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"), paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32), ) - impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) + impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) # type: ignore[arg-type] impl.num_heads = num_heads impl.head_size = head_dim impl.kv_lora_rank = value_dim @@ -438,9 +441,9 @@ def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch, ] if layout == "BLNHC": with pytest.raises(ValueError, match="does not satisfy"): - resolve_kv_cache_layout(config, supported) + resolve_kv_cache_layout(config, supported) # type: ignore[arg-type] else: - assert resolve_kv_cache_layout(config, supported).name == layout + assert resolve_kv_cache_layout(config, supported).name == layout # type: ignore[arg-type] @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) @@ -491,7 +494,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype): actual, _ = impl.forward_mqa( (q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]), kv, - metadata, + metadata, # type: ignore[arg-type] SimpleNamespace(_q_scale=None, _k_scale=None), ) references = [ diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py index aabce97fac..c71536a448 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py @@ -39,10 +39,10 @@ def _make_builder(): topk_tokens = 4 builder.device = torch.device("cpu") - builder.kv_cache_spec = SimpleNamespace(block_size=1) + builder.kv_cache_spec = SimpleNamespace(block_size=1) # type: ignore[assignment] builder.model_dtype = torch.bfloat16 builder.kv_cache_dtype = "fp8" - builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) + builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) # type: ignore[assignment] builder.topk_tokens = topk_tokens builder.req_id_per_token_buffer = torch.zeros( max_num_batched_tokens, dtype=torch.int32, device="cpu" diff --git a/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py b/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py index 311ee89a8d..b37433cd90 100644 --- a/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py +++ b/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py @@ -227,7 +227,9 @@ def _decode_metadata(case, rows, ratio, query_lens): num_decode_tokens=len(rows), num_prefills=0, num_prefill_tokens=0, - decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]), + decode=types.SimpleNamespace( # type: ignore[arg-type] + block_table=block_table, seq_lens=lens[:, None] + ), decode_row_lens=lens, decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK, decode_native=native, @@ -278,7 +280,7 @@ def _prefill_metadata( num_decode_tokens=0, num_prefills=len(case.seq_lens), num_prefill_tokens=len(rows), - prefill=types.SimpleNamespace(chunks=chunks), + prefill=types.SimpleNamespace(chunks=chunks), # type: ignore[arg-type] prefill_plans=plans, gather_launches=plan_gather_launches(chunks, plans, gather_rows), ) diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index 387e690e1b..b6e41876b6 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -955,7 +955,7 @@ def test_rocm_capture_metadata_sets_adaptive_marker(monkeypatch) -> None: ) builder = object.__new__(rocm_mod.DeepseekV4ROCMAiterMLASparseMetadataBuilder) - actual = builder.build_for_cudagraph_capture(SimpleNamespace()) + actual = builder.build_for_cudagraph_capture(SimpleNamespace()) # type: ignore[arg-type] # Patched base method ignores metadata. assert actual is metadata assert actual.for_cudagraph_capture is _on_gfx950() @@ -1563,7 +1563,7 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None: "sliding_window": window_size, }, ) - vllm_config.speculative_config = SimpleNamespace( + vllm_config.speculative_config = SimpleNamespace( # type: ignore[assignment] num_speculative_tokens=upper_query_len - 1, parallel_drafting=False, enable_adaptive_verification=True, diff --git a/tests/kernels/attention/test_triton_unified_attention_diffkv.py b/tests/kernels/attention/test_triton_unified_attention_diffkv.py index ea90b60a75..213d896c03 100644 --- a/tests/kernels/attention/test_triton_unified_attention_diffkv.py +++ b/tests/kernels/attention/test_triton_unified_attention_diffkv.py @@ -153,6 +153,7 @@ def test_triton_unified_attn_diffkv_vs_reference( else: from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func + assert fa_version is not None # FA's TMA path needs aligned singleton strides (num_kv_heads == 1). fa_k = canonicalize_singleton_dim_strides(key_cache) fa_v = canonicalize_singleton_dim_strides(value_cache) diff --git a/tests/kernels/helion/test_config_manager.py b/tests/kernels/helion/test_config_manager.py index 63f8130849..2df31b4ed6 100644 --- a/tests/kernels/helion/test_config_manager.py +++ b/tests/kernels/helion/test_config_manager.py @@ -79,7 +79,7 @@ class TestConfigSet: config_set = ConfigSet("test_kernel") with pytest.raises(KeyError, match="platform 'h100' not found"): - config_set.get_config("h100", "nonexistent") + config_set.get_config("h100", CaseKey({"batch": 32})) config_data = {"num_warps": 8, "num_stages": 4} data = { diff --git a/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py b/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py index 14a4356e14..1dfd4a3ba6 100644 --- a/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py +++ b/tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py @@ -81,7 +81,7 @@ class TestDynamicPerTokenScaledFp8QuantConfigPicker: assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32}) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] args = _generate_fake_input(16, 4096) selected_key = pick_config(args, config_keys) diff --git a/tests/kernels/helion/test_fused_qk_norm_rope.py b/tests/kernels/helion/test_fused_qk_norm_rope.py index a973533169..3f37932dd1 100644 --- a/tests/kernels/helion/test_fused_qk_norm_rope.py +++ b/tests/kernels/helion/test_fused_qk_norm_rope.py @@ -127,7 +127,7 @@ class TestFusedQkNormRopeConfigPicker: ) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] args = _generate_fake_input(16, 4096, 128) selected_key = pick_config(args, config_keys) diff --git a/tests/kernels/helion/test_pattern_matching.py b/tests/kernels/helion/test_pattern_matching.py index fc7345ca0b..debbf1bae4 100644 --- a/tests/kernels/helion/test_pattern_matching.py +++ b/tests/kernels/helion/test_pattern_matching.py @@ -38,6 +38,7 @@ from torch._inductor.pattern_matcher import ( ) from torch.fx.experimental.proxy_tensor import make_fx +from vllm.kernels.helion.case_key import CaseKey from vllm.kernels.helion.config_manager import ConfigManager from vllm.kernels.helion.register import HelionKernelWrapper @@ -45,7 +46,7 @@ from vllm.kernels.helion.register import HelionKernelWrapper @contextlib.contextmanager def _helion_mock_context(): configs = { - "default": helion.Config(block_sizes=[64], num_warps=2, num_stages=2), + CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2), } mock_config_manager = Mock(spec=ConfigManager) mock_config_manager.get_platform_configs = Mock(return_value=configs) @@ -88,7 +89,7 @@ class TestMakeFxHop: raw_kernel_func=raw_add_scale, op_name="test_make_fx", fake_impl=lambda *a, **kw: None, - config_picker=lambda args, keys: "default", + config_picker=lambda args, keys: CaseKey.default(), ) def fn(x, y): @@ -145,7 +146,7 @@ class TestMakeFxHop: raw_kernel_func=raw_silu_mul, op_name="test_pm_silu_mul", fake_impl=lambda *a, **kw: None, - config_picker=lambda args, keys: "default", + config_picker=lambda args, keys: CaseKey.default(), ) def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: diff --git a/tests/kernels/helion/test_per_token_group_fp8_quant.py b/tests/kernels/helion/test_per_token_group_fp8_quant.py index d9dd8a8817..d06f9f389b 100644 --- a/tests/kernels/helion/test_per_token_group_fp8_quant.py +++ b/tests/kernels/helion/test_per_token_group_fp8_quant.py @@ -107,7 +107,7 @@ class TestPerTokenGroupFp8QuantConfigPicker: ) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] args = _generate_fake_input(16, 4096, 128) selected_key = pick_config(args, config_keys) diff --git a/tests/kernels/helion/test_register.py b/tests/kernels/helion/test_register.py index dd4274bfaf..ae1db78224 100644 --- a/tests/kernels/helion/test_register.py +++ b/tests/kernels/helion/test_register.py @@ -397,6 +397,7 @@ class TestHelionKernelWrapper: ) assert wrapper._disabled is True + assert wrapper._disabled_reason is not None assert "No configs available" in wrapper._disabled_reason def test_disabled_wrapper_raises_on_call(self, sample_kernel): diff --git a/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py b/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py index d0650dfc7a..2839ae74c4 100644 --- a/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py +++ b/tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py @@ -89,7 +89,7 @@ class TestRmsNormDynamicPerTokenQuantConfigPicker: assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32}) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] args = _generate_fake_input(16, 4096) selected_key = pick_config(args, config_keys) diff --git a/tests/kernels/helion/test_rms_norm_per_block_quant.py b/tests/kernels/helion/test_rms_norm_per_block_quant.py index e9c003f4e3..00440cf41d 100644 --- a/tests/kernels/helion/test_rms_norm_per_block_quant.py +++ b/tests/kernels/helion/test_rms_norm_per_block_quant.py @@ -112,7 +112,7 @@ class TestRmsNormPerBlockQuantConfigPicker: ) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] args = _generate_fake_input(16, 4096, 128) selected_key = pick_config(args, config_keys) diff --git a/tests/kernels/helion/test_silu_and_mul_per_block_quant.py b/tests/kernels/helion/test_silu_and_mul_per_block_quant.py index 1cccb88538..061704cca2 100644 --- a/tests/kernels/helion/test_silu_and_mul_per_block_quant.py +++ b/tests/kernels/helion/test_silu_and_mul_per_block_quant.py @@ -96,7 +96,7 @@ class TestSiluAndMulPerBlockQuantConfigPicker: ) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] args = _generate_fake_input(16, 4096, 128) selected_key = pick_config(args, config_keys) diff --git a/tests/kernels/helion/test_silu_mul_fp8.py b/tests/kernels/helion/test_silu_mul_fp8.py index bd3131e08d..2531dd817d 100644 --- a/tests/kernels/helion/test_silu_mul_fp8.py +++ b/tests/kernels/helion/test_silu_mul_fp8.py @@ -83,7 +83,7 @@ class TestSiluMulFp8ConfigPicker: assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256}) def test_config_picker_no_configs(self): - config_keys: list[dict] = [] + config_keys: list[CaseKey] = [] input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda") scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") @@ -355,6 +355,7 @@ class TestSiluMulFp8Integration: registered_kernels = get_registered_kernels() kernel_wrapper = registered_kernels["silu_mul_fp8"] fake_impl = kernel_wrapper._fake_impl + assert fake_impl is not None fake_output = fake_impl(input_tensor, scale) diff --git a/tests/models/language/pooling/embed_utils.py b/tests/models/language/pooling/embed_utils.py index 3b818aef97..f119d42721 100644 --- a/tests/models/language/pooling/embed_utils.py +++ b/tests/models/language/pooling/embed_utils.py @@ -7,6 +7,15 @@ import pytest from tests.conftest import HfRunner from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy +from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse + + +def float_embeddings(response: EmbeddingResponse) -> list[list[float]]: + embeddings = [] + for d in response.data: + assert isinstance(d.embedding, list) + embeddings.append(d.embedding) + return embeddings def run_embedding_correctness_test( diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index 90310d6e6d..9fbe2f5792 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -53,12 +53,10 @@ def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch): imageio = ImageMediaIO() # Verify that different args pass/fail assertions as expected. - videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 1.0}) + videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0) _ = videoio.load_bytes(b"test") - videoio = VideoMediaIO( - imageio, **{"num_frames": 10, "fps": 1.0, "not_used": "not_used"} - ) + videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0, not_used="not_used") _ = videoio.load_bytes(b"test") with pytest.raises(AssertionError, match="bad num_frames"): @@ -66,11 +64,11 @@ def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch): _ = videoio.load_bytes(b"test") with pytest.raises(AssertionError, match="bad num_frames"): - videoio = VideoMediaIO(imageio, **{"num_frames": 9, "fps": 1.0}) + videoio = VideoMediaIO(imageio, num_frames=9, fps=1.0) _ = videoio.load_bytes(b"test") with pytest.raises(AssertionError, match="bad fps"): - videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 2.0}) + videoio = VideoMediaIO(imageio, num_frames=10, fps=2.0) _ = videoio.load_bytes(b"test") @@ -86,14 +84,14 @@ def test_opencv_video_io_colorspace(tmp_path, is_color: bool, fourcc: str, ext: image = Image.open(image_path) if not is_color: - image_path = f"{tmp_path}/test_grayscale_image.png" + image_path = Path(tmp_path) / "test_grayscale_image.png" image = image.convert("L") image.save(image_path) # Convert to gray RGB for comparison image = image.convert("RGB") video_path = f"{tmp_path}/test_RGB_video.{ext}" create_video_from_image( - image_path, + str(image_path), video_path, num_frames=2, is_color=is_color, @@ -385,7 +383,9 @@ def test_pynvvideocodec_unrelated_error_propagates( with pytest.raises(RuntimeError) as exc_info: PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec( - None, b"video", None + None, + b"video", + None, # type: ignore[arg-type] # Fails before target metadata is used. ) assert exc_info.value is original_error diff --git a/tests/multimodal/test_audio.py b/tests/multimodal/test_audio.py index c816e979ab..379fc3b0ba 100644 --- a/tests/multimodal/test_audio.py +++ b/tests/multimodal/test_audio.py @@ -179,7 +179,7 @@ def test_audio_resampler_invalid_method(dummy_audio): # Validated eagerly so a bad method fails at construction, not on the # first audio request. with pytest.raises(ValueError, match="Invalid resampling method"): - AudioResampler(target_sr=22050, method="invalid") + AudioResampler(target_sr=22050, method="invalid") # type: ignore[arg-type] def test_audio_resampler_no_target_sr(dummy_audio): @@ -378,6 +378,7 @@ class TestMultiModalDataParserChannelNormalization: result = parser._parse_audio_data((stereo_audio, 16000)) # Check that result is mono (1D) + assert result is not None audio_item = result.get(0) assert audio_item.ndim == 1, f"Expected 1D mono audio, got {audio_item.ndim}D" assert audio_item.shape == (3,), f"Expected shape (3,), got {audio_item.shape}" @@ -404,6 +405,7 @@ class TestMultiModalDataParserChannelNormalization: result = parser._parse_audio_data((stereo_audio, 16000)) # Check that result preserves original shape (after resampling) + assert result is not None audio_item = result.get(0) # When target_channels=None, stereo audio should be preserved assert audio_item.ndim == 2, f"Expected 2D stereo audio, got {audio_item.ndim}D" @@ -425,6 +427,7 @@ class TestMultiModalDataParserChannelNormalization: result = parser._parse_audio_data((mono_audio, 16000)) # Check that result is still mono (1D) + assert result is not None audio_item = result.get(0) assert audio_item.ndim == 1 assert audio_item.shape == (16000,) @@ -446,6 +449,7 @@ class TestMultiModalDataParserChannelNormalization: result = parser._parse_audio_data((surround_audio, 16000)) # Check that result is stereo (2 channels) + assert result is not None audio_item = result.get(0) assert audio_item.ndim == 2 assert audio_item.shape[0] == 2 # 2 channels @@ -483,6 +487,7 @@ class TestAudioPipelineE2E: # Process audio through the parser result = parser._parse_audio_data((stereo_torchaudio, 16000)) + assert result is not None audio_output = result.get(0) # Verify output is mono 1D @@ -512,6 +517,7 @@ class TestAudioPipelineE2E: # Process audio through the parser result = parser._parse_audio_data((stereo_soundfile, 16000)) + assert result is not None audio_output = result.get(0) # Verify output is mono 1D @@ -537,6 +543,7 @@ class TestAudioPipelineE2E: # Process audio through the parser result = parser._parse_audio_data((mono_pyav, 16000)) + assert result is not None audio_output = result.get(0) # Verify output is still mono 1D @@ -572,6 +579,7 @@ class TestAudioPipelineE2E: # Process audio through the parser result = parser._parse_audio_data((surround_audio, 16000)) + assert result is not None audio_output = result.get(0) # Verify output is mono 1D @@ -603,6 +611,7 @@ class TestAudioPipelineE2E: # Process audio through the parser # Note: Parser expects numpy, so we convert first (simulating real usage) result = parser._parse_audio_data((stereo_torch.numpy(), 16000)) + assert result is not None audio_output = result.get(0) # Verify output is mono 1D numpy array @@ -630,6 +639,7 @@ class TestAudioPipelineE2E: # Process audio through the parser result = parser._parse_audio_data((stereo_audio, 16000)) + assert result is not None audio_output = result.get(0) # Verify output preserves stereo (2D) @@ -654,6 +664,7 @@ class TestAudioPipelineE2E: # Process audio through the parser result = parser._parse_audio_data((stereo_48k, 48000)) + assert result is not None audio_output = result.get(0) # Verify output is mono 1D at target sample rate @@ -677,6 +688,7 @@ class TestAudioPipelineE2E: ) result = parser._parse_audio_data((short_stereo, 16000)) + assert result is not None audio_output = result.get(0) # Should still produce mono output diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index ffd7d61cf2..a4966565f6 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -11,7 +11,6 @@ import torch from vllm.config import ModelConfig, ParallelConfig, VllmConfig from vllm.config.multimodal import MultiModalConfig from vllm.multimodal.cache import ( - BaseMultiModalProcessorCache, BaseMultiModalReceiverCache, LruKeyReplicatedReceiverCache, LruKeyReplicatedSenderCache, @@ -145,6 +144,8 @@ def _compare_caches( cache_1_p0 = processor_cache_from_config(config_1) cache_1_p1 = engine_receiver_cache_from_config(config_1) + assert config_0.model_config.multimodal_config is not None + assert config_1.model_config.multimodal_config is not None cache_size_gb = max( config_0.model_config.multimodal_config.mm_processor_cache_gb, config_1.model_config.multimodal_config.mm_processor_cache_gb, @@ -280,6 +281,7 @@ def test_oversized_item_is_served_uncached(): p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type] p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type] + assert item is not None assert p0.get_and_update_item((item, []), "big")[0] is item assert not p0.is_cached_item("big") assert p1.get_and_update_item(item, "big") is item @@ -318,6 +320,7 @@ def test_mm_cache_miss_raises_and_recovers(): assert p0.is_cached_item(mm_hash) # On the next request P0 short-circuits to data=None -- the drift bug when # P1 lacks the item. + assert item is not None hit = p0.get_and_update_item((item, []), mm_hash) assert hit[0] is None @@ -326,6 +329,7 @@ def test_mm_cache_miss_raises_and_recovers(): p0.invalidate(mm_hash) assert not p0.is_cached_item(mm_hash) + assert item is not None resent = p0.get_and_update_item((item, []), mm_hash) assert resent[0] is item # MISS again -> data is resent assert p1.get_and_update_item(item, mm_hash) == item # P1 now caches it @@ -459,6 +463,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch): address_item, _ = p0.get_and_update_item((item, []), mm_hash) first = _feature(mm_hash, address_item) p1.get_and_update_features([first]) + assert first.data is not None assert torch.equal(first.data["dummy"].data, item["dummy"].data) # Request 2 (identical, fully prefix-covered): the sender hit takes @@ -476,6 +481,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch): assert covered_uncached.data is None # The address item is resolved to the cached payload. + assert covered_cached.data is not None assert torch.equal(covered_cached.data["dummy"].data, item["dummy"].data) # The hit's writer references were acknowledged by the worker, so the @@ -488,7 +494,7 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch): def _run_test_cache_eviction_lru( - p0_cache: BaseMultiModalProcessorCache, + p0_cache: LruKeyReplicatedSenderCache, p1_cache: BaseMultiModalReceiverCache, base_item_size: int, ): @@ -603,7 +609,7 @@ def test_cache_eviction_lru_cache(): # image_B is protected from eviction then image_i cannot be added. # This proving normal eviction and reuse behavior. def _run_test_cache_eviction_shm( - p0_cache: BaseMultiModalProcessorCache, + p0_cache: ShmObjectStoreSenderCache, p1_cache: BaseMultiModalReceiverCache, base_item_size: int, ): @@ -860,9 +866,9 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error): async def call_release(): if use_async: - await AsyncLLM.release_kv_cache_memory(engine) + await AsyncLLM.release_kv_cache_memory(engine) # type: ignore[arg-type] else: - LLMEngine.release_kv_cache_memory(engine) + LLMEngine.release_kv_cache_memory(engine) # type: ignore[arg-type] if release_error: with pytest.raises(RuntimeError, match=release_error): @@ -894,9 +900,10 @@ def test_sleep_wake_preserves_mm_cache_consistency(): """Regression for vllm-project/vllm#42995.""" from vllm import LLM, SamplingParams from vllm.assets.image import ImageAsset + from vllm.inputs import TextPrompt image = ImageAsset("stop_sign").pil_image - prompt = { + prompt: TextPrompt = { "prompt": _SLEEP_VISION_PROMPT, "multi_modal_data": {"image": image}, } diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index 971d97a2ca..b8d9dc7c9d 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -131,6 +131,7 @@ def test_hash_collision_video_num_frames(): } video = MediaWithBytes((frames, metadata), source) items = MultiModalDataParser()._parse_video_data([video]) + assert items is not None return items.get_all_items_for_hash()[0] hasher = MultiModalHasher @@ -155,6 +156,7 @@ def test_hash_video_tensor_frames(): } video = MediaWithBytes((frames, metadata), source) items = MultiModalDataParser()._parse_video_data([video]) + assert items is not None return items.get_all_items_for_hash()[0] np_frames = np.zeros((2, 8, 8, 3), dtype=np.uint8) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 485ea31659..63c6080b2d 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import time +from collections.abc import Sequence from contextlib import nullcontext from copy import deepcopy from types import SimpleNamespace @@ -868,7 +869,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid): ], ) def test_budget_caps_prevent_dummy_input_validation_failure( - model_id, user_limit, supported_limit + model_id, user_limit, supported_limit, monkeypatch ): limit_mm_per_prompt = {"image": user_limit} @@ -878,7 +879,9 @@ def test_budget_caps_prevent_dummy_input_validation_failure( ) processor = MULTIMODAL_REGISTRY.create_processor(model_config) - processor.info.get_supported_mm_limits = lambda: {"image": supported_limit} + monkeypatch.setattr( + processor.info, "get_supported_mm_limits", lambda: {"image": supported_limit} + ) # This is what budget.py uses to derive mm_counts allowed = processor.info.allowed_mm_limits @@ -936,7 +939,7 @@ def test_hf_processor_init_kwargs( ) processor = ctx.get_hf_processor( - DummyProcessor, # type: ignore[arg-type] + DummyProcessor, **inference_kwargs, ) assert processor.a == expected_kwargs["a"] @@ -967,7 +970,7 @@ def test_hf_processor_call_kwargs( tokenizer=None, ) - processor = ctx.get_hf_processor(DummyProcessor) # type: ignore[arg-type] + processor = ctx.get_hf_processor(DummyProcessor) result = ctx.call_hf_processor(processor, {}, inference_kwargs) assert result == expected_kwargs @@ -1278,7 +1281,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities( processor: SimpleNamespace, expected: dict[str, set[str]], ) -> None: - info = BaseProcessingInfo(SimpleNamespace()) + info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type] info.__dict__["supported_mm_limits"] = supported_mm_limits monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor) @@ -1288,7 +1291,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities( def test_supported_mm_processor_kwargs_is_cached( monkeypatch: pytest.MonkeyPatch, ) -> None: - info = BaseProcessingInfo(SimpleNamespace()) + info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type] expected = {"images_kwargs": {"size"}} calls = 0 @@ -1541,7 +1544,10 @@ def test_processor_inputs_hashes_scope_kwargs_by_modality(): "video": [np.zeros((2, 8, 8, 3), dtype=np.uint8)], } ) - mm_uuid_items = {"image": ["image-uuid"], "video": ["video-uuid"]} + mm_uuid_items: dict[str, Sequence[str | None]] = { + "image": ["image-uuid"], + "video": ["video-uuid"], + } def get_hashes(video_frames: int, image_size: int, video_size: int): return ProcessorInputs( @@ -1621,7 +1627,7 @@ def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right): [(None, 491520, 491520), (True, 491520, 8192), (True, 128, 128), (False, 128, 128)], ) def test_dummy_inputs_scheduler_budget( - chunked_prefill, max_model_len, expected_seq_len + chunked_prefill, max_model_len, expected_seq_len, monkeypatch ): ctx = build_model_context( "llava-hf/llava-v1.6-mistral-7b-hf", @@ -1634,11 +1640,9 @@ def test_dummy_inputs_scheduler_budget( ctx.model_config, tokenizer=ctx.tokenizer, ) - processor.apply = lambda *args, **kwargs: {"prompt_token_ids": [7]} - - kwargs = {} + scheduler_config = None if chunked_prefill is not None: - kwargs["scheduler_config"] = SchedulerConfig( + scheduler_config = SchedulerConfig( max_model_len=max_model_len, is_encoder_decoder=False, max_num_batched_tokens=8192, @@ -1646,5 +1650,10 @@ def test_dummy_inputs_scheduler_budget( enable_chunked_prefill=chunked_prefill, ) - result = processor.get_dummy_mm_inputs({"image": 1}, **kwargs) + monkeypatch.setattr( + processor, "apply", lambda *args, **kwargs: {"prompt_token_ids": [7]} + ) + result = processor.get_dummy_mm_inputs( + {"image": 1}, scheduler_config=scheduler_config + ) assert len(result["prompt_token_ids"]) == expected_seq_len diff --git a/tests/multimodal/test_registry.py b/tests/multimodal/test_registry.py index f395d36933..1c4e69ba3c 100644 --- a/tests/multimodal/test_registry.py +++ b/tests/multimodal/test_registry.py @@ -24,4 +24,4 @@ def test_create_processor_error_uses_served_model_name(): ValueError, match="friendly-model-name is not a multimodal model", ): - MULTIMODAL_REGISTRY.create_processor(model_config) + MULTIMODAL_REGISTRY.create_processor(model_config) # type: ignore[arg-type] diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 432521c11f..3c0b9db209 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -7,6 +7,7 @@ import sys import threading from contextlib import ExitStack, contextmanager from pathlib import Path +from typing import Any import numpy as np import numpy.typing as npt @@ -83,24 +84,28 @@ def _fresh_decoder_pool(): @VIDEO_LOADER_REGISTRY.register("test_video_loader_1") class TestVideoLoader1(VideoLoader): @classmethod - def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray: - return FAKE_OUTPUT_1 + def load_bytes( + cls, data: bytes, num_frames: int = -1, **kwargs + ) -> tuple[npt.NDArray, dict[str, Any]]: + return FAKE_OUTPUT_1, {} @VIDEO_LOADER_REGISTRY.register("test_video_loader_2") class TestVideoLoader2(VideoLoader): @classmethod - def load_bytes(cls, data: bytes, num_frames: int = -1) -> npt.NDArray: - return FAKE_OUTPUT_2 + def load_bytes( + cls, data: bytes, num_frames: int = -1, **kwargs + ) -> tuple[npt.NDArray, dict[str, Any]]: + return FAKE_OUTPUT_2, {} def test_video_loader_registry(): custom_loader_1 = VIDEO_LOADER_REGISTRY.load("test_video_loader_1") - output_1 = custom_loader_1.load_bytes(b"test") + output_1, _ = custom_loader_1.load_bytes(b"test") np.testing.assert_array_equal(output_1, FAKE_OUTPUT_1) custom_loader_2 = VIDEO_LOADER_REGISTRY.load("test_video_loader_2") - output_2 = custom_loader_2.load_bytes(b"test") + output_2, _ = custom_loader_2.load_bytes(b"test") np.testing.assert_array_equal(output_2, FAKE_OUTPUT_2) @@ -488,7 +493,7 @@ def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot(): old_decoder = FakeDecoder() slot = PyNvVideoCodecDecoderSlot(FakeStream()) - slot.decoder = old_decoder + slot.decoder = old_decoder # type: ignore[assignment] # Fake retained GPU decoder. slot.source_path = "valid.mp4" class FakeNvc: @@ -596,7 +601,9 @@ def test_pynvvideocodec_h200_recovers_after_unsupported_8k(): _pynv_decoder_pool.max_slots = old_max -def test_pynvvideocodec_cross_subclass_shares_single_pool(): +def test_pynvvideocodec_cross_subclass_shares_single_pool( + monkeypatch: pytest.MonkeyPatch, +): """Regression test for GHSA-j682-9xp5-rrf3. Multiple subclasses of PyNvVideoCodecVideoBackendMixin must share the @@ -623,11 +630,12 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool(): with _fresh_decoder_pool() as pool: pool.max_slots = 2 - orig_create = PyNvVideoCodecVideoBackendMixin._create_decoder_slot - PyNvVideoCodecVideoBackendMixin._create_decoder_slot = classmethod( - fake_create_slot - ) - try: + with monkeypatch.context() as m: + m.setattr( + PyNvVideoCodecVideoBackendMixin, + "_create_decoder_slot", + classmethod(fake_create_slot), + ) with ExitStack() as stack: stack.enter_context(MixinSubclassA._borrow_decoder_slot()) stack.enter_context(MixinSubclassB._borrow_decoder_slot()) @@ -652,8 +660,6 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool(): assert create_count == 2 assert len(pool.slots) == 2 - finally: - PyNvVideoCodecVideoBackendMixin._create_decoder_slot = orig_create @pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"]) @@ -662,7 +668,7 @@ def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object): VideoBackend.load_bytes( b"fake video", backend=PYNVVIDEOCODEC_VIDEO_BACKEND, - hw_decoders=hw_decoders, # type: ignore[arg-type] + hw_decoders=hw_decoders, ) @@ -799,6 +805,7 @@ def test_video_processor_from_model_repo( ) backend = get_video_loader_backend_for_processor(video_processor) + assert backend is not None loader = VIDEO_LOADER_REGISTRY.load(backend) assert isinstance(loader, expected_loader_cls), ( f"{model_repo!r}: backend={backend!r} loaded " @@ -1693,7 +1700,9 @@ def test_glm5next_backend_codec_parity(tmp_path, backend): path = _write_gray_video(tmp_path, total_frames, fps) # Dense default sampling (gap 5) and a sparse max_frames cap (gap 20). for max_frames in (None, 6): - kwargs = {} if max_frames is None else {"max_frames": max_frames} + kwargs: dict[str, Any] = ( + {} if max_frames is None else {"max_frames": max_frames} + ) expected = glm_sample_frame_indices( total_frames, float(fps), 12.0, max_frame_count=max_frames ) diff --git a/tests/quantization/test_auto_gptq.py b/tests/quantization/test_auto_gptq.py index e586155225..fbf405210a 100644 --- a/tests/quantization/test_auto_gptq.py +++ b/tests/quantization/test_auto_gptq.py @@ -123,8 +123,8 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases(): method = object.__new__(AutoGPTQMoEMethod) method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {}) method.input_dtype = None - method.experts_cls = None - method.moe = SimpleNamespace(w13_num_shards=2) + method.experts_cls = None # type: ignore[assignment] # Weight creation does not select experts. + method.moe = SimpleNamespace(w13_num_shards=2) # type: ignore[assignment] layer = torch.nn.Module() method.create_weights( @@ -170,7 +170,7 @@ def test_routed_experts_loads_per_expert_biases(): ("w1", torch.tensor([1.0, 2.0, 3.0, 4.0])), ("w3", torch.tensor([5.0, 6.0, 7.0, 8.0])), ): - assert RoutedExperts.weight_loader( + assert RoutedExperts.weight_loader( # type: ignore[call-overload] loader, w13_bias, loaded, @@ -180,7 +180,7 @@ def test_routed_experts_loads_per_expert_biases(): return_success=True, ) - assert RoutedExperts.weight_loader( + assert RoutedExperts.weight_loader( # type: ignore[call-overload] loader, w2_bias, torch.tensor([9.0, 10.0, 11.0, 12.0]), diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index f2eeb9e9d9..dc09805610 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -193,7 +193,7 @@ class DummyFusedMoE: def make_config(**overrides) -> INCConfig: - kwargs = { + kwargs: dict[str, Any] = { "weight_bits": 4, "group_size": 128, "sym": True, @@ -208,7 +208,7 @@ def make_config(**overrides) -> INCConfig: def make_layer_config(**overrides) -> INCLayerConfig: - kwargs = { + kwargs: dict[str, Any] = { "bits": 4, "group_size": 128, "sym": True, @@ -423,7 +423,7 @@ def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix( config = make_config(extra_config={"model.moe_layer": {"bits": 16}}) layer = object.__new__(RoutedExperts) - layer.moe_config = None + layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub. method = config.get_quant_method(layer, "moe_layer") @@ -730,7 +730,7 @@ def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None: ) layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() + layer.moe_config = DummyMoeConfig() # type: ignore[assignment] layer_config = make_layer_config(bits=bits) method = INCWna16Scheme().get_moe_method( make_config(), layer, "model.layers.0.mlp", layer_config @@ -861,7 +861,7 @@ def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe( config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16") layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() + layer.moe_config = DummyMoeConfig() # type: ignore[assignment] method = config.get_quant_method(layer, "model.layers.0.mlp") @@ -890,7 +890,7 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe( config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4") layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() + layer.moe_config = DummyMoeConfig() # type: ignore[assignment] ignored_method = config.get_quant_method( object.__new__(LinearBase), "model.layers.0.self_attn.q_proj" @@ -1186,7 +1186,7 @@ def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None: ) layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() + layer.moe_config = DummyMoeConfig() # type: ignore[assignment] method = INCWna16Scheme().get_moe_method( make_config(), layer, @@ -1669,7 +1669,7 @@ def test_inc_get_quant_method_unquantized_moe_returns_unquantized( when extra_config has bits >= 16.""" config = make_config(extra_config={"layer": {"bits": 16}}) layer = object.__new__(RoutedExperts) - layer.moe_config = None # UnquantizedFusedMoEMethod accepts moe_config + layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub. class DummyUnquantizedFusedMoEMethod: def __init__(self, moe_config) -> None: @@ -2094,7 +2094,7 @@ def _with_w4a8_kernel(monkeypatch) -> None: def _dispatch(layer_config=None): return INCWna16Scheme().get_linear_method( - object(), object(), "layer", layer_config or make_layer_config() + make_config(), object(), "layer", layer_config or make_layer_config() ) diff --git a/tests/quantization/test_cutlass_w4a16.py b/tests/quantization/test_cutlass_w4a16.py index 8bf0f0a408..9ce1ec220e 100644 --- a/tests/quantization/test_cutlass_w4a16.py +++ b/tests/quantization/test_cutlass_w4a16.py @@ -99,6 +99,7 @@ def test_machete_rejects_invalid_config( ) can_impl, reason = MacheteLinearKernel.can_implement(config) assert not can_impl + assert reason is not None assert expected_reason in reason diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index e391d516da..f880064e37 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention.attention import ( set_default_quant_scales, ) from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.fp8 import ( Fp8Config, Fp8LinearMethod, @@ -79,17 +80,22 @@ def test_deepseek_v41_mxfp8_scale_loading( for module in (model_module, linear_module): monkeypatch.setattr(module, "get_tensor_model_parallel_world_size", lambda: 2) monkeypatch.setattr(module, "get_tensor_model_parallel_rank", lambda: tp_rank) - tp_args = dict(quant_config=quant_config, bias=False) block = torch.nn.Module() block.attn = torch.nn.Module() - block.attn.wq_b = ColumnParallelLinear(128, 128, **tp_args) - block.attn.fused_wqa_wkv = MergedColumnParallelLinear(128, [128, 64], **tp_args) + block.attn.wq_b = ColumnParallelLinear( + 128, 128, quant_config=quant_config, bias=False + ) + block.attn.fused_wqa_wkv = MergedColumnParallelLinear( + 128, [128, 64], quant_config=quant_config, bias=False + ) block.ffn = torch.nn.Module() block.ffn.shared_experts = torch.nn.Module() block.ffn.shared_experts.gate_up_proj = MergedColumnParallelLinear( - 128, [128, 128], **tp_args + 128, [128, 128], quant_config=quant_config, bias=False + ) + block.ffn.shared_experts.down_proj = RowParallelLinear( + 128, 128, quant_config=quant_config, bias=False ) - block.ffn.shared_experts.down_proj = RowParallelLinear(128, 128, **tp_args) block.ffn.experts = torch.nn.Module() block.ffn.experts.register_parameter( "weight_scale", torch.nn.Parameter(torch.empty(3, 4, dtype=torch.uint8), False) @@ -111,9 +117,11 @@ def test_deepseek_v41_mxfp8_scale_loading( model.config = SimpleNamespace(num_attention_heads=2) model.quant_config = quant_config model.use_sequence_parallel = False - model.get_expert_mapping = lambda: [ - ("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1") - ] + monkeypatch.setattr( + model, + "get_expert_mapping", + lambda: [("experts.weight_scale", "experts.0.w1.weight_scale", 0, "w1")], + ) checkpoints = [ ("attn.wq_b", "attn.wq_b", 128, 128, 0, None), @@ -211,7 +219,10 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales( vllm_config = SimpleNamespace( quant_config=SimpleNamespace(weight_block_size=weight_block_size) ) - resolved = model_module._linear_scale_param_name(vllm_config, expert_dtype) + resolved = model_module._linear_scale_param_name( + vllm_config, # type: ignore[arg-type] + expert_dtype, + ) assert resolved == scale_name mapper = vl_module._make_deepseek_v4_vl_weights_mapper(expert_dtype, resolved) @@ -373,7 +384,7 @@ def test_deepgemm_mxfp8_preserves_weight_and_scale_values( @pytest.mark.parametrize("config_source", ["deepseek", "mxfp8"]) @pytest.mark.parametrize("num_tokens", [1, 7, 128]) def test_mxfp8_bmm_loads_and_projects_grouped_weights( - dist_init, default_vllm_config, prequantized, config_source, num_tokens + dist_init, default_vllm_config, monkeypatch, prequantized, config_source, num_tokens ): """BMM metadata set after construction selects grouped weight processing.""" from vllm.model_executor.kernels.linear.mxfp8.deep_gemm import ( @@ -398,10 +409,11 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights( pytest.skip("DeepGEMM MXFP8 BMM requires Blackwell") default_vllm_config.model_config = SimpleNamespace(dtype=torch.bfloat16) - quant_config = DeepseekV4FP8Config( + deepseek_config = DeepseekV4FP8Config( is_checkpoint_fp8_serialized=True, weight_block_size=[32, 32] ) - quant_config._resolved_expert_dtype = "fp4" + deepseek_config._resolved_expert_dtype = "fp4" + quant_config: QuantizationConfig = deepseek_config if config_source == "mxfp8": quant_config = get_quantization_config("mxfp8").from_config( {"quant_method": "mxfp8"} @@ -432,7 +444,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights( model.config = SimpleNamespace(num_attention_heads=2) model.quant_config = quant_config model.use_sequence_parallel = False - model.get_expert_mapping = lambda: [] + monkeypatch.setattr(model, "get_expert_mapping", lambda: []) model.load_weights( [ ("layers.0.attn.wo_a.weight", weight), @@ -443,6 +455,7 @@ def test_mxfp8_bmm_loads_and_projects_grouped_weights( weight.float().reshape(8, 32, 16, 32) * torch.exp2(scales.float() - 127)[:, None, :, None] ).reshape(2, 128, 512) + assert isinstance(linear.quant_method, ModelOptLinearMethod) linear.quant_method.process_weights_after_loading(linear) linear.quant_method.process_weights_after_loading(linear) assert isinstance(linear.quant_method.kernel, DeepGemmMxfp8BmmLinearKernel) @@ -785,7 +798,7 @@ def test_kv_cache_scale_sync_to_host_copies(): set_default_quant_scales(layer, register_buffer=True) layer.kv_cache_dtype = "fp8" - method = BaseKVCacheMethod(quant_config=None) + method = BaseKVCacheMethod(quant_config=None) # type: ignore[arg-type] # Scale synchronization does not use quant_config. method.create_weights(layer) # 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale. checkpoint_scale = torch.tensor(0.3, dtype=torch.float32) diff --git a/tests/quantization/test_gfx950_moe.py b/tests/quantization/test_gfx950_moe.py index b71e952b76..422fb20a3d 100644 --- a/tests/quantization/test_gfx950_moe.py +++ b/tests/quantization/test_gfx950_moe.py @@ -8,6 +8,7 @@ These tests run on real hardware — no mocks. Skipped on non-GFX950 platforms. import pytest import torch +from vllm.config.kernel import MoEBackend from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, @@ -53,7 +54,7 @@ def disable_rocm_aiter(monkeypatch: pytest.MonkeyPatch): set_rocm_aiter(monkeypatch, False) -def _make_w4a4_moe_config(moe_backend: str = "auto") -> FusedMoEConfig: +def _make_w4a4_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig: from vllm.model_executor.layers.fused_moe.activation import MoEActivation return FusedMoEConfig( diff --git a/tests/quantization/test_int8_moe_oracle.py b/tests/quantization/test_int8_moe_oracle.py index 9580eb9ab3..aff0354527 100644 --- a/tests/quantization/test_int8_moe_oracle.py +++ b/tests/quantization/test_int8_moe_oracle.py @@ -10,6 +10,7 @@ CUDA (SM >= 7.5), ROCm, or XPU — not just gfx950. import pytest import torch +from vllm.config.kernel import MoEBackend from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEParallelConfig, @@ -44,7 +45,7 @@ requires_int8_moe = pytest.mark.skipif( DEVICE = current_platform.device_type -def _make_int8_moe_config(moe_backend: str = "auto") -> FusedMoEConfig: +def _make_int8_moe_config(moe_backend: MoEBackend = "auto") -> FusedMoEConfig: from vllm.model_executor.layers.fused_moe.activation import MoEActivation return FusedMoEConfig( diff --git a/tests/quantization/test_lm_head.py b/tests/quantization/test_lm_head.py index b89be3d055..8e331c4e16 100644 --- a/tests/quantization/test_lm_head.py +++ b/tests/quantization/test_lm_head.py @@ -15,6 +15,7 @@ from tests.quantization.utils import load_model_without_vllm_runner from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 CompressedTensorsConfig, ) @@ -80,6 +81,7 @@ def test_quantized_lm_head_matches_linear( dtype=torch.bfloat16, head_dtype=None ) default_vllm_config.kernel_config.linear_backend = "humming" + quant_config: QuantizationConfig if quant_format is None: quant_config = ModelOptNvFp4Config( quant_method=preset, diff --git a/tests/quantization/test_modelopt.py b/tests/quantization/test_modelopt.py index d7b52723cb..eea4b6218d 100644 --- a/tests/quantization/test_modelopt.py +++ b/tests/quantization/test_modelopt.py @@ -843,6 +843,7 @@ def test_modelopt_mixed_precision_builds_w4a16_sibling_config(): } config = m.ModelOptMixedPrecisionConfig.from_config(hf_quant_config) + assert isinstance(config, m.ModelOptMixedPrecisionConfig) assert config.nvfp4_config.quant_method == "NVFP4" assert config.w4a16_nvfp4_config.quant_method == "W4A16_NVFP4" diff --git a/tests/quantization/test_moe_wna16.py b/tests/quantization/test_moe_wna16.py index 344b8ecbd7..abbcb86916 100644 --- a/tests/quantization/test_moe_wna16.py +++ b/tests/quantization/test_moe_wna16.py @@ -218,11 +218,13 @@ def test_gptq_nfirst_weights_pass_through_for_triton(): def test_moe_wna16_setup_forwards_selected_backend(monkeypatch): method = object.__new__(MoeWNA16Method) - method.experts_cls = object + method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel. method.wna16_backend = WNA16MoEBackend.HUMMING - method.moe = object() + method.moe = object() # type: ignore[assignment] # Opaque mocked-config sentinel. quant_config = object() - method.get_fused_moe_quant_config = lambda layer: quant_config + monkeypatch.setattr( + method, "get_fused_moe_quant_config", lambda layer: quant_config + ) layer = SimpleNamespace(_expert_routing_tables=lambda: (None, None, None)) captured = {} kernel = object() @@ -233,7 +235,7 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch): monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel) - method._setup_kernel(layer) + method._setup_kernel(layer) # type: ignore[arg-type] # Routing-table stub. assert method.moe_kernel is kernel assert captured["backend"] == WNA16MoEBackend.HUMMING @@ -278,7 +280,7 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch): ), ) - assert method.get_fused_moe_quant_config(layer) is quant_config + assert method.get_fused_moe_quant_config(layer) is quant_config # type: ignore[arg-type] # Identity sentinel. @pytest.mark.skipif( diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index c7495326e2..c8a75bf1f0 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -16,6 +16,7 @@ from tests.quantization.utils import ( _test_online_quant_peak_mem_impl, is_quant_method_supported, load_model_without_vllm_runner, + quant_config_args, ) from vllm import _custom_ops as ops from vllm._aiter_ops import rocm_aiter_ops @@ -180,7 +181,7 @@ def test_qianfan_online_fp8_keeps_vision_layers_unquantized( config = OnlineQuantizationConfig(args) other_config = OnlineQuantizationConfig(args) model = QianfanOCRForConditionalGeneration.__new__( - QianfanOCRForConditionalGeneration + QianfanOCRForConditionalGeneration # type: ignore[type-abstract] ) model._patch_quant_config( SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)), @@ -209,9 +210,9 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( monkeypatch, ) -> None: method = object.__new__(Nvfp4OnlineMoEMethod) - method.moe = SimpleNamespace(is_act_and_mul=True) - method.nvfp4_backend = object() - method.experts_cls = object + method.moe = SimpleNamespace(is_act_and_mul=True) # type: ignore[assignment] + method.nvfp4_backend = object() # type: ignore[assignment] # Opaque mocked-backend sentinel. + method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel. method.moe_quant_config = None method.moe_kernel = None @@ -239,7 +240,7 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( ) make_kernel = Mock(return_value=kernel) get_quant_config = Mock(return_value=quant_config) - method.get_fused_moe_quant_config = get_quant_config + monkeypatch.setattr(method, "get_fused_moe_quant_config", get_quant_config) monkeypatch.setattr( "vllm.model_executor.layers.quantization.online.nvfp4." @@ -355,7 +356,7 @@ def test_online_prequantized_compatibility( checkpoint_config = checkpoint_config_factory() checkpoint_config.online_quantization_config = OnlineQuantizationConfig( - QuantizationConfigArgs(linear="mxfp8") + quant_config_args(linear="mxfp8") ) config = checkpoint_config @@ -408,7 +409,7 @@ def test_online_ignore_keeps_checkpoint_quantization_linear( quant_config = _fully_quantized_quark_config() prefix = "model.layers.0.self_attn.o_proj" quant_config.online_quantization_config = OnlineQuantizationConfig( - QuantizationConfigArgs(linear="mxfp8", ignore=[prefix]) + quant_config_args(linear="mxfp8", ignore=[prefix]) ) monkeypatch.setattr( quant_config.online_quantization_config, @@ -437,7 +438,7 @@ def test_online_quantization_rejects_prequantized_moe( prefix = "model.layers.0.mlp.experts" quant_config = _fully_quantized_quark_config() quant_config.online_quantization_config = OnlineQuantizationConfig( - QuantizationConfigArgs(linear="mxfp4", moe="mxfp4") + quant_config_args(linear="mxfp4", moe="mxfp4") ) with pytest.raises(ValueError, match="pre-quantized layer"): @@ -460,7 +461,7 @@ def test_activation_only_override_applies_to_checkpoint_method( model_config = ModelConfig( model=str(tmp_path), quantization="compressed-tensors", - quantization_config=QuantizationConfigArgs(moe={"activation": "mxfp8"}), + quantization_config=quant_config_args(moe={"activation": "mxfp8"}), hf_overrides={ "quantization_config": { "quant_method": "compressed-tensors", @@ -514,7 +515,7 @@ def test_online_overlay_loads_checkpoint_config_file(tmp_path) -> None: model_config = ModelConfig( model=str(tmp_path), quantization="awq", - quantization_config=QuantizationConfigArgs(linear="mxfp8"), + quantization_config=quant_config_args(linear="mxfp8"), ) result = weight_utils.get_quant_config(model_config, LoadConfig()) @@ -574,7 +575,7 @@ def test_online_shorthand_selects_checkpoint_or_online_config( def test_log_online_quantization_for_composable_config(monkeypatch) -> None: """Composable configs log their nested online quantization results.""" - online_config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8")) + online_config = OnlineQuantizationConfig(quant_config_args(linear="mxfp8")) online_config.quantized_layers = { "model.layers.0.self_attn.o_proj": ("linear", "mxfp8", None), "model.layers.1.self_attn.o_proj": ("linear", "mxfp8", None), @@ -1164,7 +1165,7 @@ def test_online_quantization_records_global_config( default_vllm_config, dist_init ) -> None: default_vllm_config.model_config = ModelConfig() - config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_block")) + config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_block")) prefix = "model.layers.0.self_attn.o_proj" layer = ColumnParallelLinear( input_size=1, @@ -1218,7 +1219,7 @@ def test_online_quantization_targets_reject_unsupported_layer() -> None: def test_log_online_quantization(default_vllm_config, monkeypatch) -> None: - config = OnlineQuantizationConfig(QuantizationConfigArgs(linear="fp8_per_tensor")) + config = OnlineQuantizationConfig(quant_config_args(linear="fp8_per_tensor")) config.quantized_layers = { "model.layers.0.mlp.down_proj": ("linear", "fp8_per_tensor", None), "model.layers.1.mlp.down_proj": ("linear", "fp8_per_tensor", None), @@ -1431,7 +1432,7 @@ def test_online_int8_moe_w2_scale_matches_unsharded(monkeypatch) -> None: layer.w2_weight = torch.nn.Parameter(w2_in, requires_grad=False) layer.num_experts = layer.local_num_experts = w13.shape[0] method = SimpleNamespace(moe=SimpleNamespace(tp_size=moe_tp_size)) - Int8OnlineMoEMethod._quantize_weights(method, layer) + Int8OnlineMoEMethod._quantize_weights(method, layer) # type: ignore[arg-type] return layer.w2_weight, layer.w2_scale full_weight, full_scale = quantize(w2, 1) diff --git a/tests/quantization/test_online_mxfp4.py b/tests/quantization/test_online_mxfp4.py index 9495a15ba6..5f6ca98c15 100644 --- a/tests/quantization/test_online_mxfp4.py +++ b/tests/quantization/test_online_mxfp4.py @@ -10,6 +10,7 @@ against the pure-torch reference in `reference_mxfp4.py`. import pytest import torch +from vllm.config.kernel import MoEBackend from vllm.config.model import ModelConfig from vllm.model_executor.kernels.linear import _POSSIBLE_MXFP4_KERNELS from vllm.model_executor.kernels.linear.mxfp4.aiter import ( @@ -282,7 +283,7 @@ def test_online_mxfp4_tp_weight_quant_matches_unsharded(tp_size: int): ], ) def test_online_mxfp4_moe_matches_quark( - moe_backend: str, + moe_backend: MoEBackend, unpadded_hidden_size: int, unpadded_intermediate_size: int, default_vllm_config, diff --git a/tests/quantization/test_quantization_config_args.py b/tests/quantization/test_quantization_config_args.py index 0bf203ac45..3723c6be30 100644 --- a/tests/quantization/test_quantization_config_args.py +++ b/tests/quantization/test_quantization_config_args.py @@ -6,6 +6,7 @@ from unittest.mock import Mock import pytest +from tests.quantization.utils import quant_config_args, quant_spec from vllm.config.quantization import ( QUANT_KEY_NAMES, QuantizationConfigArgs, @@ -30,7 +31,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( def test_quant_spec_resolves_string_to_quant_key(): - spec = QuantSpec(weight="mxfp8", activation="fp8_per_token") + spec = quant_spec(weight="mxfp8", activation="fp8_per_token") assert spec.weight == kMxfp8Dynamic assert spec.activation == kFp8DynamicTokenSym @@ -42,12 +43,12 @@ def test_quant_spec_accepts_quant_key_directly(): def test_quant_spec_string_representation_uses_quantization_name(): - assert str(QuantSpec(weight="mxfp4")) == "mxfp4" + assert str(quant_spec(weight="mxfp4")) == "mxfp4" def test_quant_spec_rejects_unknown_name(): with pytest.raises(ValueError, match="unknown quantization name"): - QuantSpec(weight="not_a_real_format") + quant_spec(weight="not_a_real_format") # ---- QuantizationConfigArgs string shorthand on linear/moe ---------------- @@ -55,7 +56,7 @@ def test_quant_spec_rejects_unknown_name(): def test_args_linear_string_resolves_via_quant_key_names(): # A bare QUANT_KEY_NAMES entry desugars to QuantSpec(weight=). - args = QuantizationConfigArgs(linear="fp8_per_block_static") + args = quant_config_args(linear="fp8_per_block_static") assert args.linear == QuantSpec(weight=kFp8Static128BlockSym) assert args.moe is None @@ -65,7 +66,7 @@ def test_args_moe_string_resolves_via_online_shorthand(): # (so `linear: "fp8_per_block"` and `moe: "fp8_per_block"` produce the # same per-layer-kind spec the `--quantization fp8_per_block` shorthand # would). - args = QuantizationConfigArgs(moe="fp8_per_block") + args = quant_config_args(moe="fp8_per_block") assert args.moe == QuantSpec(weight=kFp8Static128BlockSym) @@ -73,11 +74,11 @@ def test_args_string_shorthand_missing_slot_raises(): # int8_per_channel_weight_only sets only `moe`; using it on `linear` # has no defined spec and should raise rather than silently no-op. with pytest.raises(ValueError, match="does not define a linear spec"): - QuantizationConfigArgs(linear="int8_per_channel_weight_only") + quant_config_args(linear="int8_per_channel_weight_only") def test_args_accepts_dict_form(): - args = QuantizationConfigArgs(moe={"activation": "mxfp8"}) + args = quant_config_args(moe={"activation": "mxfp8"}) assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic) @@ -91,6 +92,7 @@ def test_targets_reject_non_string_keys(): def test_resolve_shorthand_only_populates_both_slots(): args = resolve_quantization_config("fp8_per_block", None) + assert args is not None assert args.linear == QuantSpec(weight=kFp8Static128BlockSym) assert args.moe == QuantSpec(weight=kFp8Static128BlockSym) @@ -105,6 +107,7 @@ def test_resolve_int8_shorthand_leaves_linear_unset(): # int8_per_channel_weight_only is MoE-only; linear stays None so that # OnlineQuantizationConfig leaves Linear layers in full precision. args = resolve_quantization_config("int8_per_channel_weight_only", None) + assert args is not None assert args.linear is None assert args.moe == QuantSpec(weight=kInt8StaticChannelSym) @@ -113,6 +116,7 @@ def test_resolve_quantization_config_only(): # When only `quantization_config` is given (e.g. for an already-quantized # checkpoint that needs an activation override), it's returned as-is. args = resolve_quantization_config(None, {"moe": {"activation": "mxfp8"}}) + assert args is not None assert args.linear is None assert args.moe == QuantSpec(weight=None, activation=kMxfp8Dynamic) @@ -124,13 +128,14 @@ def test_resolve_merges_explicit_over_shorthand(): "fp8_per_tensor", {"linear": "fp8_per_block"}, ) + assert args is not None assert args.linear == QuantSpec(weight=kFp8Static128BlockSym) assert args.moe == QuantSpec(weight=kFp8StaticTensorSym) def test_resolve_quantization_config_with_checkpoint_quantization(): args = resolve_quantization_config("gptq", {"linear": "fp8_per_block"}) - assert args == QuantizationConfigArgs(linear="fp8_per_block") + assert args == quant_config_args(linear="fp8_per_block") # ---- QUANT_KEY_NAMES coverage -------------------------------------------- @@ -140,15 +145,15 @@ def test_quant_key_names_round_trip(): # Every advertised name should round-trip through QuantSpec without error # and produce the same QuantKey it maps to. for name, expected in QUANT_KEY_NAMES.items(): - assert QuantSpec(weight=name).weight == expected, name - assert QuantSpec(activation=name).activation == expected, name + assert quant_spec(weight=name).weight == expected, name + assert quant_spec(activation=name).activation == expected, name def test_static_block_weight_paired_with_dynamic_block_activation(): # The block-FP8 shorthand pair: 128x128 static weights + 1x128 dynamic # activations. Pinning this so renames in QUANT_KEY_NAMES don't quietly # rewire the kernel dispatch. - spec = QuantSpec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic") + spec = quant_spec(weight="fp8_per_block_static", activation="fp8_per_block_dynamic") assert spec.weight == kFp8Static128BlockSym assert spec.activation == kFp8Dynamic128Sym diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 54d3127af0..097f03bb00 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -22,7 +22,7 @@ from packaging import version from tests.quantization.utils import load_model_without_vllm_runner from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops from vllm.config import VllmConfig, set_current_vllm_config -from vllm.config.cache import CacheConfig +from vllm.config.cache import CacheConfig, CacheDType from vllm.forward_context import set_forward_context from vllm.model_executor import parameter from vllm.model_executor.kernels.linear.scaled_mm.aiter import ( @@ -1081,6 +1081,7 @@ def test_quant_method_dispatch_mxfp8_moe_raises(default_vllm_config): ) wk, ak, mcls = config.get_quant_method_target("experts", RoutedExperts) assert mcls is QuarkOCP_MX_MoEMethod + assert wk is not None # The OCP MX MoE constructor should fail loudly for MXFP8. fake_moe_config = MagicMock() with pytest.raises(ValueError, match="MXFP8 experts are not supported"): @@ -1321,7 +1322,6 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch): scheme = QuarkW8A8Fp8.__new__(QuarkW8A8Fp8) scheme.weight_qscheme = "per_channel" scheme.is_static_input_scheme = False - scheme.input_qscheme = "per_channel" scheme.activation_quant_key = kFp8DynamicTokenSym scheme.weight_quant_key = kFp8StaticChannelSym scheme.out_dtype = dtype @@ -1343,7 +1343,7 @@ def test_quark_fp8_ptpc_exposes_kernel_input_quant_key(monkeypatch): @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) def test_quark_fp8_w_per_tensor_a_per_tensor( - kv_cache_dtype: str, monkeypatch, dist_init, workspace_init + kv_cache_dtype: CacheDType, monkeypatch, dist_init, workspace_init ): model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test" checkpoint_scales = {} @@ -2314,7 +2314,7 @@ class TestQuarkInt4Format: None, quant_config.quant_config["global_quant_config"]["weight"], quant_config.pack_method, - moe_config, + moe_config, # type: ignore[arg-type] ) layer = _FakeLayer(moe_config) @@ -2325,7 +2325,10 @@ class TestQuarkInt4Format: torch.zeros(1, 16, 8 // tp_size, dtype=torch.uint8), requires_grad=False, ) - loader = method.get_weight_loader(layer, weight_loader=None) + loader = method.get_weight_loader( + layer, # type: ignore[arg-type] + weight_loader=None, + ) loader( param, raw_zp.clone(), diff --git a/tests/quantization/test_register_quantization_config.py b/tests/quantization/test_register_quantization_config.py index 227c604ccf..69b8db1d1a 100644 --- a/tests/quantization/test_register_quantization_config.py +++ b/tests/quantization/test_register_quantization_config.py @@ -75,7 +75,7 @@ class CustomQuantConfig(QuantizationConfig): def get_name(self) -> QuantizationMethods: """Name of the quantization method.""" - return "custom_quant" + return "custom_quant" # type: ignore[return-value] # Runtime registration extends built-in names. def get_supported_act_dtypes(self) -> list[torch.dtype]: """List of supported activation dtypes.""" diff --git a/tests/quantization/test_torchao.py b/tests/quantization/test_torchao.py index 3c839a687e..04514cfe25 100644 --- a/tests/quantization/test_torchao.py +++ b/tests/quantization/test_torchao.py @@ -12,6 +12,7 @@ from vllm.config import set_current_vllm_config from vllm.forward_context import set_forward_context from vllm.model_executor.layers.attention import Attention from vllm.model_executor.model_loader import get_model_loader +from vllm.model_executor.model_loader.default_loader import DefaultModelLoader from vllm.platforms import current_platform if current_platform.is_rocm(): @@ -126,6 +127,7 @@ def test_online_quant_config_dict_json(vllm_runner, enable_pickle): def load_weights(model): model_loader = get_model_loader(load_config) + assert isinstance(model_loader, DefaultModelLoader) weights_iterator = model_loader.get_all_weights(model_config, model) model.load_weights(weights_iterator) diff --git a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py index 9effb4f7fc..48d7f13aef 100644 --- a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py +++ b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py @@ -34,7 +34,7 @@ def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch): outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass( backend=NvFp4MoeBackend.FLASHINFER_CUTLASS, - layer=layer, + layer=layer, # type: ignore[arg-type] w13=w13, w13_scale=w13_scale, w13_scale_2=weight_scale, diff --git a/tests/quantization/test_turboquant.py b/tests/quantization/test_turboquant.py index 9f6beb3eb8..e60f763c2c 100644 --- a/tests/quantization/test_turboquant.py +++ b/tests/quantization/test_turboquant.py @@ -300,7 +300,7 @@ class TestTurboQuantKVCacheSpec: # The layer builds an unpacked spec; the worker's spec-collection # loop applies TQ slot packing via the backend's customize_spec hook. - spec = Attention.get_kv_cache_spec(layer, vllm_config) + spec = Attention.get_kv_cache_spec(layer, vllm_config) # type: ignore[arg-type] assert isinstance(spec, FullAttentionSpec) assert spec.kv_quant_mode.is_turboquant assert spec.state_content_bytes is None diff --git a/tests/quantization/utils.py b/tests/quantization/utils.py index 4b1b0343a8..ed9776ad1c 100644 --- a/tests/quantization/utils.py +++ b/tests/quantization/utils.py @@ -6,18 +6,31 @@ from typing import Any import regex as re import torch +from pydantic import TypeAdapter from vllm.config import ( CompilationConfig, + CompilationMode, ModelConfig, VllmConfig, set_current_vllm_config, ) +from vllm.config.quantization import QuantizationConfigArgs, QuantSpec from vllm.model_executor.layers.quantization import get_quantization_config from vllm.model_executor.model_loader.default_loader import DefaultModelLoader from vllm.platforms import current_platform +def quant_spec(**kwargs: Any) -> QuantSpec: + """Build a `QuantSpec` from user-facing names such as `"mxfp8"`.""" + return TypeAdapter(QuantSpec).validate_python(kwargs) + + +def quant_config_args(**kwargs: Any) -> QuantizationConfigArgs: + """Build `QuantizationConfigArgs` from user-facing shorthand values.""" + return TypeAdapter(QuantizationConfigArgs).validate_python(kwargs) + + def _limit_num_hidden_layers( model: torch.nn.Module, num_hidden_layers: int | None ) -> None: @@ -59,7 +72,9 @@ def load_model_without_vllm_runner( **(model_config_kwargs or {}), ) vllm_config_args = dict(vllm_config_kwargs or {}) - vllm_config_args.setdefault("compilation_config", CompilationConfig(mode=0)) + vllm_config_args.setdefault( + "compilation_config", CompilationConfig(mode=CompilationMode.NONE) + ) vllm_config = VllmConfig(model_config=model_config, **vllm_config_args) hf_overrides = (model_config_kwargs or {}).get("hf_overrides") or {} num_hidden_layers = hf_overrides.get("num_hidden_layers") diff --git a/tests/reasoning/test_base_thinking_reasoning_parser.py b/tests/reasoning/test_base_thinking_reasoning_parser.py index fb50686062..14792b210b 100644 --- a/tests/reasoning/test_base_thinking_reasoning_parser.py +++ b/tests/reasoning/test_base_thinking_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest @@ -39,8 +39,10 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" @pytest.fixture(scope="module") -def test_tokenizer(): - tokenizer = AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) +def test_tokenizer() -> PreTrainedTokenizerBase: + tokenizer: PreTrainedTokenizerBase = AutoTokenizer.from_pretrained( + REASONING_MODEL_NAME + ) # Add custom test tokens test_tokens = ["", "", "", ""] existing_tokens = set(tokenizer.get_vocab().keys()) @@ -66,7 +68,7 @@ class TestBaseThinkingReasoningParserInit: def test_initialization_with_missing_tokenizer(self): """Test that initialization fails without tokenizer.""" with pytest.raises(ValueError, match="model tokenizer must be passed"): - TestThinkingReasoningParser(None) + TestThinkingReasoningParser(None) # type: ignore[arg-type] def test_initialization_with_missing_tokens(self, test_tokenizer): """Test that initialization fails when tokens are not in vocabulary.""" diff --git a/tests/reasoning/test_deepseekr1_reasoning_parser.py b/tests/reasoning/test_deepseekr1_reasoning_parser.py index 91f0c93653..d4b810da64 100644 --- a/tests/reasoning/test_deepseekr1_reasoning_parser.py +++ b/tests/reasoning/test_deepseekr1_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.reasoning import ReasoningParser, ReasoningParserManager @@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" @pytest.fixture(scope="module") -def deepseek_r1_qwen_tokenizer(): +def deepseek_r1_qwen_tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) @@ -279,10 +279,10 @@ def test_reasoning( # Test extract_content if param_dict["content"] is not None: - content = parser.extract_content_ids(output_ids) - assert content == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids( + content_ids = parser.extract_content_ids(output_ids) + assert content_ids == deepseek_r1_qwen_tokenizer.convert_tokens_to_ids( deepseek_r1_qwen_tokenizer.tokenize(param_dict["content"]) ) else: - content = parser.extract_content_ids(output) - assert content == [] + content_ids = parser.extract_content_ids(output) + assert content_ids == [] diff --git a/tests/reasoning/test_deepseekv3_reasoning_parser.py b/tests/reasoning/test_deepseekv3_reasoning_parser.py index d6bbea2f91..5886c8d277 100644 --- a/tests/reasoning/test_deepseekv3_reasoning_parser.py +++ b/tests/reasoning/test_deepseekv3_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from vllm.entrypoints.generate.base.protocol import DeltaMessage from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest @@ -17,7 +17,7 @@ REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-V3.1" @pytest.fixture(scope="module") -def tokenizer(): +def tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) diff --git a/tests/reasoning/test_ernie45_reasoning_parser.py b/tests/reasoning/test_ernie45_reasoning_parser.py index dbf5507ae6..8c821667a1 100644 --- a/tests/reasoning/test_ernie45_reasoning_parser.py +++ b/tests/reasoning/test_ernie45_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.reasoning import ReasoningParser, ReasoningParserManager @@ -13,7 +13,7 @@ REASONING_MODEL_NAME = "baidu/ERNIE-4.5-21B-A3B-Thinking" @pytest.fixture(scope="module") -def ernie45_tokenizer(): +def ernie45_tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) diff --git a/tests/reasoning/test_glm4_moe_reasoning_parser.py b/tests/reasoning/test_glm4_moe_reasoning_parser.py index dcfba7e55d..0fcba43654 100644 --- a/tests/reasoning/test_glm4_moe_reasoning_parser.py +++ b/tests/reasoning/test_glm4_moe_reasoning_parser.py @@ -4,9 +4,11 @@ import copy import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction +from vllm.parser.engine.adapters import ParserEngineReasoningAdapter +from vllm.parser.glm47_moe import Glm47MoeParser from vllm.reasoning import ReasoningParser, ReasoningParserManager parser_name = "glm45" @@ -17,7 +19,7 @@ REASONING_MODEL_NAME = "zai-org/GLM-4.7" @pytest.fixture(scope="module") -def glm45_tokenizer(): +def glm45_tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) @@ -261,13 +263,20 @@ def glm53_style_tokenizer(glm45_tokenizer): return tokenizer +def _glm_engine(parser: ReasoningParser) -> Glm47MoeParser: + assert isinstance(parser, ParserEngineReasoningAdapter) + engine = parser._parser_engine + assert isinstance(engine, Glm47MoeParser) + return engine + + @pytest.mark.parametrize( "disable_kwargs", [{"enable_thinking": False}, {"thinking": False}] ) def test_glm53_template_forces_reasoning(disable_kwargs: dict, glm53_style_tokenizer): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(glm53_style_tokenizer, chat_template_kwargs=disable_kwargs) - assert parser._parser_engine.thinking_enabled + assert _glm_engine(parser).thinking_enabled output = glm53_style_tokenizer.tokenize(GLM53_LEAK["output"]) output_tokens: list[str] = [ @@ -298,7 +307,7 @@ def test_glm47_template_honors_thinking_disable(glm45_tokenizer): parser = ReasoningParserManager.get_reasoning_parser(parser_name)( glm45_tokenizer, chat_template_kwargs={"enable_thinking": False} ) - assert not parser._parser_engine.thinking_enabled + assert not _glm_engine(parser).thinking_enabled output = glm45_tokenizer.tokenize(GLM53_LEAK["output"]) output_tokens: list[str] = [ diff --git a/tests/reasoning/test_granite_thinking_reasoning_parser.py b/tests/reasoning/test_granite_thinking_reasoning_parser.py index 39ce3bf436..0d347a3a30 100644 --- a/tests/reasoning/test_granite_thinking_reasoning_parser.py +++ b/tests/reasoning/test_granite_thinking_reasoning_parser.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TypedDict +from typing import TypedDict, cast import pytest import regex as re +from transformers import PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -52,8 +53,8 @@ class GraniteThinkingTokenizer: @pytest.fixture -def tokenizer(): - return GraniteThinkingTokenizer() +def tokenizer() -> PreTrainedTokenizerBase: + return cast(PreTrainedTokenizerBase, GraniteThinkingTokenizer()) # ── Basic reasoning extraction (non-streaming + streaming) ─────────── @@ -173,7 +174,7 @@ def tokenizer(): ], ) def test_granite_thinking_reasoning( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, streaming: bool, param_dict: ReasoningCase, ): @@ -195,7 +196,7 @@ def test_granite_thinking_reasoning( def test_granite_thinking_no_content_after_end_token( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -215,7 +216,7 @@ def test_granite_thinking_no_content_after_end_token( @pytest.mark.parametrize("streaming", [False, True]) def test_granite_thinking_whitespace_only_content( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, streaming: bool, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) @@ -235,7 +236,7 @@ def test_granite_thinking_whitespace_only_content( def test_granite_thinking_unterminated_think_block( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -260,7 +261,7 @@ def test_granite_thinking_unterminated_think_block( def test_granite_thinking_disabled_moves_into_content( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -285,7 +286,7 @@ def test_granite_thinking_disabled_moves_into_content( def test_granite_thinking_disabled_with_leading_newline( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # With enable_thinking=False, model output goes through the swap # path: content is initially None (all text classified as @@ -316,7 +317,7 @@ def test_granite_thinking_disabled_with_leading_newline( def test_granite_thinking_force_nonempty_content_moves_into_content( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -338,7 +339,7 @@ def test_granite_thinking_force_nonempty_content_moves_into_content( def test_granite_thinking_force_nonempty_no_swap_when_newlines_only( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # When IS present and content is newlines-only, lstrip # removes them but the swap should NOT fire — content was present, @@ -363,7 +364,7 @@ def test_granite_thinking_force_nonempty_no_swap_when_newlines_only( def test_granite_thinking_force_nonempty_swaps_when_content_absent( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # When IS present but content is truly absent (zero # characters after , e.g. max_tokens cut), swap fires. @@ -387,7 +388,7 @@ def test_granite_thinking_force_nonempty_swaps_when_content_absent( def test_granite_thinking_force_nonempty_keeps_real_content( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -412,7 +413,7 @@ def test_granite_thinking_force_nonempty_keeps_real_content( def test_granite_thinking_keeps_truncated_reasoning( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -472,7 +473,7 @@ def _run_parse_delta(parser, tokenizer, text, request): def test_granite_thinking_streaming_enable_thinking_false( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # With enable_thinking=False, the parser (constructed without # kwargs) starts in REASONING state. All text streams as reasoning @@ -494,7 +495,7 @@ def test_granite_thinking_streaming_enable_thinking_false( def test_granite_thinking_streaming_strips_leading_newline( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): request = ChatCompletionRequest( model="test-model", @@ -511,7 +512,7 @@ def test_granite_thinking_streaming_strips_leading_newline( def test_granite_thinking_streaming_promotes_reasoning_to_content( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): request = ChatCompletionRequest( model="test-model", @@ -527,7 +528,7 @@ def test_granite_thinking_streaming_promotes_reasoning_to_content( def test_granite_thinking_streaming_no_promotion_with_real_content( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): request = ChatCompletionRequest( model="test-model", @@ -548,7 +549,7 @@ def test_granite_thinking_streaming_no_promotion_with_real_content( def test_granite_thinking_streaming_no_promotion_without_opt_in( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, ): request = ChatCompletionRequest(model="test-model", messages=[]) parser = _make_reasoning_parser(tokenizer) @@ -564,7 +565,7 @@ def test_granite_thinking_streaming_no_promotion_without_opt_in( @pytest.mark.parametrize("streaming", [False, True]) def test_granite_thinking_empty_think_block( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, streaming: bool, ): # \nHello — empty reasoning, content after newline. @@ -592,7 +593,7 @@ def test_granite_thinking_empty_think_block( ], ) def test_granite_thinking_streaming_chunking_independent( - tokenizer: GraniteThinkingTokenizer, + tokenizer: PreTrainedTokenizerBase, text: str, expected_reasoning: str, expected_content: str, diff --git a/tests/reasoning/test_holo2_reasoning_parser.py b/tests/reasoning/test_holo2_reasoning_parser.py index aee2774c5e..2e22e2c694 100644 --- a/tests/reasoning/test_holo2_reasoning_parser.py +++ b/tests/reasoning/test_holo2_reasoning_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.reasoning import ReasoningParser, ReasoningParserManager @@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "HCompany/Holo2-4B" @pytest.fixture(scope="module") -def tokenizer(): +def tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) diff --git a/tests/reasoning/test_k2_horizon_reasoning_parser.py b/tests/reasoning/test_k2_horizon_reasoning_parser.py index 1a554770f1..01a7968a6d 100644 --- a/tests/reasoning/test_k2_horizon_reasoning_parser.py +++ b/tests/reasoning/test_k2_horizon_reasoning_parser.py @@ -3,6 +3,7 @@ import string from collections.abc import Sequence +from typing import cast import pytest @@ -10,6 +11,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionReque from vllm.parser.parser_manager import ParserManager from vllm.reasoning import ReasoningParserManager from vllm.reasoning.k2_horizon_reasoning_parser import K2HorizonReasoningParser +from vllm.tokenizers import TokenizerLike pytestmark = pytest.mark.skip_global_cleanup @@ -59,6 +61,10 @@ class K2Tokenizer: return "".join(inverse_vocab[token_id] for token_id in ids) +def _k2_tokenizer() -> TokenizerLike: + return cast(TokenizerLike, K2Tokenizer()) + + def _request(**kwargs) -> ChatCompletionRequest: return ChatCompletionRequest( model="k2-horizon", @@ -70,7 +76,7 @@ def _request(**kwargs) -> ChatCompletionRequest: @pytest.mark.parametrize("effort", ["high", "medium", "low"]) def test_reasoning_effort_selects_tokens(effort: str): parser = K2HorizonReasoningParser( - K2Tokenizer(), + _k2_tokenizer(), chat_template_kwargs={"reasoning_effort": effort}, ) @@ -78,7 +84,7 @@ def test_reasoning_effort_selects_tokens(effort: str): def test_reasoning_effort_precedence_and_default(): - tokenizer = K2Tokenizer() + tokenizer = _k2_tokenizer() request = _request( reasoning_effort="medium", chat_template_kwargs={"reasoning_effort": "low"}, @@ -103,13 +109,13 @@ def test_reasoning_effort_precedence_and_default(): def test_invalid_reasoning_effort_raises(effort: str | None): with pytest.raises(ValueError, match="Unsupported reasoning_effort"): K2HorizonReasoningParser( - K2Tokenizer(), + _k2_tokenizer(), chat_template_kwargs={"reasoning_effort": effort}, ) def test_non_streaming_explicit_and_implicit_boundaries(): - parser = K2HorizonReasoningParser(K2Tokenizer()) + parser = K2HorizonReasoningParser(_k2_tokenizer()) request = _request() assert parser.extract_reasoning("plananswer", request) == ( @@ -126,7 +132,7 @@ def test_non_streaming_explicit_and_implicit_boundaries(): @pytest.mark.parametrize("reasoning", ["plan", " ", "\n", " \n"]) def test_character_split_reasoning_stream(reasoning: str): - parser = K2HorizonReasoningParser(K2Tokenizer()) + parser = K2HorizonReasoningParser(_k2_tokenizer()) reasoning_parts: list[str] = [] content_parts: list[str] = [] @@ -150,7 +156,7 @@ def test_character_split_reasoning_stream(reasoning: str): def test_latest_boundary_controls_reasoning_state(): - tokenizer = K2Tokenizer() + tokenizer = _k2_tokenizer() parser = K2HorizonReasoningParser(tokenizer) vocab = tokenizer.get_vocab() @@ -159,7 +165,7 @@ def test_latest_boundary_controls_reasoning_state(): def test_composed_streaming_reasoning_to_tool_handoff(): - tokenizer = K2Tokenizer() + tokenizer = _k2_tokenizer() tools = [ { "type": "function", diff --git a/tests/reasoning/test_kimi_k2_reasoning_parser.py b/tests/reasoning/test_kimi_k2_reasoning_parser.py index e9690b2faa..b350eb10f1 100644 --- a/tests/reasoning/test_kimi_k2_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k2_reasoning_parser.py @@ -6,6 +6,7 @@ import pytest from vllm.entrypoints.generate.base.protocol import DeltaMessage from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.parser.kimi_k2 import KimiK2Parser from vllm.reasoning.kimi_k2_reasoning_parser import KimiK2ReasoningParser from vllm.tokenizers import get_tokenizer @@ -65,9 +66,13 @@ def test_extract_reasoning_tool_section_ends_reasoning(kimi_k2_tokenizer): def test_streaming_reasoning_then_content(kimi_k2_tokenizer): """Token-by-token streaming: reasoning tokens then content after .""" parser = KimiK2ReasoningParser(kimi_k2_tokenizer) + parser_engine = parser._parser_engine + assert isinstance(parser_engine, KimiK2Parser) - think_id = parser._parser_engine._start_token_id - end_think_id = parser._parser_engine._end_token_id + think_id = parser_engine._start_token_id + end_think_id = parser_engine._end_token_id + assert think_id is not None + assert end_think_id is not None # Use a real token ID from the tokenizer for regular content regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0] @@ -123,9 +128,13 @@ def test_streaming_reasoning_then_content(kimi_k2_tokenizer): def test_streaming_tool_section_ends_reasoning(kimi_k2_tokenizer): """<|tool_calls_section_begin|> in delta ends reasoning during streaming.""" parser = KimiK2ReasoningParser(kimi_k2_tokenizer) + parser_engine = parser._parser_engine + assert isinstance(parser_engine, KimiK2Parser) - think_id = parser._parser_engine._start_token_id - tool_begin_id = parser._parser_engine._tool_section_start_token_id + think_id = parser_engine._start_token_id + tool_begin_id = parser_engine._tool_section_start_token_id + assert think_id is not None + assert tool_begin_id is not None regular_id = kimi_k2_tokenizer.encode("hello", add_special_tokens=False)[0] # Tool section token arrives — should transition from reasoning to content diff --git a/tests/reasoning/test_kimi_k3_reasoning_parser.py b/tests/reasoning/test_kimi_k3_reasoning_parser.py index 45c5511128..8dcac4a83a 100644 --- a/tests/reasoning/test_kimi_k3_reasoning_parser.py +++ b/tests/reasoning/test_kimi_k3_reasoning_parser.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import cast + import pytest from vllm.entrypoints.generate.base.protocol import DeltaMessage @@ -10,6 +12,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.parser.kimi_k3 import KimiK3Parser from vllm.parser.parser_manager import ParserManager from vllm.reasoning.kimi_k3_reasoning_parser import KimiK3ReasoningParser +from vllm.tokenizers import TokenizerLike pytestmark = pytest.mark.skip_global_cleanup @@ -25,10 +28,13 @@ RESPONSE_OPEN_IDS = [ord(ch) for ch in RESPONSE_OPEN] class DummyTokenizer: + def __len__(self) -> int: + return 1 + def get_vocab(self) -> dict[str, int]: return {} - def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: + def encode(self, text: str, *args, **kwargs) -> list[int]: if text == THINK_OPEN: return [1, 2, 3] if text == THINK_CLOSE: @@ -36,6 +42,10 @@ class DummyTokenizer: return [ord(ch) for ch in text] +def _dummy_tokenizer() -> TokenizerLike: + return cast(TokenizerLike, DummyTokenizer()) + + class ReasoningOnlyParser(KimiK3Parser): reasoning_parser_cls = KimiK3ReasoningParser @@ -84,7 +94,7 @@ def test_extract_reasoning_with_generation_prefix_consumed(): def test_delegating_parser_strips_response_wrapper_without_tool_parser(): - parser = ReasoningOnlyParser(DummyTokenizer()) + parser = ReasoningOnlyParser(_dummy_tokenizer()) request = ChatCompletionRequest(model="test-model", messages=[]) reasoning, content, tool_calls = parser.parse( @@ -165,7 +175,7 @@ def test_count_reasoning_tokens_is_zero_when_thinking_disabled(): def test_count_reasoning_tokens_through_delegating_parser(): - parser = ReasoningOnlyParser(DummyTokenizer()) + parser = ReasoningOnlyParser(_dummy_tokenizer()) assert parser.count_reasoning_tokens([*OPEN_IDS, 9, *CLOSE_IDS, 11]) == 1 @@ -253,7 +263,8 @@ def test_thinking_disabled_streams_content(): def test_delegating_parser_thinking_false_streams_response_content(): parser = ReasoningOnlyParser( - DummyTokenizer(), chat_template_kwargs={"thinking": False} + _dummy_tokenizer(), + chat_template_kwargs={"thinking": False}, ) request = ChatCompletionRequest( model="test-model", diff --git a/tests/reasoning/test_minimax_m2_append_reasoning_parser.py b/tests/reasoning/test_minimax_m2_append_reasoning_parser.py index eefe5e3eff..7a74cb10a9 100644 --- a/tests/reasoning/test_minimax_m2_append_reasoning_parser.py +++ b/tests/reasoning/test_minimax_m2_append_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.reasoning import ReasoningParser, ReasoningParserManager @@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "MiniMaxAI/MiniMax-M2" @pytest.fixture(scope="module") -def minimax_m2_tokenizer(): +def minimax_m2_tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) diff --git a/tests/reasoning/test_minimax_m2_reasoning_parser.py b/tests/reasoning/test_minimax_m2_reasoning_parser.py index 6f8001dcae..7dfb84289d 100644 --- a/tests/reasoning/test_minimax_m2_reasoning_parser.py +++ b/tests/reasoning/test_minimax_m2_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.reasoning import ReasoningParser, ReasoningParserManager @@ -15,7 +15,7 @@ REASONING_MODEL_NAME = "MiniMaxAI/MiniMax-M2" @pytest.fixture(scope="module") -def minimax_m2_tokenizer(): +def minimax_m2_tokenizer() -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) @@ -195,10 +195,10 @@ def test_reasoning( # Test extract_content if param_dict["content"] is not None: - content = parser.extract_content_ids(output_ids) - assert content == minimax_m2_tokenizer.convert_tokens_to_ids( + content_ids = parser.extract_content_ids(output_ids) + assert content_ids == minimax_m2_tokenizer.convert_tokens_to_ids( minimax_m2_tokenizer.tokenize(param_dict["content"]) ) else: - content = parser.extract_content_ids(output) - assert content == [] + content_ids = parser.extract_content_ids(output) + assert content_ids == [] diff --git a/tests/reasoning/test_nemotron_v3_reasoning_parser.py b/tests/reasoning/test_nemotron_v3_reasoning_parser.py index 325df23662..8fb24a41af 100644 --- a/tests/reasoning/test_nemotron_v3_reasoning_parser.py +++ b/tests/reasoning/test_nemotron_v3_reasoning_parser.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TypedDict +from typing import TypedDict, cast import pytest import regex as re +from transformers import PreTrainedTokenizerBase from tests.reasoning.utils import run_reasoning_extraction from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest @@ -48,8 +49,8 @@ class FakeNemotronTokenizer: @pytest.fixture -def tokenizer(): - return FakeNemotronTokenizer() +def tokenizer() -> PreTrainedTokenizerBase: + return cast(PreTrainedTokenizerBase, FakeNemotronTokenizer()) @pytest.mark.parametrize( @@ -94,7 +95,7 @@ def tokenizer(): ], ) def test_nemotron_v3_reasoning( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, streaming: bool, param_dict: ReasoningCase, ): @@ -113,7 +114,7 @@ def test_nemotron_v3_reasoning( def test_nemotron_v3_without_thinking_moves_into_content( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -137,7 +138,7 @@ def test_nemotron_v3_without_thinking_moves_into_content( def test_nemotron_v3_force_nonempty_content_moves_into_content( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -159,7 +160,7 @@ def test_nemotron_v3_force_nonempty_content_moves_into_content( def test_nemotron_v3_force_nonempty_keeps_real_content( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # When real content follows the closing tag nothing is promoted: the # content after is returned as-is and reasoning stays separate. @@ -183,7 +184,7 @@ def test_nemotron_v3_force_nonempty_keeps_real_content( def test_nemotron_v3_with_thinking_keeps_truncated_reasoning( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) parser = parser_cls(tokenizer) @@ -242,7 +243,7 @@ def _run_parse_delta(parser, tokenizer, text, request): def test_nemotron_v3_streaming_promotes_reasoning_to_content( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # Model never closes : reasoning streams normally AND is duplicated # into content on the terminal delta. @@ -260,7 +261,7 @@ def test_nemotron_v3_streaming_promotes_reasoning_to_content( def test_nemotron_v3_streaming_no_promotion_with_real_content( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): request = ChatCompletionRequest( model="test-model", @@ -279,7 +280,7 @@ def test_nemotron_v3_streaming_no_promotion_with_real_content( def test_nemotron_v3_streaming_no_promotion_without_opt_in( - tokenizer: FakeNemotronTokenizer, + tokenizer: PreTrainedTokenizerBase, ): # Without enable_thinking=False / force_nonempty_content the fallback must # stay disabled: the response stays reasoning-only, content empty. diff --git a/tests/reasoning/test_qwen3_reasoning_parser.py b/tests/reasoning/test_qwen3_reasoning_parser.py index f42458560f..412eb2db4d 100644 --- a/tests/reasoning/test_qwen3_reasoning_parser.py +++ b/tests/reasoning/test_qwen3_reasoning_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from transformers import AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from tests.reasoning.utils import ( StreamingReasoningReconstructor, @@ -24,7 +24,7 @@ REASONING_MODEL_NAMES = [ @pytest.fixture(scope="module", params=REASONING_MODEL_NAMES) -def qwen3_tokenizer(request): +def qwen3_tokenizer(request) -> PreTrainedTokenizerBase: return AutoTokenizer.from_pretrained(request.param) diff --git a/tests/reasoning/test_step3p5_reasoning_parser.py b/tests/reasoning/test_step3p5_reasoning_parser.py index aeaf940213..a65ac080f2 100644 --- a/tests/reasoning/test_step3p5_reasoning_parser.py +++ b/tests/reasoning/test_step3p5_reasoning_parser.py @@ -4,7 +4,8 @@ import pytest from tests.reasoning.utils import run_reasoning_extraction -from vllm.reasoning import ReasoningParser, ReasoningParserManager +from vllm.reasoning import ReasoningParserManager +from vllm.reasoning.step3p5_reasoning_parser import Step3p5ReasoningParser from vllm.tokenizers import get_tokenizer parser_name = "step3p5" @@ -283,9 +284,8 @@ def test_reasoning( output_tokens: list[str] = [ step3p5_tokenizer.convert_tokens_to_string([token]) for token in output ] - parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( - step3p5_tokenizer - ) + parser = ReasoningParserManager.get_reasoning_parser(parser_name)(step3p5_tokenizer) + assert isinstance(parser, Step3p5ReasoningParser) reasoning, content = run_reasoning_extraction( parser, output_tokens, streaming=streaming @@ -317,7 +317,7 @@ def test_reasoning( # Test extract_content if param_dict["content"] is not None: - content = parser.extract_content_ids(output_ids) + content_ids = parser.extract_content_ids(output_ids) # Fixed expected token ids for specific test cases test_id = ( request.node.callspec.id if hasattr(request.node, "callspec") else None @@ -332,7 +332,7 @@ def test_reasoning( expected_content_ids = step3p5_tokenizer.convert_tokens_to_ids( step3p5_tokenizer.tokenize(param_dict["content"]) ) - assert content == expected_content_ids + assert content_ids == expected_content_ids else: - content = parser.extract_content_ids(output_ids) - assert content == [] + content_ids = parser.extract_content_ids(output_ids) + assert content_ids == [] diff --git a/tests/reasoning/utils.py b/tests/reasoning/utils.py index 6e529c36c8..aeefc86532 100644 --- a/tests/reasoning/utils.py +++ b/tests/reasoning/utils.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import cast + +from transformers import PreTrainedTokenizerBase + from vllm.entrypoints.generate.base.protocol import DeltaMessage from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.reasoning import ReasoningParser @@ -102,10 +106,11 @@ def run_reasoning_extraction_streaming( reconstructor = StreamingReasoningReconstructor() previous_text = "" previous_tokens: list[int] = [] + tokenizer = cast(PreTrainedTokenizerBase, reasoning_parser.model_tokenizer) for delta in model_deltas: token_delta = [ - reasoning_parser.vocab.get(token) - for token in reasoning_parser.model_tokenizer.tokenize(delta) + reasoning_parser.vocab[token] + for token in tokenizer.tokenize(delta) if token in reasoning_parser.vocab ] current_text = previous_text + delta diff --git a/tests/v1/logits_processors/test_correctness.py b/tests/v1/logits_processors/test_correctness.py index 9b12e7412e..ce5d8a14d8 100644 --- a/tests/v1/logits_processors/test_correctness.py +++ b/tests/v1/logits_processors/test_correctness.py @@ -3,7 +3,7 @@ import random from collections.abc import Callable -from typing import NamedTuple, TypeAlias +from typing import NamedTuple, NoReturn, TypeAlias import numpy as np import pytest @@ -13,6 +13,7 @@ from tests.utils import create_new_process_for_each_test from tests.v1.sample.utils import ( LogitsprocsTestFakes, create_fake_logits, + create_mock_reasoning_config, create_penalty_tensor, create_prompt_tokens_tensor, fake_apply_logitsprocs, @@ -103,14 +104,6 @@ class LogitsProcsRequestParams: return f"MyClass({summ})" -class MockReasoningConfig: - """Minimal reasoning config for ``ThinkingBudgetStateHolder`` tests.""" - - reasoning_start_token_ids = [THINK_START_TOKEN_ID] - reasoning_end_token_ids = [THINK_END_TOKEN_ID] - enabled = True - - def _generate_fake_sampling_metadata( num_output_tokens: int, batch_size: int, @@ -131,7 +124,9 @@ def _generate_fake_sampling_metadata( ) vllm_config = VllmConfig() - vllm_config.reasoning_config = MockReasoningConfig() + vllm_config.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) logitsprocs = build_logitsprocs( vllm_config=vllm_config, @@ -242,7 +237,7 @@ def _raise_error_invalid( request_params: LogitsProcsRequestParams, step_idx: int, err_cls: type[Exception] = ValueError, -) -> None: +) -> NoReturn: raise err_cls( f"Validation failed for step={step_idx}, " f"batch_index={batch_index}, " @@ -268,6 +263,7 @@ def _logit_bias_validate( ) -> None: """Validate logit bias logitproc applied correctly.""" logit_bias = request_params.params.logit_bias + assert logit_bias is not None logits_old = test_fakes.logits[persistent_batch[batch_index].workload_index].cpu() logits_new = logits_new[batch_index].cpu() for token_id in range(VOCAB_SIZE): @@ -368,9 +364,7 @@ def _min_tokens_validate( ref_num_out_tokens = len(request_params.out_tokens) min_reached = ref_num_out_tokens >= MIN_TOKENS_LEN_THRESHOLD ref_all_stop_token_ids = request_params.params.all_stop_token_ids - mt_lp: MinTokensLogitsProcessor = next( - test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor) - ) + mt_lp = next(test_fakes.get_logitsprocs_by_cls(MinTokensLogitsProcessor)) assert isinstance(mt_lp, MinTokensLogitsProcessor) min_tok = mt_lp.min_toks.get(batch_index, None) @@ -689,7 +683,7 @@ class LogitsprocTestHelpers(NamedTuple): gen_request_fxn: Callable | None = None -logitsprocs_test_mapping = { +logitsprocs_test_mapping: dict[LogitprocType, LogitsprocTestHelpers] = { STR_NO_LOGITPROC: LogitsprocTestHelpers(eval_fxn=_none_validate), LogitBiasLogitsProcessor: LogitsprocTestHelpers( gen_request_fxn=_logit_bias_params, eval_fxn=_logit_bias_validate @@ -706,22 +700,22 @@ logitsprocs_test_mapping = { } -def _get_test_cases() -> list[list[str]]: +def _get_test_cases() -> list[list[LogitprocType]]: """Each test case is a set of logitsprocs.""" - logitsprocs_types = list(logitsprocs_test_mapping.keys()) + logitsprocs_types: list[LogitprocType] = list(logitsprocs_test_mapping.keys()) # Isolate thinking-budget handling from other processors to avoid cross-talk. thinking_id: LogitprocType = STR_THINKING_BUDGET - other_processors = [ + other_processors: list[LogitprocType] = [ p for p in logitsprocs_types if p != STR_NO_LOGITPROC and p != thinking_id ] - return ( - [[STR_NO_LOGITPROC]] - + [[logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors] - + [other_processors] - + [[thinking_id]] + test_cases: list[list[LogitprocType]] = [[STR_NO_LOGITPROC]] + test_cases.extend( + [logitproc_type, STR_NO_LOGITPROC] for logitproc_type in other_processors ) + test_cases.extend([other_processors, [thinking_id]]) + return test_cases def _generate_fake_step_update( @@ -774,6 +768,7 @@ def _generate_fake_step_update( for add_req_params in workload_params[wdx : (wdx + num_step_add_replace)]: # Replace as many removed requests as possible with added requests add_remove_idx = batch_update_builder.pop_removed() + assert add_remove_idx is not None batch_update_builder.added.append( ( add_remove_idx, @@ -976,13 +971,6 @@ def test_logitsprocs( step_idx += 1 -class MockReasoningNoEndTokens: - """Reasoning config with no end token ids (disables enforcement in holder).""" - - reasoning_start_token_ids = [THINK_START_TOKEN_ID] - reasoning_end_token_ids: list[int] = [] - - def test_maybe_create_thinking_budget_holder_without_reasoning(): cfg = VllmConfig() assert cfg.reasoning_config is None @@ -1000,7 +988,9 @@ def test_maybe_create_thinking_budget_holder_without_reasoning(): def test_thinking_budget_holder_has_tracked_after_sync_add(): vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1030,7 +1020,9 @@ def test_thinking_budget_holder_has_tracked_after_sync_add(): def test_thinking_budget_holder_sync_remove_clears_state(): vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1060,7 +1052,9 @@ def test_thinking_budget_holder_sync_remove_clears_state(): def test_thinking_budget_holder_sync_add_without_budget_drops_row(): vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1081,7 +1075,9 @@ def test_thinking_budget_holder_sync_add_without_budget_drops_row(): def test_thinking_budget_holder_swap_exchanges_state(): vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1125,7 +1121,9 @@ def test_thinking_budget_holder_swap_exchanges_state(): def test_thinking_budget_holder_unidirectional_move(): vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1163,7 +1161,9 @@ def test_thinking_budget_holder_unidirectional_move(): def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins(): vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1197,7 +1197,7 @@ def test_thinking_budget_holder_update_state_repeat_indices_last_row_wins(): def test_thinking_budget_holder_spec_mode_tensor_layout(): h = ThinkingBudgetStateHolder( - MockReasoningConfig(), + create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]), 8, 2, torch.device("cpu"), @@ -1209,7 +1209,7 @@ def test_thinking_budget_holder_spec_mode_tensor_layout(): def test_thinking_budget_holder_empty_end_tokens_disables_row(): vc = VllmConfig() - vc.reasoning_config = MockReasoningNoEndTokens() + vc.reasoning_config = create_mock_reasoning_config([THINK_START_TOKEN_ID], []) h = ThinkingBudgetStateHolder( vc.reasoning_config, vc.scheduler_config.max_num_seqs, @@ -1253,7 +1253,9 @@ def test_thinking_budget_enforced_without_penalties(): passing an empty list (the pre-fix behavior) prevents budget enforcement. """ vc = VllmConfig() - vc.reasoning_config = MockReasoningConfig() + vc.reasoning_config = create_mock_reasoning_config( + [THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID] + ) budget = 3 # allow 3 thinking tokens h = ThinkingBudgetStateHolder( @@ -1346,7 +1348,11 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index """Test thinking budget enforced for a long thinking run, then a natural end marker.""" h = ThinkingBudgetStateHolder( - MockReasoningConfig(), 8, 0, torch.device("cpu"), False + create_mock_reasoning_config([THINK_START_TOKEN_ID], [THINK_END_TOKEN_ID]), + 8, + 0, + torch.device("cpu"), + False, ) h.sync_batch( BatchUpdate( @@ -1356,8 +1362,8 @@ def test_thinking_budget_long_thinking_section_end_marker_found_at_correct_index moved=(), ) ) - start = MockReasoningConfig.reasoning_start_token_ids - end = MockReasoningConfig.reasoning_end_token_ids + start = [THINK_START_TOKEN_ID] + end = [THINK_END_TOKEN_ID] out: list[int] = list(start) h.update_state([out], None, None) @@ -1396,13 +1402,9 @@ class TestThinkingBudgetReentry: @staticmethod def _make_holder(end_token_ids: list[int]) -> ThinkingBudgetStateHolder: - class FakeReasoningConfig: - reasoning_start_token_ids = [TestThinkingBudgetReentry.THINK_START] - reasoning_end_token_ids: list[int] = [] - enabled = True - - cfg = FakeReasoningConfig() - cfg.reasoning_end_token_ids = end_token_ids + cfg = create_mock_reasoning_config( + [TestThinkingBudgetReentry.THINK_START], end_token_ids + ) return ThinkingBudgetStateHolder( reasoning_config=cfg, max_num_seqs=8, @@ -1526,21 +1528,12 @@ class TestThinkingBudgetNaturalEndReentry: @staticmethod def _make_holder(end_token_ids): - from dataclasses import dataclass - from vllm.v1.sample.thinking_budget_state import ( ThinkingBudgetStateHolder, ) - @dataclass - class FakeReasoningConfig: - reasoning_start_token_ids: list[int] - reasoning_end_token_ids: list[int] - enabled: bool = True - - cfg = FakeReasoningConfig( - reasoning_start_token_ids=[TestThinkingBudgetNaturalEndReentry.THINK_START], - reasoning_end_token_ids=end_token_ids, + cfg = create_mock_reasoning_config( + [TestThinkingBudgetNaturalEndReentry.THINK_START], end_token_ids ) return ThinkingBudgetStateHolder( reasoning_config=cfg, @@ -1830,21 +1823,11 @@ class TestThinkingBudgetNaturalEndReentry: natural exit, continue_thinking is cleared and Block 2 enforcement works correctly. """ - from dataclasses import dataclass from unittest.mock import MagicMock from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder - @dataclass - class FakeReasoningConfig: - reasoning_start_token_ids: list[int] - reasoning_end_token_ids: list[int] - enabled: bool = True - - cfg = FakeReasoningConfig( - reasoning_start_token_ids=[self.THINK_START], - reasoning_end_token_ids=self.THINK_END_SINGLE, - ) + cfg = create_mock_reasoning_config([self.THINK_START], self.THINK_END_SINGLE) holder = ThinkingBudgetStateHolder( reasoning_config=cfg, max_num_seqs=8, diff --git a/tests/v1/logits_processors/utils.py b/tests/v1/logits_processors/utils.py index 76877b655b..bcf9b74a50 100644 --- a/tests/v1/logits_processors/utils.py +++ b/tests/v1/logits_processors/utils.py @@ -57,15 +57,20 @@ prompts = [ class DummyLogitsProcessor(LogitsProcessor): """Fake logit processor to support unit testing and examples.""" - @classmethod - def validate_params(cls, params: SamplingParams): - target_token: int | None = params.extra_args and params.extra_args.get( - "target_token" + @staticmethod + def _get_target_token(params: SamplingParams) -> int | None: + target_token = ( + params.extra_args.get(DUMMY_LOGITPROC_ARG) if params.extra_args else None ) if target_token is not None and not isinstance(target_token, int): raise VLLMValidationError( f"target_token value {target_token} {type(target_token)} is not int" ) + return target_token + + @classmethod + def validate_params(cls, params: SamplingParams): + cls._get_target_token(params) def __init__( self, vllm_config: "VllmConfig", device: torch.device, is_pin_memory: bool @@ -78,8 +83,7 @@ class DummyLogitsProcessor(LogitsProcessor): def update_state(self, batch_update: BatchUpdate | None): def extract_extra_arg(params: SamplingParams) -> int | None: - self.validate_params(params) - return params.extra_args and params.extra_args.get("target_token") + return self._get_target_token(params) process_dict_updates( self.req_info, diff --git a/tests/v1/sample/test_diffusion_gemma_reads.py b/tests/v1/sample/test_diffusion_gemma_reads.py index 769481b06b..7bd04c0379 100644 --- a/tests/v1/sample/test_diffusion_gemma_reads.py +++ b/tests/v1/sample/test_diffusion_gemma_reads.py @@ -341,6 +341,7 @@ def test_batch_allowed_needs_one_shared_set(): assert states.batch_allowed([]) is None shared = states.batch_allowed([0]) + assert shared is not None assert shared.tolist() == [1, 2, 3] assert shared.dtype == torch.int64 assert states.batch_allowed([0, 0]) is shared diff --git a/tests/v1/sample/test_diffusion_sampler_stats.py b/tests/v1/sample/test_diffusion_sampler_stats.py index 589c28c0a6..dbb34ec3b7 100644 --- a/tests/v1/sample/test_diffusion_sampler_stats.py +++ b/tests/v1/sample/test_diffusion_sampler_stats.py @@ -34,6 +34,8 @@ def test_stats_match_reference(vocab: int, dtype: torch.dtype): ) assert torch.equal(argmax, ref_argmax) torch.testing.assert_close(entropy, ref_entropy, atol=2e-3, rtol=1e-3) + assert probs is not None + assert ref_probs is not None torch.testing.assert_close(probs.float(), ref_probs.float(), atol=5e-3, rtol=2e-2) assert probs.dtype == torch.bfloat16 assert probs.shape == logits.shape @@ -100,6 +102,7 @@ def test_masked_logits_keep_a_finite_entropy(): assert entropy[0].item() < 1e-6 and entropy[1].item() < 1e-6 assert abs(entropy[2].item() - math.log(2.0)) < 1e-4 assert torch.equal(argmax[[0, 1, 3]], torch.tensor([3, 3, 3], device="cuda")) + assert probs is not None assert probs[0, 3].item() == 1.0 and probs[0].sum().item() == 1.0 _, _, ref_entropy, _ = sample_row_stats_reference(logits, temps, 1, None) assert torch.isfinite(ref_entropy).all() diff --git a/tests/v1/sample/test_dry.py b/tests/v1/sample/test_dry.py index c724dc681b..f70d85eba4 100644 --- a/tests/v1/sample/test_dry.py +++ b/tests/v1/sample/test_dry.py @@ -340,7 +340,7 @@ def _make_v2_state( device=device, all_token_ids=SimpleNamespace(gpu=all_tokens), ) - state = DryState(_make_vllm_config(), req_states) + state = DryState(_make_vllm_config(), req_states) # type: ignore[arg-type] if tokenizer is not None: # Re-resolve the default set now that a tokenizer is installed. state._tokenizer = tokenizer @@ -446,7 +446,7 @@ def test_spec_decode_is_refused_at_construction(): ) config = _make_vllm_config(speculative_config=SimpleNamespace()) with pytest.raises(ValueError, match="speculative decoding"): - DryState(config, req_states) + DryState(config, req_states) # type: ignore[arg-type] def test_v2_draft_expanded_logits_raise(): @@ -917,7 +917,10 @@ def test_loads_through_the_loader_by_fqcn(): total_len=None, ) procs = loader.build_custom_logits_processors( - _make_vllm_config(), req_states, False, [DRY_FQCN] + _make_vllm_config(), + req_states, # type: ignore[arg-type] + False, + [DRY_FQCN], ) assert [type(p) for p in procs] == [DryState] diff --git a/tests/v1/sample/test_head_dtype.py b/tests/v1/sample/test_head_dtype.py index ce07919577..1a8681a2a2 100644 --- a/tests/v1/sample/test_head_dtype.py +++ b/tests/v1/sample/test_head_dtype.py @@ -16,6 +16,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, UnquantizedEmbeddingMethod, + VocabParallelEmbedding, ) @@ -32,10 +33,17 @@ class _FakeLmHead: self.tp_size = 1 +def _fake_lm_head( + weight: torch.Tensor, quantized: bool = False, shard_indices: object | None = None +) -> VocabParallelEmbedding: + # The dtype tests only need the projection fields, not distributed setup. + return _FakeLmHead(weight, quantized, shard_indices) # type: ignore[return-value] + + def _build_processor(vocab_size: int) -> LogitsProcessor: lp = LogitsProcessor(vocab_size) # The TP gather is orthogonal to the dtype behavior under test. - lp._gather_logits = lambda logits: logits + lp._gather_logits = lambda logits: logits # type: ignore[method-assign] return lp @@ -47,8 +55,9 @@ def test_fp32_head_runs_projection_in_fp32(default_vllm_config): hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16) weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16) - logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None) + logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None) + assert logits is not None assert logits.dtype == torch.float32 assert torch.isfinite(logits).all() expected = torch.nn.functional.linear(hidden_states.float(), weight.float()) @@ -66,8 +75,9 @@ def test_non_fp32_head_dtype_uses_cast_path(default_vllm_config): hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16) weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16) - logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None) + logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None) + assert logits is not None assert logits.dtype == torch.float16 expected = torch.nn.functional.linear(hidden_states.half(), weight.half()) torch.testing.assert_close(logits, expected) @@ -82,7 +92,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config): hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16) weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16) - lm_head = _FakeLmHead(weight) + lm_head = _fake_lm_head(weight) with mock.patch.object( lm_head.quant_method, @@ -94,6 +104,7 @@ def test_head_dtype_equal_to_model_dtype_uses_quant_method(default_vllm_config): logits = lp._get_logits(hidden_states, lm_head, None) apply_mock.assert_called_once() + assert logits is not None assert logits.dtype == torch.bfloat16 @@ -120,9 +131,10 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config): with mock.patch( "vllm.model_executor.layers.logits_processor.F.linear" ) as linear_mock: - logits = lp._get_logits(hidden_states, _FakeLmHead(weight), None) + logits = lp._get_logits(hidden_states, _fake_lm_head(weight), None) linear_mock.assert_not_called() + assert logits is not None assert logits.dtype == torch.float32 expected = torch.nn.functional.linear(hidden_states.float(), weight.float()) torch.testing.assert_close(logits, expected) @@ -131,7 +143,7 @@ def test_fp32_head_uses_mm_fast_path_on_device(default_vllm_config): def test_fp32_head_rejects_quantized_lm_head(default_vllm_config): lp = _build_processor(64) lp.head_dtype = torch.float32 - lm_head = _FakeLmHead(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True) + lm_head = _fake_lm_head(torch.randn(64, 16, dtype=torch.bfloat16), quantized=True) with pytest.raises(ValueError, match="unquantized"): lp._get_logits(torch.randn(4, 16, dtype=torch.bfloat16), lm_head, None) @@ -197,7 +209,7 @@ def test_get_top_tokens_honors_head_dtype(default_vllm_config): hidden_states = torch.randn(4, hidden_size, dtype=torch.bfloat16) weight = torch.randn(vocab_size, hidden_size, dtype=torch.bfloat16) - lm_head = _FakeLmHead( + lm_head = _fake_lm_head( weight, shard_indices=types.SimpleNamespace( num_org_vocab_padding=0, org_vocab_start_index=0 @@ -236,7 +248,9 @@ def test_fp32_head_e2e_no_nan(): for output in outputs: for completion in output.outputs: + assert completion.logprobs is not None for token_id, position in zip(completion.token_ids, completion.logprobs): + assert position is not None # The sampled token survived filtering, so its logprob is finite. assert math.isfinite(position[token_id].logprob) # No returned logprob is NaN. diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index 85551e80b4..78524acbb5 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -24,6 +24,7 @@ from vllm import SamplingParams from vllm.config.model import LogprobsMode from vllm.distributed import cleanup_dist_env_and_memory from vllm.exceptions import VLLMValidationError +from vllm.logprobs import Logprob from vllm.platforms import current_platform from vllm.v1.engine.input_processor import InputProcessor @@ -145,7 +146,7 @@ def _repeat_logprob_config( def _run_and_validate( vllm_model: VllmRunner, test_prompts: list[str], - vllm_sampling_params: SamplingParams, + vllm_sampling_params: SamplingParams | list[SamplingParams], hf_logprobs: list[list[torch.Tensor]], hf_outputs: list[tuple[list[int], str]], logprob_prompt_logprob_list: BatchLogprobsSpecType, @@ -163,11 +164,12 @@ def _run_and_validate( # Extract request-level (prompt)logprobs config num_top_logprobs, num_top_prompt_logprobs = logprob_prompt_logprob + assert vllm_result.prompt_token_ids is not None # Test whether sampled token output is consistent between vLLM and HF # vLLM prompt+completion should match HF output if temperature == 0.0: assert ( - vllm_result.prompt_token_ids + vllm_result.outputs[0].token_ids + vllm_result.prompt_token_ids + list(vllm_result.outputs[0].token_ids) == hf_output[0] ) else: @@ -191,7 +193,9 @@ def _run_and_validate( # Confirm that the output token appears among the logprobs assert token_id in logprobs - token_in_topk = logprobs[token_id].rank <= num_top_logprobs + rank = logprobs[token_id].rank + assert rank is not None + token_in_topk = rank <= num_top_logprobs # If the output token is not included in the top K # logprob, it can return 1 more data @@ -207,8 +211,10 @@ def _run_and_validate( output_text = vllm_result.outputs[0].text output_string_from_most_likely_tokens_lst: list[str] = [] - for top_logprobs in vllm_result.outputs[0].logprobs: - top_logprob = next(iter(top_logprobs.values())) + for output_logprobs in vllm_result.outputs[0].logprobs: + assert output_logprobs is not None + top_logprob = next(iter(output_logprobs.values())) + assert top_logprob.decoded_token is not None output_string_from_most_likely_tokens_lst.append( top_logprob.decoded_token ) @@ -226,8 +232,9 @@ def _run_and_validate( # Compare vLLM sample logprobs to HF vllm_sample_logprobs = vllm_result.outputs[0].logprobs - for i, top_logprobs in enumerate(vllm_sample_logprobs): - for token_id, sample_logprob in top_logprobs.items(): + for i, sample_logprobs in enumerate(vllm_sample_logprobs): + assert sample_logprobs is not None + for token_id, sample_logprob in sample_logprobs.items(): if temperature == 0.0 or i == 0: logprob = sample_logprob.logprob torch.testing.assert_close( @@ -271,9 +278,9 @@ def _run_and_validate( # Confirm that the prompt token appears among the logprobs assert prompt_token_id in prompt_logprobs - token_in_topk = ( - prompt_logprobs[prompt_token_id].rank <= num_top_prompt_logprobs - ) + prompt_rank = prompt_logprobs[prompt_token_id].rank + assert prompt_rank is not None + token_in_topk = prompt_rank <= num_top_prompt_logprobs # If the prompt token is not included in the top K # logprob, it can return 1 more data @@ -294,9 +301,10 @@ def _run_and_validate( # 1:. vllm_prompt_logprobs = vllm_result.prompt_logprobs[1:] for i, vllm_prompt_logprob_dict in enumerate(vllm_prompt_logprobs): - for token_id, logprob in vllm_prompt_logprob_dict.items(): + assert vllm_prompt_logprob_dict is not None + for token_id, prompt_lp in vllm_prompt_logprob_dict.items(): torch.testing.assert_close( - logprob.logprob, + prompt_lp.logprob, hf_logprob[0][i][token_id].item(), atol=2e-2, rtol=2e-2, @@ -478,12 +486,12 @@ def test_prompt_logprob_token_ids_require_v2_model_runner(): params = SamplingParams(prompt_logprob_token_ids=[1, 2]) with patch.object(SamplingParams, "verify"): with pytest.raises(VLLMValidationError, match="V2 model runner"): - InputProcessor._validate_params(processor, params, ("generate",)) + InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type] processor.vllm_config.use_v2_model_runner = True - InputProcessor._validate_params(processor, params, ("generate",)) + InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type] processor.vllm_config.cache_config.kv_sharing_fast_prefill = True with pytest.raises(VLLMValidationError, match="fast-prefill"): - InputProcessor._validate_params(processor, params, ("generate",)) + InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type] def test_none_logprobs(vllm_model, example_prompts): @@ -607,7 +615,9 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode): total_token_with_logprobs = 0 positive_values = 0 for output in results[0].outputs: + assert output.logprobs is not None for logprobs in output.logprobs: + assert logprobs is not None for token_id in logprobs: logprob = logprobs[token_id] if logprobs_mode in ("raw_logprobs", "processed_logprobs"): @@ -648,6 +658,7 @@ def test_prompt_logprobs_mode(): ) assert results[0].prompt_logprobs is not None assert results[0].prompt_logprobs[1] is not None + assert results[0].prompt_token_ids is not None tok_id = results[0].prompt_token_ids[1] values[mode] = results[0].prompt_logprobs[1][tok_id].logprob finally: @@ -1228,10 +1239,12 @@ def test_spec_decode_logprobs( [prompt, prompt], [sampling_params, penalty_sampling_params] ) # Collect logprobs outputs from reference LLM. - ref_logprobs = [] + ref_logprobs: list[Logprob] = [] for results in ref_results: for output in results.outputs: + assert output.logprobs is not None for logprobs in output.logprobs: + assert logprobs is not None ref_logprobs.extend(logprobs.values()) del ref_llm torch.accelerator.empty_cache() @@ -1249,10 +1262,12 @@ def test_spec_decode_logprobs( [prompt, prompt], [sampling_params, penalty_sampling_params] ) # Collect logprobs outputs from spec decode LLM. - spec_logprobs = [] + spec_logprobs: list[Logprob] = [] for results in spec_results: for output in results.outputs: + assert output.logprobs is not None for logprobs in output.logprobs: + assert logprobs is not None spec_logprobs.extend(logprobs.values()) del spec_llm torch.accelerator.empty_cache() @@ -1326,6 +1341,7 @@ def test_prompt_logprobs_with_chunking_and_preemption(): "Unexpected number of prompt logprob positions" ) + assert sampling_params.prompt_logprobs is not None # Each position should have the requested number of logprobs for pos, logprobs_dict in enumerate(prompt_logprobs): if logprobs_dict is not None: # First token may be None diff --git a/tests/v1/sample/test_sampler.py b/tests/v1/sample/test_sampler.py index b6d4fcc9e8..c23fe6c0ce 100644 --- a/tests/v1/sample/test_sampler.py +++ b/tests/v1/sample/test_sampler.py @@ -333,6 +333,7 @@ def test_sampler_repetition_penalty( for batch_idx in range(batch_size): non_penalized_token_id = logits[batch_idx].argmax().item() penalized_token_id = logits[batch_idx].argmin().item() + assert sampling_metadata.prompt_token_ids is not None prompt_tokens = sampling_metadata.prompt_token_ids[batch_idx][:].tolist() output_tokens = sampling_metadata.output_token_ids[batch_idx] if repetition_penalty > 1.0: diff --git a/tests/v1/sample/test_thinking_budget_state.py b/tests/v1/sample/test_thinking_budget_state.py index 943734e8ee..5ec85996b7 100644 --- a/tests/v1/sample/test_thinking_budget_state.py +++ b/tests/v1/sample/test_thinking_budget_state.py @@ -4,6 +4,7 @@ import torch +from tests.v1.sample.utils import create_mock_reasoning_config from vllm.sampling_params import SamplingParams from vllm.v1.sample.logits_processor.interface import ( BatchUpdate, @@ -12,14 +13,9 @@ from vllm.v1.sample.logits_processor.interface import ( from vllm.v1.sample.thinking_budget_state import ThinkingBudgetStateHolder -class _MockReasoningConfig: - reasoning_start_token_ids = [151667] - reasoning_end_token_ids = [151668] - - def _make_holder() -> ThinkingBudgetStateHolder: return ThinkingBudgetStateHolder( - _MockReasoningConfig(), + create_mock_reasoning_config([151667], [151668]), 8, 0, torch.device("cpu"), diff --git a/tests/v1/sample/test_trace_replay_params.py b/tests/v1/sample/test_trace_replay_params.py index d5736f1773..0a70859f84 100644 --- a/tests/v1/sample/test_trace_replay_params.py +++ b/tests/v1/sample/test_trace_replay_params.py @@ -104,7 +104,8 @@ def test_trace_decode_token_ids_rejects_speculative_decoding(): params = SamplingParams(trace_decode_token_ids=[1]) with pytest.raises(ValueError, match="not supported with speculative decoding"): params._validate_trace_replay( - _make_model_config(vocab_size=100), speculative_config=object() + _make_model_config(vocab_size=100), + speculative_config=object(), # type: ignore[arg-type] # Only non-None presence is tested. ) diff --git a/tests/v1/sample/utils.py b/tests/v1/sample/utils.py index e4e9f21092..88c1c3f99e 100644 --- a/tests/v1/sample/utils.py +++ b/tests/v1/sample/utils.py @@ -4,16 +4,28 @@ from collections.abc import Iterator from enum import Enum from typing import NamedTuple +from unittest.mock import MagicMock import regex as re import torch from vllm import CompletionOutput +from vllm.config import ReasoningConfig from vllm.utils.torch_utils import make_tensor_with_pad from vllm.v1.sample.logits_processor import BatchUpdate, LogitsProcessor from vllm.v1.sample.metadata import SamplingMetadata +def create_mock_reasoning_config( + start_token_ids: list[int], end_token_ids: list[int] +) -> ReasoningConfig: + return MagicMock( + spec=ReasoningConfig, + reasoning_start_token_ids=start_token_ids, + reasoning_end_token_ids=end_token_ids, + ) + + class BatchLogprobsComposition(Enum): """Types of logprobs configs to include in test batch.""" @@ -141,7 +153,11 @@ def compute_correct_cumulative_logprob(completion_output: CompletionOutput) -> f token_ids = completion_output.token_ids logprobs = completion_output.logprobs assert logprobs is not None - return sum([lp[tok_id].logprob for tok_id, lp in zip(token_ids, logprobs)]) + values: list[float] = [] + for tok_id, lp in zip(token_ids, logprobs): + assert lp is not None + values.append(lp[tok_id].logprob) + return sum(values) def create_fake_logits(batch_size: int, vocab_size: int) -> torch.Tensor: diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 392b64d992..514a84d931 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -25,9 +25,9 @@ def make_manager( num_reqs, num_steps = confidences.shape manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) manager.num_speculative_steps = num_steps - manager._stale_confidences = [SimpleNamespace(np=confidences)] + manager._stale_confidences = [SimpleNamespace(np=confidences)] # type: ignore[list-item] # CPU-only confidence buffer. manager._stale_idx = 0 - manager.req_states = SimpleNamespace( + manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture. req_id_to_index={"low": 0, "high": 1}, num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32), prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)), @@ -153,11 +153,11 @@ def test_manager_checks_target_varlen_cudagraph_bound( manager = maybe_create_adaptive_verification_manager( enable_adaptive_verification=True, attn_groups=groups, - req_states=SimpleNamespace(num_speculative_steps=7), + req_states=SimpleNamespace(num_speculative_steps=7), # type: ignore[arg-type] query_start_loc=object(), num_bonus_tokens=1, max_total_logits=1, - vllm_config=None, + vllm_config=None, # type: ignore[arg-type] target_layer_names={"target"}, additional_attn_cg_support=additional, ) @@ -176,6 +176,7 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): {"low": 3, "high": 3}, {"low": [1, 2], "high": [3, 4]}, ) + assert manager._batch_budget is not None valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget assert draft_budget == 1 @@ -183,14 +184,18 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): assert num_non_draft_tokens == {"low": 1, "high": 1} -def test_profiled_batches_seed_cost_curves_via_consumer(): +def test_profiled_batches_seed_cost_curves_via_consumer(monkeypatch): manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) - manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) + manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture. + max_num_batched_tokens=4096, max_num_reqs=64 + ) manager.num_speculative_steps = 7 manager.num_bonus_tokens = 1 curves: dict[str, list[tuple[int, float]]] = {} - manager.set_cost_curves = lambda draft, verify: curves.update( - draft=draft, verify=verify + monkeypatch.setattr( + manager, + "set_cost_curves", + lambda draft, verify: curves.update(draft=draft, verify=verify), ) timings = [ @@ -258,6 +263,7 @@ def test_budget_caps_at_one_rejection_sampler_chunk(): {"low": 3, "high": 3}, {"low": [1, 2], "high": [3, 4]}, ) + assert manager._batch_budget is not None _, _, draft_budget = manager._batch_budget assert draft_budget <= 1 @@ -285,6 +291,7 @@ def test_zero_budget_rebuilds_cpu_cu_num_logits(): {"low": 3, "high": 3, "prefill": 40}, {"low": [1, 2], "high": [3, 4]}, ) + assert manager._batch_budget is not None _, _, draft_budget = manager._batch_budget assert draft_budget == 0 @@ -323,6 +330,7 @@ def test_zero_budget_keeps_one_grammar_row_per_scheduled_draft(): manager.get_num_tokens( {"low": 3, "high": 3, "prefill": 40}, scheduled_spec_decode_tokens ) + assert manager._batch_budget is not None assert manager._batch_budget[2] == 0 req_ids = ["low", "high", "prefill"] diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index c508768b39..d5517ed61b 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -98,7 +98,7 @@ def test_draft_quant_exclusions_include_global_layer_indices(): ] ) - _add_global_draft_layer_exclusions(quant_config, 88, 5) + _add_global_draft_layer_exclusions(quant_config, 88, 5) # type: ignore[arg-type] assert "layers.88.mlp_conv*" in quant_config.exclude_modules assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules @@ -171,7 +171,7 @@ def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch): allocating one here would claim a proposal the walk never sampled from. """ _stub_base(monkeypatch, None) - speculator = DFlash2Speculator(None, torch.device("cpu")) + speculator = DFlash2Speculator(None, torch.device("cpu")) # type: ignore[arg-type] # Base initialization is stubbed. assert speculator.draft_logits is None @@ -183,7 +183,7 @@ def test_selector_asks_for_fp32_proposal_logits(): candidate row often enough that the walk and the rejection sampler checking it would no longer read the same distribution. """ - dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) + dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) # type: ignore[arg-type] # Constant spec ignores self/config. assert dtype is torch.float32 assert fill == float("-inf") @@ -309,9 +309,9 @@ def test_candidate_model_decoder_layer_cls(monkeypatch, variant): vllm_config.compilation_config = mock_current_vllm_config.compilation_config # 3. Instantiate the model under meta device to avoid parameter allocation issues - with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): + with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): # type: ignore[arg-type] model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr - model = model_cls(vllm_config=vllm_config) + model = model_cls(vllm_config=vllm_config) # type: ignore[arg-type] # 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass) assert len(model.layers) == 2 @@ -350,7 +350,7 @@ def test_conv_projections_use_draft_quant_config(monkeypatch): quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True ) layer = DFlash2Qwen3DecoderLayer( - SimpleNamespace( + SimpleNamespace( # type: ignore[arg-type] speculative_config=SimpleNamespace(num_speculative_tokens=7), model_config=SimpleNamespace(dtype=torch.bfloat16), ), @@ -434,13 +434,15 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch): for projection in projections ] qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers( - model, layers_attn, has_bias=False + model, # type: ignore[arg-type] + layers_attn, + has_bias=False, ) assert model._fused_kv_weight is None assert all(not hasattr(projection, "weight") for projection in projections) actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv( - model, + model, # type: ignore[arg-type] context_states, num_ctx=2, num_layers=2, diff --git a/tests/v1/spec_decode/test_dflash_lookahead.py b/tests/v1/spec_decode/test_dflash_lookahead.py index d2980fb610..55d039b05e 100644 --- a/tests/v1/spec_decode/test_dflash_lookahead.py +++ b/tests/v1/spec_decode/test_dflash_lookahead.py @@ -141,9 +141,9 @@ def test_dflash_drafter_window_reserves_bonus_token(): speculative_config=_dflash_speculative_config(NUM_SPECULATIVE_TOKENS), ) # window = 4, so 96 fits (96 + 4 == 100) but 97 does not (97 + 4 == 101) - assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) - assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) - assert not input_fits_in_drafter(dflash_runner, None) # no metadata + assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) # type: ignore[arg-type] + assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type] + assert not input_fits_in_drafter(dflash_runner, None) # type: ignore[arg-type] # No metadata. # Other drafters don't reserve the bonus token, so 97 fits (97 + 3 == 100). plain_runner = SimpleNamespace( @@ -151,4 +151,4 @@ def test_dflash_drafter_window_reserves_bonus_token(): effective_drafter_max_model_len=100, speculative_config=SimpleNamespace(use_dflash=lambda: False), ) - assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) + assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type] diff --git a/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py b/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py index 2c508864c6..4efe53451e 100644 --- a/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py +++ b/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py @@ -56,6 +56,7 @@ def _make_predictor(vocab_size: int) -> DeepSeekV4MultiTokenPredictor: ): shared_head = SharedHead(config=config, prefix="shared_head") + assert shared_head.head is not None generator = torch.Generator().manual_seed(7) with torch.no_grad(): shared_head.head.weight.normal_(0.0, 0.02, generator=generator) @@ -157,7 +158,9 @@ def test_wrapper_forwards_spec_step_idx(): hidden_states = torch.empty(0) wrapper = object.__new__(DeepSeekV4MTP) - wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args) + wrapper.model = SimpleNamespace( # type: ignore[assignment] + get_top_tokens=lambda *args: args + ) assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2) assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0) diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py index 392135f7b0..f3971b34e9 100644 --- a/tests/v1/spec_decode/test_dynamic_sd_cug.py +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -25,7 +25,7 @@ def _create_vllm_config_for_dsd( max_num_seqs: int, max_spec_tokens: int, *, - cudagraph_mode: str = "FULL_AND_PIECEWISE", + cudagraph_mode: CUDAGraphMode = CUDAGraphMode.FULL_AND_PIECEWISE, use_dynamic_sd: bool = True, num_spec_per_batch_size: list[tuple[int, int, int]] | None = None, ) -> MagicMock: @@ -199,7 +199,7 @@ def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch): vllm_config = _create_vllm_config_for_dsd( max_num_seqs=512, max_spec_tokens=max_spec_tokens, - cudagraph_mode="FULL_AND_PIECEWISE", + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, use_dynamic_sd=True, ) manager = gpu_cudagraph_utils.CudaGraphManager( @@ -316,7 +316,7 @@ def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch): vllm_config = _create_vllm_config_for_dsd( max_num_seqs=max_num_seqs, max_spec_tokens=max_spec_tokens, - cudagraph_mode="FULL_AND_PIECEWISE", + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, use_dynamic_sd=False, ) manager = gpu_cudagraph_utils.CudaGraphManager( @@ -382,7 +382,7 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch): vllm_config = _create_vllm_config_for_dsd( max_num_seqs=max_num_seqs, max_spec_tokens=max_spec_tokens, - cudagraph_mode="FULL_AND_PIECEWISE", + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, use_dynamic_sd=True, num_spec_per_batch_size=num_spec_per_batch_size, ) diff --git a/tests/v1/spec_decode/test_eagle.py b/tests/v1/spec_decode/test_eagle.py index 9b3bfd73d0..2ddd0a9774 100644 --- a/tests/v1/spec_decode/test_eagle.py +++ b/tests/v1/spec_decode/test_eagle.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Literal from unittest import mock import numpy as np @@ -31,6 +32,7 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.spec_decode.dflash import DFlashProposer from vllm.v1.spec_decode.draft_model import DraftModelProposer from vllm.v1.spec_decode.eagle import EagleProposer +from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer from vllm.v1.spec_decode.metadata import SpecDecodeMetadata from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch @@ -46,13 +48,13 @@ DEVICE_TYPE = current_platform.device_type def _create_proposer( - method: str, + method: Literal["eagle", "eagle3", "draft_model", "dflash"], num_speculative_tokens: int, attention_backend: str | None = None, parallel_drafting: bool = False, - rejection_sample_method: str = "standard", - draft_sample_method: str = "greedy", -) -> EagleProposer: + rejection_sample_method: Literal["standard", "synthetic", "block"] = "standard", + draft_sample_method: Literal["greedy", "probabilistic"] = "greedy", +) -> SpecDecodeBaseProposer: # Method-dependent setup if method == "eagle": target_model_dir = model_dir @@ -102,9 +104,14 @@ def _create_proposer( max_model_len=model_config.max_model_len, is_encoder_decoder=model_config.is_encoder_decoder, ), - attention_config=AttentionConfig(backend=attention_backend), + attention_config=AttentionConfig( + backend=AttentionBackendEnum[attention_backend] + if attention_backend + else None + ), ) + proposer: SpecDecodeBaseProposer if method == "dflash": proposer = DFlashProposer(vllm_config=vllm_config, device=device) elif "eagle" in method: @@ -135,7 +142,7 @@ def test_prepare_next_token_ids(): ) mock_num_scheduled_tokens = {req_id: 0 for req_id in req_ids} - mock_requests = {} + mock_requests: dict[str, CachedRequestState] = {} for req_id in req_ids: mock_request = mock.MagicMock(spec=CachedRequestState) # Each request will have a backup next token id of 10, 20, 30, 40 @@ -948,13 +955,12 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch): attn_metadata_builder = attn_metadata_builder_cls( kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config), - layer_names=proposer._draft_attn_layer_names, + layer_names=sorted(proposer._draft_attn_layer_names), vllm_config=proposer.vllm_config, device=device, ) - # Mock runner and draft_attn_groups for attention metadata building - proposer.runner = mock.MagicMock() + # Mock draft_attn_groups for attention metadata building. mock_attn_group = mock.MagicMock() mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder mock_attn_group.layer_names = list(proposer._draft_attn_layer_names) @@ -1054,11 +1060,10 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch): ) attn_metadata_builder = attn_metadata_builder_cls( kv_cache_spec=create_standard_kv_cache_spec(proposer.vllm_config), - layer_names=proposer._draft_attn_layer_names, + layer_names=sorted(proposer._draft_attn_layer_names), vllm_config=proposer.vllm_config, device=device, ) - proposer.runner = mock.MagicMock() mock_attn_group = mock.MagicMock() mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder mock_attn_group.layer_names = list(proposer._draft_attn_layer_names) @@ -1133,6 +1138,7 @@ def test_set_inputs_first_pass_dflash(): num_speculative_tokens = 3 proposer = _create_proposer("dflash", num_speculative_tokens) + assert isinstance(proposer, DFlashProposer) mask_token_id = proposer.parallel_drafting_token_id # Setup batch with 3 requests diff --git a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py index 2c4b951be7..aa6a65661d 100644 --- a/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py +++ b/tests/v1/spec_decode/test_eagle_draft_attn_metadata.py @@ -93,7 +93,7 @@ def _run_build( with patch.object(base_speculator, "build_attn_metadata", fake_build_attn_metadata): EagleSpeculator._build_uniform_attn_metadata( - fake, # type: ignore[arg-type] + fake, batch_desc=batch_desc, num_reqs=num_reqs, num_query_per_req=num_query_per_req, diff --git a/tests/v1/spec_decode/test_extract_hidden_states.py b/tests/v1/spec_decode/test_extract_hidden_states.py index 7eb8eca023..0c9197bb6a 100644 --- a/tests/v1/spec_decode/test_extract_hidden_states.py +++ b/tests/v1/spec_decode/test_extract_hidden_states.py @@ -51,7 +51,7 @@ def _create_proposer( target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=num_speculative_tokens, - draft_model_config={ + draft_model_config={ # type: ignore[arg-type] "hf_config": { "eagle_aux_hidden_state_layer_ids": layer_ids, } @@ -103,7 +103,7 @@ def test_proposer_initialization_missing_layer_ids(): target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=1, - draft_model_config={ + draft_model_config={ # type: ignore[arg-type] "hf_config": {} # Missing eagle_aux_hidden_state_layer_ids }, ) @@ -146,7 +146,7 @@ def test_prepare_next_token_ids_padded(): mock_input_batch.vocab_size = 100 mock_input_batch.num_tokens_no_spec = np.array([5] * num_requests) - mock_requests = {} + mock_requests: dict[str, CachedRequestState] = {} for req_id in req_ids: mock_request = mock.MagicMock(spec=CachedRequestState) # Each request will have a backup next token id of 10, 20, 30, 40 @@ -363,7 +363,7 @@ def test_extract_hidden_states_text_only_config_regression(): target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=1, - draft_model_config={ + draft_model_config={ # type: ignore[arg-type] "hf_config": { "eagle_aux_hidden_state_layer_ids": [1, 2, 3, 4], } @@ -450,7 +450,7 @@ def test_extract_hidden_states_speculative_config_vlm(): target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=1, - draft_model_config={ + draft_model_config={ # type: ignore[arg-type] "hf_config": { "eagle_aux_hidden_state_layer_ids": [1, 2], } diff --git a/tests/v1/spec_decode/test_lilicorr.py b/tests/v1/spec_decode/test_lilicorr.py index 14bad45f6b..88c91696f3 100644 --- a/tests/v1/spec_decode/test_lilicorr.py +++ b/tests/v1/spec_decode/test_lilicorr.py @@ -11,6 +11,7 @@ from torch import nn from vllm.config import ( CompilationConfig, + CompilationMode, DeviceConfig, LoadConfig, VllmConfig, @@ -122,7 +123,7 @@ def test_lilicorr_matches_exported_head(head_width, slots, dtype): set_current_vllm_config( VllmConfig( device_config=DeviceConfig("cuda"), - compilation_config=CompilationConfig(mode=0), + compilation_config=CompilationConfig(mode=CompilationMode.NONE), ) ), ): @@ -183,23 +184,23 @@ def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch): with set_current_vllm_config( VllmConfig( device_config=DeviceConfig("cpu"), - compilation_config=CompilationConfig(mode=0), + compilation_config=CompilationConfig(mode=CompilationMode.NONE), ) ): for slots in (1, 3): spec.num_speculative_tokens = slots head = LiLiCorr( - vllm_config=SimpleNamespace(speculative_config=spec) + vllm_config=SimpleNamespace(speculative_config=spec) # type: ignore[arg-type] ).lilicorr assert head.slot_embedding.shape == (1, 1, 3, 1, 8) assert head.relative_slot_bias.shape == (2, 7) spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False with pytest.raises(ValueError, match="lilicorr_enabled"): - LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type] spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True spec.num_speculative_tokens = 4 with pytest.raises(ValueError, match="trained block_size"): - LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type] @pytest.mark.parametrize("tp_size", [1, 2]) @@ -268,7 +269,11 @@ def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_siz ) monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1)) ids, values = logits_processor.LogitsProcessor.get_top_k_tokens( - fake, lm_head, torch.empty(2, 1), 2, return_log_probs=True + fake, # type: ignore[arg-type] + lm_head, # type: ignore[arg-type] + torch.empty(2, 1), + 2, + return_log_probs=True, ) expected = torch.cat(valid_shards, -1).log_softmax(-1).topk(2) torch.testing.assert_close(ids, expected.indices) @@ -286,13 +291,13 @@ def test_context_anchor_is_last_committed_normalized_feature(): anchor_valid=torch.ones(4, dtype=torch.bool), ) batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6])) - LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) # type: ignore[arg-type] torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]])) assert spec.anchor_valid.tolist() == [True, True, False, False] assert not spec.anchor_hidden[2:].any() # A smaller reordered batch must not retain an old request's anchor. batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3])) - LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) # type: ignore[arg-type] torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1])) assert not spec.anchor_hidden[1:].any() @@ -316,7 +321,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil monkeypatch.setattr(DFlashSpeculator, "__init__", init_base) cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator - spec = cls(None, torch.device("cpu")) + spec = cls(None, torch.device("cpu")) # type: ignore[arg-type] hidden = torch.arange(36).float().view(9, 4) spec._run_model = lambda *args: hidden spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8]) @@ -329,7 +334,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil spec.draft_logits = None spec.use_fp64_gumbel = False spec.enable_adaptive_verification = True - spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) + spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) # type: ignore[assignment] candidates = torch.arange(24).view(6, 4) log_probs = candidates.float().log_softmax(-1) scores = torch.randn(3, 2, 4, 4) @@ -349,22 +354,35 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil compute_candidates=lambda h: (candidates, log_probs), model=SimpleNamespace(lilicorr=score, candidate_selector=score), ) - spec.target_embeddings = lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4) if lilicorr: + assert isinstance(spec, LiLiCorrSpeculator) + monkeypatch.setattr( + spec, + "target_embeddings", + lambda ids: ids.float().unsqueeze(-1).expand(*ids.shape, 4), + raising=False, + ) spec.anchor_hidden.copy_(torch.arange(12).view(3, 4)) spec.anchor_valid[:2] = True - spec.candidate_sampler.sample = sample - spec._maybe_predict_acceptance = lambda *args: adaptive_inputs.extend(args) + monkeypatch.setattr(spec.candidate_sampler, "sample", sample) + monkeypatch.setattr( + spec, "_maybe_predict_acceptance", lambda *args: adaptive_inputs.extend(args) + ) spec._generate_draft(3, 9, None, None, None) expected_ids = candidates.view(3, 2, 4) - expected_first = spec.target_embeddings(expected_ids) if lilicorr else expected_ids + expected_first = ( + spec.target_embeddings(expected_ids) + if isinstance(spec, LiLiCorrSpeculator) + else expected_ids + ) torch.testing.assert_close(scoring_inputs[0], expected_first) torch.testing.assert_close(scoring_inputs[1], log_probs.view(3, 2, 4)) torch.testing.assert_close( scoring_inputs[2], hidden[spec.sample_indices].view(3, 2, 4) ) if lilicorr: + assert isinstance(spec, LiLiCorrSpeculator) torch.testing.assert_close(scoring_inputs[3], spec.anchor_hidden) torch.testing.assert_close(scoring_inputs[4], spec.anchor_valid) else: @@ -472,7 +490,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads( ): from vllm.model_executor.models.lilicorr import LiLiCorr, LiLiCorrForCausalLM - wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) + wrapper = LiLiCorrForCausalLM.__new__(LiLiCorrForCausalLM) # type: ignore[type-abstract] nn.Module.__init__(wrapper) wrapper.model = LiLiCorr.__new__(LiLiCorr) nn.Module.__init__(wrapper.model) @@ -480,7 +498,7 @@ def test_checkpoint_coverage_rejects_incomplete_or_wrong_heads( with set_current_vllm_config( VllmConfig( device_config=DeviceConfig("cpu"), - compilation_config=CompilationConfig(mode=0), + compilation_config=CompilationConfig(mode=CompilationMode.NONE), ) ): wrapper.model.lilicorr = LiLiCorrHead( @@ -615,10 +633,10 @@ def owned_head_model(monkeypatch): with set_current_vllm_config( VllmConfig( device_config=DeviceConfig("cpu"), - compilation_config=CompilationConfig(mode=0), + compilation_config=CompilationConfig(mode=CompilationMode.NONE), ) ): - model = LiLiCorrForCausalLM(vllm_config=vllm_config) + model = LiLiCorrForCausalLM(vllm_config=vllm_config) # type: ignore[arg-type] return model, vllm_config return build @@ -720,7 +738,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch with set_current_vllm_config( VllmConfig( device_config=DeviceConfig("cpu"), - compilation_config=CompilationConfig(mode=0), + compilation_config=CompilationConfig(mode=CompilationMode.NONE), ) ): head = LiLiCorrHead( @@ -728,7 +746,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch block_size=4, rms_norm_eps=1e-6, config=_config(hidden_size=8), - quant_config=quant_config, + quant_config=quant_config, # type: ignore[arg-type] prefix="model.lilicorr", ) with torch.no_grad(): diff --git a/tests/v1/spec_decode/test_llm_base_proposer.py b/tests/v1/spec_decode/test_llm_base_proposer.py index 5510c36e60..f9c1795b97 100644 --- a/tests/v1/spec_decode/test_llm_base_proposer.py +++ b/tests/v1/spec_decode/test_llm_base_proposer.py @@ -16,6 +16,7 @@ from types import SimpleNamespace import pytest import vllm.v1.spec_decode.llm_base_proposer as llm_base_proposer +from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.spec_decode.eagle import EagleProposer SCHEDULER_BLOCK_SIZE = 256 @@ -52,7 +53,7 @@ def _make_proposer( monkeypatch.setattr(llm_base_proposer, "AttentionGroup", _FakeAttentionGroup) proposer = EagleProposer.__new__(EagleProposer) - proposer.vllm_config = None + proposer.vllm_config = None # type: ignore[assignment] # Patched backend discovery does not read config. proposer.device = None proposer._draft_attn_layer_names = set(layer_names) proposer.kv_cache_gid = -1 @@ -61,10 +62,10 @@ def _make_proposer( return proposer -def _make_kv_cache_config(layer_names: set[str]) -> SimpleNamespace: +def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig: spec = SimpleNamespace(block_size=SCHEDULER_BLOCK_SIZE) group = SimpleNamespace(layer_names=list(layer_names), kv_cache_spec=spec) - return SimpleNamespace(kv_cache_groups=[group]) + return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value] def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch): @@ -81,7 +82,9 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch): assert proposer.block_size == KERNEL_BLOCK_SIZE assert proposer.block_size != SCHEDULER_BLOCK_SIZE # The metadata builder keeps receiving the kernel block size as well. - assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE + group = proposer.draft_attn_groups[0] + assert isinstance(group, _FakeAttentionGroup) + assert group.kernel_block_size == KERNEL_BLOCK_SIZE def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch): diff --git a/tests/v1/spec_decode/test_llm_base_proposer_sampling.py b/tests/v1/spec_decode/test_llm_base_proposer_sampling.py index cb390b992f..7531b8b6cd 100644 --- a/tests/v1/spec_decode/test_llm_base_proposer_sampling.py +++ b/tests/v1/spec_decode/test_llm_base_proposer_sampling.py @@ -85,7 +85,7 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race(): def test_mtp_model_returns_tuple(architecture: str, expected: bool): proposer = object.__new__(SpecDecodeBaseProposer) proposer.method = "mtp" - proposer.draft_model_config = SimpleNamespace( + proposer.draft_model_config = SimpleNamespace( # type: ignore[assignment] # Architecture-only config. hf_config=SimpleNamespace(architectures=[architecture]) ) diff --git a/tests/v1/spec_decode/test_mtp.py b/tests/v1/spec_decode/test_mtp.py index e3803509e3..46c7064b0d 100644 --- a/tests/v1/spec_decode/test_mtp.py +++ b/tests/v1/spec_decode/test_mtp.py @@ -220,7 +220,6 @@ def test_mtp_propose(num_speculative_tokens, monkeypatch): device=device, ) - proposer.runner = mock.MagicMock() mock_attn_group = mock.MagicMock() mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder mock_attn_group.layer_names = list(proposer._draft_attn_layer_names) diff --git a/tests/v1/spec_decode/test_mtp_structured_output.py b/tests/v1/spec_decode/test_mtp_structured_output.py index 4e12a6f49d..5e4312227d 100644 --- a/tests/v1/spec_decode/test_mtp_structured_output.py +++ b/tests/v1/spec_decode/test_mtp_structured_output.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Backend and engine-reasoner checks that are not manager-flow tests.""" +from typing import Literal + import pytest from transformers import AutoTokenizer @@ -17,7 +19,9 @@ TOKENIZER = "gpt2" NUM_SPEC_TOKENS = 4 -def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'): +def _make_manager_and_request( + backend: Literal["xgrammar", "guidance"], prompt_str: str = '{"a": "b"}' +): tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) prompt = tokenizer.encode(prompt_str) @@ -33,6 +37,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'): sampling_params = SamplingParams( structured_outputs=StructuredOutputsParams(json='{"type": "object"}'), ) + assert sampling_params.structured_outputs is not None sampling_params.structured_outputs._backend = backend sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) @@ -43,6 +48,7 @@ def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'): pooling_params=None, ) manager.grammar_init(request) + assert request.structured_output_request is not None while not request.structured_output_request._check_grammar_completion(): continue diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index c4e704ae24..eb95dbdc61 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -168,7 +168,7 @@ def test_watermarked_bonus_uses_target_key(): watermarker=GumbelWatermarker(key=42, context_width=2), ) expected = GumbelWatermarker(key=42, context_width=2).sample( - target_logits.unsqueeze(0), contexts[1:], lambda _: None + target_logits.unsqueeze(0), contexts[1:] ) assert num_sampled.item() == 2 @@ -196,7 +196,7 @@ def test_watermarked_recovery_uses_target_key(): watermarker=GumbelWatermarker(key=42, context_width=2), ) expected = GumbelWatermarker(key=42, context_width=2).sample( - target_logits.unsqueeze(0), contexts[:1], lambda _: None + target_logits.unsqueeze(0), contexts[:1] ) assert num_sampled.item() == 1 @@ -225,9 +225,7 @@ def test_watermarked_recovery_supports_smaller_draft_vocabulary(): watermarking=torch.tensor([True], device="cuda"), watermarker=watermarker, ) - expected = watermarker.sample( - target_logits[:4].unsqueeze(0), contexts[:1], lambda _: None - ) + expected = watermarker.sample(target_logits[:4].unsqueeze(0), contexts[:1]) assert num_sampled.item() == 1 assert sampled[0, 0] == expected.token_ids[0] @@ -702,7 +700,7 @@ def test_verify_rejects_unproposed_drafts(): first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0] prefill_len = first_pos + 1 rejection_sampler = object.__new__(RejectionSampler) - rejection_sampler.sampler = SimpleNamespace( + rejection_sampler.sampler = SimpleNamespace( # type: ignore[assignment] apply_sampling_params=lambda logits, *args: logits, sampling_states=SimpleNamespace( temperature=SimpleNamespace(gpu=inputs["temperature"]), diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index dbef99b353..4bef23869b 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -28,27 +28,36 @@ import regex as re # Fixing a directory means moving it from SEPARATE_GROUPS to here. Without that # move the fixes are not enforced because "tests" claims every file below it. SILENT_GROUPS = [ + "tests/benchmarks", "tests/compile/correctness_e2e", "tests/compile/fullgraph", "tests/compile/fusions_e2e", "tests/compile/passes", "tests/config", + "tests/entrypoints/anthropic", "tests/entrypoints/generate", + "tests/entrypoints/multimodal", + "tests/entrypoints/pooling", "tests/entrypoints/serve", "tests/entrypoints/tool_parsers", "tests/entrypoints/unit_tests", "tests/entrypoints/weight_transfer", + "tests/kernels/attention", "tests/kernels/core", + "tests/kernels/helion", "tests/kernels/mamba", "tests/kernels/quantization", "tests/model_executor/layers", "tests/models/language", "tests/models/quantization", + "tests/multimodal", "tests/plugins/bge_m3_sparse_plugin", "tests/plugins/prithvi_io_processor_plugin", "tests/plugins/vllm_add_dummy_platform", "tests/plugins/vllm_add_dummy_stat_logger", "tests/plugins_tests", + "tests/quantization", + "tests/reasoning", "tests/samplers", "tests/spec_decode", "tests/tokenizers_", @@ -59,8 +68,11 @@ SILENT_GROUPS = [ "tests/v1/determinism", "tests/v1/distributed", "tests/v1/executor", + "tests/v1/logits_processors", + "tests/v1/sample", "tests/v1/shutdown", "tests/v1/simple_kv_offload", + "tests/v1/spec_decode", "tests/v1/structured_output", ] @@ -68,18 +80,12 @@ SILENT_GROUPS = [ # from "skip" to "silent", move its directory to SILENT_GROUPS. SEPARATE_GROUPS = [ "tests", - "tests/benchmarks", "tests/compile", "tests/distributed", - "tests/entrypoints/anthropic", "tests/entrypoints/llm", - "tests/entrypoints/multimodal", "tests/entrypoints/openai", - "tests/entrypoints/pooling", "tests/entrypoints/speech_to_text", "tests/kernels", - "tests/kernels/attention", - "tests/kernels/helion", "tests/kernels/moe", "tests/lora", "tests/model_executor", @@ -87,10 +93,7 @@ SEPARATE_GROUPS = [ "tests/models", "tests/models/test_initialization.py", "tests/models/multimodal", - "tests/multimodal", "tests/parser", - "tests/quantization", - "tests/reasoning", "tests/renderers", "tests/tool_parsers", "tests/v1", @@ -101,10 +104,7 @@ SEPARATE_GROUPS = [ "tests/v1/engine", "tests/v1/kv_connector", "tests/v1/kv_offload", - "tests/v1/logits_processors", "tests/v1/metrics", - "tests/v1/sample", - "tests/v1/spec_decode", "tests/v1/streaming_input", "tests/v1/worker", ] From 5a4351ab096268944bf29a9b804800066309baa9 Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Wed, 30 Sep 2026 13:08:24 +0200 Subject: [PATCH 110/248] [MM] Keep device input normalization fused when encoder compilation is enabled (#59195) Signed-off-by: Jakub Zakrzewski Co-authored-by: Codex --- .../layers/test_mm_input_norm.py | 19 +++++++++++++++++++ .../layers/fusion/mm_input_norm.py | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/tests/model_executor/layers/test_mm_input_norm.py b/tests/model_executor/layers/test_mm_input_norm.py index fc114eccd0..e3df570acc 100644 --- a/tests/model_executor/layers/test_mm_input_norm.py +++ b/tests/model_executor/layers/test_mm_input_norm.py @@ -6,6 +6,8 @@ import pytest import torch +from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config +from vllm.config.compilation import CompilationMode from vllm.model_executor.layers.fusion.mm_input_norm import ( FusedMMInputNorm, IdentityInputNorm, @@ -263,6 +265,23 @@ class TestFusedMMInputNormKernel: class TestFusedMMInputNormConstruction: """Weight/bias buffer semantics at init time.""" + def test_input_norm_is_always_enabled(self): + config = VllmConfig( + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + compile_mm_encoder=True, + custom_ops=["none"], + ) + ) + with set_current_vllm_config(config): + FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE) + assert ( + config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1 + ) + assert not config.compilation_config.disabled_custom_ops[ + "fused_mm_input_norm" + ] + def test_identity_config_buffers(self): """Numerically identity parameters still build plain weight/bias buffers; the identity shortcut lives in ``from_model_config``.""" diff --git a/vllm/model_executor/layers/fusion/mm_input_norm.py b/vllm/model_executor/layers/fusion/mm_input_norm.py index 5127f1007e..d125f8d2d2 100644 --- a/vllm/model_executor/layers/fusion/mm_input_norm.py +++ b/vllm/model_executor/layers/fusion/mm_input_norm.py @@ -246,6 +246,10 @@ class FusedMMInputNorm(CustomOp): self.register_buffer("weight", (rescale_factor / std).to(device)) self.register_buffer("bias", (-mean / std).to(device)) + @classmethod + def enabled(cls) -> bool: + return True + @property def input_dtype(self) -> torch.dtype | None: return torch.uint8 From 863475eb9cd78ae6a4a22f1b90df7fe85e0af438 Mon Sep 17 00:00:00 2001 From: yuvalluria <131433771+yuvalluria@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:13:23 +0300 Subject: [PATCH 111/248] docs: add gemma-2, SmolLM2, Qwen2.5-Coder, Llama-3.2-1B to batch invariance tested models (#54441) Signed-off-by: Yuval Luria Co-authored-by: Claude Sonnet 4.6 --- docs/features/batch_invariance.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 0fad4682fc..ff704348b5 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -127,7 +127,7 @@ Batch invariance has been tested and verified on the following models: - **Qwen3-VL (Vision-Language)**: `Qwen/Qwen3-VL-2B-Instruct`, `Qwen/Qwen3-VL-4B-Instruct` (single image and video inputs) - **Qwen3 (MoE)**: `Qwen/Qwen3-30B-A3B`, `Qwen/Qwen3-Next-80B-A3B-Instruct`, `Qwen/Qwen3-30B-A3B-Thinking-2507-FP8` - **Qwen2.5**: `Qwen/Qwen2.5-0.5B-Instruct`, `Qwen/Qwen2.5-1.5B-Instruct`, `Qwen/Qwen2.5-3B-Instruct`, `Qwen/Qwen2.5-7B-Instruct`, `Qwen/Qwen2.5-14B-Instruct`, `Qwen/Qwen2.5-32B-Instruct` -- **Llama 3**: Llama3.1 and 3.2 series, `meta-llama/Llama-3.2-3B-Instruct` for example +- **Llama 3**: Llama3.1 and 3.2 series, `meta-llama/Llama-3.2-1B-Instruct`, `meta-llama/Llama-3.2-3B-Instruct` for example - **GPT-OSS**: `openai/gpt-oss-20b`, `openai/gpt-oss-120b` - **Mistral**: `mistralai/Mistral-7B-v0.3` - **Phi series**: `microsoft/Phi-3.5-mini-instruct` @@ -135,6 +135,8 @@ Batch invariance has been tested and verified on the following models: - **Granite 3.1 (Dense)**: `ibm-granite/granite-3.1-2b-instruct`, `ibm-granite/granite-3.1-8b-instruct` - **EXAONE 4.0 series**: `LGAI-EXAONE/EXAONE-4.0-1.2B`, `LGAI-EXAONE/EXAONE-4.0.1-32B`, `LGAI-EXAONE/EXAONE-4.0-32B` - **OLMo 2**: `allenai/OLMo-2-0425-1B-Instruct` +- **SmolLM2**: `HuggingFaceTB/SmolLM2-1.7B-Instruct` +- **PLaMo3**: `pfnet/plamo-3-nict-2b-base` Other models may also work, but these have been explicitly validated. If you encounter issues with a specific model, please report them on the [GitHub issue tracker](https://github.com/vllm-project/vllm/issues/new/choose). From 72e7874fa669617fd20c716e08cb486032f542c1 Mon Sep 17 00:00:00 2001 From: Rukhaiya2004 <162583766+Rukhaiya2004@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:14:48 +0530 Subject: [PATCH 112/248] [Hardware][Power] Enable W4A16 (AWQ & GPTQ) quantization on POWER10 using VSX (#59149) Signed-off-by: Rukhaiya Co-authored-by: Li, Jiang --- cmake/cpu_extension.cmake | 3 +- csrc/cpu/cpu_types_vsx.hpp | 46 ++++++++++++++++++- csrc/cpu/cpu_wna16.cpp | 33 ++++++++++++- csrc/cpu/torch_bindings.cpp | 3 +- docs/getting_started/installation/cpu.md | 4 +- tests/quantization/test_cpu_wna16.py | 14 +++++- .../kernels/linear/mixed_precision/cpu.py | 5 ++ 7 files changed, 99 insertions(+), 9 deletions(-) diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 94dd188822..664e309e94 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -530,8 +530,9 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) endif() endif() -if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) +if (POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND) set(VLLM_EXT_SRC + "csrc/cpu/cpu_wna16.cpp" "csrc/cpu/shm.cpp" ${VLLM_EXT_SRC}) endif() diff --git a/csrc/cpu/cpu_types_vsx.hpp b/csrc/cpu/cpu_types_vsx.hpp index eaea921d98..31065508d7 100644 --- a/csrc/cpu/cpu_types_vsx.hpp +++ b/csrc/cpu/cpu_types_vsx.hpp @@ -417,7 +417,7 @@ struct FP32Vec8 : public Vec { tmp.val[1] = two_x; FP32Vec8 temp_vec(tmp); FP32Vec8 exp_vec = temp_vec.exp(); - vector float e = temp_vec.exp().reg.val[0]; + vector float e = exp_vec.reg.val[0]; vector float num = vec_sub(e, one); vector float den = vec_add(e, one); @@ -624,6 +624,24 @@ struct FP32Vec16 : public Vec { explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {} + // De-pack 16 x 4-bit nibbles from a 64-bit value and convert with offset. + // Used by WNA16 (AWQ/GPTQ) dequantization. + // Fully vectorized using VSX / AltiVec intrinsics with zero stack stores. + FORCE_INLINE explicit FP32Vec16(int64_t value, const FP32Vec16& lut) { + uint64_t u = static_cast(value); + float f[16]; +#pragma GCC unroll 16 + for (int i = 0; i < 16; i++) { + f[i] = static_cast((u >> (i * 4)) & 0xF); + } + // Extract base offset from LUT[0] (0.0f for AWQ, -8.0f for GPTQ) + __vector float offset = vec_splats(vec_extract(lut.reg.val[0], 0)); + reg.val[0] = vec_add(vec_xl(0, f), offset); + reg.val[1] = vec_add(vec_xl(16, f), offset); + reg.val[2] = vec_add(vec_xl(32, f), offset); + reg.val[3] = vec_add(vec_xl(48, f), offset); + } + // FP8 stub: dead code on PowerPC (fp8 KV cache is x86-only), needed for // load_b_pair_vec template to compile on all platforms. explicit FP32Vec16(const BF16Vec32&, int) : reg{} {} @@ -1102,6 +1120,32 @@ inline void prefetch(const void* addr) { __asm__ __volatile__("dcbt 0, %0" : : "r"(addr) : "memory"); } +static void interleave_save(const BF16Vec16& vec0, const BF16Vec16& vec1, + void* ptr) { + __vector signed short h0 = vec_mergeh(vec0.reg.val[0], vec1.reg.val[0]); + __vector signed short l0 = vec_mergel(vec0.reg.val[0], vec1.reg.val[0]); + __vector signed short h1 = vec_mergeh(vec0.reg.val[1], vec1.reg.val[1]); + __vector signed short l1 = vec_mergel(vec0.reg.val[1], vec1.reg.val[1]); + + vec_xst(h0, 0, reinterpret_cast(ptr)); + vec_xst(l0, 16, reinterpret_cast(ptr)); + vec_xst(h1, 32, reinterpret_cast(ptr)); + vec_xst(l1, 48, reinterpret_cast(ptr)); +} + +static void interleave_save(const FP16Vec16& vec0, const FP16Vec16& vec1, + void* ptr) { + __vector signed short h0 = vec_mergeh(vec0.reg.val[0], vec1.reg.val[0]); + __vector signed short l0 = vec_mergel(vec0.reg.val[0], vec1.reg.val[0]); + __vector signed short h1 = vec_mergeh(vec0.reg.val[1], vec1.reg.val[1]); + __vector signed short l1 = vec_mergel(vec0.reg.val[1], vec1.reg.val[1]); + + vec_xst(h0, 0, reinterpret_cast(ptr)); + vec_xst(l0, 16, reinterpret_cast(ptr)); + vec_xst(h1, 32, reinterpret_cast(ptr)); + vec_xst(l1, 48, reinterpret_cast(ptr)); +} + struct INT8Vec64 { __vector signed char data[4]; INT8Vec64() = default; diff --git a/csrc/cpu/cpu_wna16.cpp b/csrc/cpu/cpu_wna16.cpp index d3e42660f2..209c89454e 100644 --- a/csrc/cpu/cpu_wna16.cpp +++ b/csrc/cpu/cpu_wna16.cpp @@ -7,6 +7,9 @@ #if defined(__riscv_v) #include "cpu/micro_gemm/cpu_micro_gemm_rvv.hpp" #endif +#if defined(__powerpc__) + #include "cpu/micro_gemm/cpu_micro_gemm_vsx.hpp" +#endif #include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp" #define VLLM_DISPATCH_CASE_16B_TYPES(...) \ @@ -104,8 +107,8 @@ class Dequantizer4b { scalar_vec_t output_vec_0(wb_0); scalar_vec_t output_vec_1(wb_1); - // AMX needs to interleave K elements to pack as 32 bits - if constexpr (isa == ISA::AMX) { + // AMX and VSX need to interleave K elements to pack as 32 bits + if constexpr (isa == ISA::AMX || isa == ISA::VSX) { vec_op::interleave_save(output_vec_0, output_vec_1, curr_weight); } else { output_vec_0.save(curr_weight); @@ -303,6 +306,8 @@ void cpu_gemm_wna16( return ISA::VEC; } else if (isa_hint == "rvv") { return ISA::RVV; + } else if (isa_hint == "vsx") { + return ISA::VSX; } else { TORCH_CHECK(false, "unsupported isa hint: " + isa_hint); } @@ -378,6 +383,30 @@ void cpu_gemm_wna16( scales_group_stride, zeros_group_stride, group_size, pack_factor); return; } + } else if (isa == ISA::VSX) { + TORCH_CHECK(input.scalar_type() == at::ScalarType::BFloat16, + "VSX WNA16 GEMM only supports BFloat16"); + using gemm_t = cpu_micro_gemm::MicroGemm; + if (has_zp) { + using dequantizer_t = Dequantizer4b; + cpu_gemm_wna16_impl( + input.data_ptr(), q_weight.data_ptr(), + output.data_ptr(), scales.data_ptr(), zeros_ptr, + bias.has_value() ? bias->data_ptr() : nullptr, a_m_size, + b_n_size, a_k_size, a_m_stride, output_m_stride, + scales_group_stride, zeros_group_stride, group_size, pack_factor); + return; + } + { + using dequantizer_t = Dequantizer4b; + cpu_gemm_wna16_impl( + input.data_ptr(), q_weight.data_ptr(), + output.data_ptr(), scales.data_ptr(), zeros_ptr, + bias.has_value() ? bias->data_ptr() : nullptr, a_m_size, + b_n_size, a_k_size, a_m_stride, output_m_stride, + scales_group_stride, zeros_group_stride, group_size, pack_factor); + return; + } } }); } diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 918daf5a8b..6bbb06efa6 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -902,7 +902,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { ops.def("dynamic_per_token_scaled_fp8_quant() -> ()", placeholder_op); // WNA16 -#if defined(__AVX512F__) || defined(__riscv_v) || defined(__s390x__) +#if defined(__AVX512F__) || defined(__riscv_v) || defined(__s390x__) || \ + defined(__powerpc__) ops.def( "cpu_gemm_wna16(Tensor input, Tensor q_weight, Tensor(a2!) output, " "Tensor scales, Tensor? zeros, Tensor? bias, SymInt " diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 53b8eb596d..20834529b1 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -312,8 +312,8 @@ vLLM CPU supports data parallel (DP), tensor parallel (TP) and pipeline parallel ### Which quantization configs does vLLM CPU support? - vLLM CPU supports quantizations: - - AWQ (x86, s390x) - - GPTQ (x86, s390x) + - AWQ (x86, s390x, Power) + - GPTQ (x86, s390x, Power) - compressed-tensor INT8 W8A8 (x86, s390x only) ### Why do I see `get_mempolicy: Operation not permitted` when running in Docker? diff --git a/tests/quantization/test_cpu_wna16.py b/tests/quantization/test_cpu_wna16.py index 2af633be6b..d15d4b8147 100644 --- a/tests/quantization/test_cpu_wna16.py +++ b/tests/quantization/test_cpu_wna16.py @@ -3,7 +3,7 @@ import pytest import torch -from vllm.platforms import current_platform +from vllm.platforms import CpuArchEnum, current_platform if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True) @@ -54,7 +54,7 @@ def test_cpu_gemm_wna16_3d_input(): MODELS = [ "TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ", - "Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", + "Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4", # without g_idx "RedHatAI/Qwen3-1.7B-quantized.w4a16", # with zp "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc", "Qwen/Qwen3-0.6B-FP8", # FP8 W8A16 block-quantized linear @@ -67,6 +67,16 @@ MODELS = [ DTYPE = ["bfloat16"] +@pytest.mark.skipif( + current_platform.get_cpu_architecture() != CpuArchEnum.POWERPC, + reason="PowerPC-only test", +) +def test_cpu_wna16_power_bf16_uses_vsx(): + """Use the POWER10 BF16 MMA microkernel instead of generic vectors.""" + assert _get_isa_hint(torch.bfloat16) == "vsx" + assert _get_isa_hint(torch.float16) == "vec" + + @pytest.mark.parametrize("model", MODELS) @pytest.mark.parametrize("dtype", DTYPE) def test_cpu_quant(vllm_runner, model, dtype): diff --git a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py index 19f5283acb..a9c13051e2 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/cpu.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/cpu.py @@ -213,5 +213,10 @@ def _get_isa_hint(dtype: torch.dtype) -> str: return "amx" elif current_platform.get_cpu_architecture() == CpuArchEnum.RISCV: return "rvv" + elif ( + dtype == torch.bfloat16 + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + ): + return "vsx" else: return "vec" From 3d5625cc3c2474994107047d7a6ecb8e4008f9b0 Mon Sep 17 00:00:00 2001 From: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Date: Wed, 30 Sep 2026 08:16:26 -0400 Subject: [PATCH 113/248] [Security] Authenticate shared-memory multimodal cache handles (#59357) Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Cyrus Leung --- tests/distributed/test_shm_buffer.py | 6 +- tests/distributed/test_shm_storage.py | 69 +++++- .../test_generate_stream.py | 53 ++++- tests/multimodal/test_cache.py | 208 ++++++++++++++++++ .../shm_object_storage.py | 166 ++++++++++++-- .../scale_out/token_in_token_out/serving.py | 20 ++ vllm/multimodal/cache/base.py | 8 + vllm/multimodal/cache/shm.py | 85 ++++++- 8 files changed, 574 insertions(+), 41 deletions(-) diff --git a/tests/distributed/test_shm_buffer.py b/tests/distributed/test_shm_buffer.py index 6f61c28322..7a919124db 100644 --- a/tests/distributed/test_shm_buffer.py +++ b/tests/distributed/test_shm_buffer.py @@ -108,10 +108,10 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase): # Clear the buffer self.ring_buffer.clear() - # Check that metadata is empty and IDs reset + # Check that metadata is empty and IDs keep increasing self.assertEqual(len(self.ring_buffer.metadata), 0) - self.assertEqual(self.ring_buffer.monotonic_id_start, 0) - self.assertEqual(self.ring_buffer.monotonic_id_end, 0) + self.assertEqual(self.ring_buffer.monotonic_id_start, 3) + self.assertEqual(self.ring_buffer.monotonic_id_end, 3) self.assertEqual(self.ring_buffer.data_buffer_start, 0) self.assertEqual(self.ring_buffer.data_buffer_end, 0) diff --git a/tests/distributed/test_shm_storage.py b/tests/distributed/test_shm_storage.py index fb7d5528c0..762ffad8e6 100644 --- a/tests/distributed/test_shm_storage.py +++ b/tests/distributed/test_shm_storage.py @@ -227,10 +227,68 @@ class TestSingleWriterShmObjectStorage(unittest.TestCase): self.assertEqual(len(self.storage.id_index), 0) self.assertEqual(len(self.storage.ring_buffer.metadata), 0) - # Verify that new items can be added after clearing + # Verify that new items can be added after clearing. IDs keep + # increasing so handles issued before the clear cannot alias new data. address, monotonic_id = self.storage.put("new_item", "new_value") self.assertIn("new_item", self.storage.key_index) - self.assertEqual((address, monotonic_id), (0, 0)) + self.assertEqual((address, monotonic_id), (0, 5)) + + def _open_reader(self) -> SingleWriterShmObjectStorage: + reader = SingleWriterShmObjectStorage.create_from_handle(self.storage.handle()) + self.addCleanup(reader.close) + return reader + + def test_reader_get_and_touch_each_require_valid_signature(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + reader = self._open_reader() + + reader.touch("key", address, monotonic_id, signature) + self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value") + + flipped = [signature[0] ^ 1, *signature[1:]] + for bad in (None, flipped, [signature], signature[:-1], [256] * 32): + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.get(address, monotonic_id, bad, "key") + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.touch("key", address, monotonic_id, bad) + + def test_signature_is_bound_to_key_and_address(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + reader = self._open_reader() + + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.get(address, monotonic_id, signature, "other_key") + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + reader.get(address + 1, monotonic_id, signature, "key") + + def test_writer_rejects_handle_no_longer_in_key_index(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + self.storage.verify_signature("key", address, monotonic_id, signature) + + self.storage.clear() + + with self.assertRaisesRegex(ValueError, "SHM handle signature"): + self.storage.verify_signature("key", address, monotonic_id, signature) + + def test_handle_issued_before_clear_reads_own_data_until_reused(self): + address, monotonic_id = self.storage.put("key", "value") + signature = self.storage.get_signature("key") + reader = self._open_reader() + + self.storage.clear() + + # Requests still draining after a clear keep reading their own data. + self.assertEqual(reader.get(address, monotonic_id, signature, "key"), "value") + + self.storage.put("key", "new_value") + + with self.assertRaisesRegex(ValueError, "modified or is invalid"): + reader.get(address, monotonic_id, signature, "key") + with self.assertRaisesRegex(ValueError, "modified or is invalid"): + reader.touch("key", address, monotonic_id, signature) # Reader process function @@ -242,11 +300,11 @@ def reader_process(process_id, storage_handle, items_to_read): errors = [] - for key, original_value, address, monotonic_id in items_to_read: + for key, original_value, address, monotonic_id, signature in items_to_read: time.sleep(random.random() / 100) try: # Read data from shared memory - retrieved_value = reader_storage.get(address, monotonic_id) + retrieved_value = reader_storage.get(address, monotonic_id, signature, key) # Verify data integrity assert retrieved_value == original_value @@ -289,7 +347,8 @@ def run_multiprocess_example(): for key, value in test_data: print(f"Storing {key}: {value}") address, monotonic_id = storage.put(key, value) - stored_items.append((key, value, address, monotonic_id)) + signature = storage.get_signature(key) + stored_items.append((key, value, address, monotonic_id, signature)) print(f" -> Stored at address {address}, ID {monotonic_id}") print("\n--- Retrieving Data ---") diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index f7c6f05e7f..28f8a505f8 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -15,14 +15,25 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.scale_out.token_in_token_out.mm_features import ( placeholder_ranges_from_engine_input, ) +from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import ( + encode_mm_kwargs_item, +) from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, GenerateResponse, + MultiModalFeatures, + PlaceholderRangeInfo, ) from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens +from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import GenerationError from vllm.logprobs import Logprob -from vllm.multimodal.inputs import PlaceholderRange +from vllm.multimodal.inputs import ( + MultiModalBatchedField, + MultiModalFieldElem, + MultiModalKwargsItem, + PlaceholderRange, +) from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers import renderer_from_config from vllm.renderers.online_renderer import OnlineRenderer @@ -285,6 +296,46 @@ async def test_non_stream_error(): await serving.serve_tokens(request) +@pytest.mark.asyncio +async def test_serve_tokens_rejects_invalid_shm_handle_before_engine_handoff(): + engine = _mock_engine() + engine.generate = MagicMock() + serving = _build_serving_tokens(engine) + cache = MagicMock() + cache.validate_input_item.side_effect = ValueError( + "Invalid SHM handle signature for cache key." + ) + serving.online_renderer.renderer._mm_processor_cache = cache + forged_item = MultiModalKwargsItem( + { + "address": MultiModalFieldElem(data=224, field=MultiModalBatchedField()), + "monotonic_id": MultiModalFieldElem( + data=4242, + field=MultiModalBatchedField(), + ), + } + ) + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=1), + model=MODEL_NAME, + features=MultiModalFeatures( + mm_hashes={"image": ["image_A"]}, + mm_placeholders={"image": [PlaceholderRangeInfo(offset=0, length=1)]}, + kwargs_data={"image": [encode_mm_kwargs_item(forged_item)]}, + ), + ) + + response = await serving.serve_tokens(request) + + assert isinstance(response, ErrorResponse) + cache.validate_input_item.assert_called_once() + validated_item, validated_hash = cache.validate_input_item.call_args.args + assert validated_hash == "image_A" + assert validated_item["address"].data == 224 + engine.generate.assert_not_called() + + @pytest.mark.asyncio async def test_stream_basic(): """Streaming returns SSE chunks with correct token_ids and ends with [DONE].""" diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index a4966565f6..0416734c24 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -29,6 +29,7 @@ from vllm.multimodal.cache.base import ( ) from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.inputs import ( + MultiModalBatchedField, MultiModalFeatureSpec, MultiModalFieldElem, MultiModalKwargsItem, @@ -42,6 +43,7 @@ from vllm.utils.async_utils import make_async from vllm.utils.mem_constants import GiB_bytes, MiB_bytes from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.llm_engine import LLMEngine +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder from ..models.utils import build_model_context from ..utils import create_new_process_for_each_test @@ -778,6 +780,212 @@ def test_cache_eviction_shm_cache(): _run_test_cache_eviction_shm(sender_cache, receiver_cache, base_item_size=MiB_bytes) +def test_shm_receiver_requires_sender_handle_for_current_cache_key(): + vllm_config = VllmConfig( + model_config=ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + mm_processor_cache_type="shm", + mm_shm_cache_max_object_size_mb=1, + mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, + ), + ) + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(1024) + + try: + sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + + valid_feature = MultiModalFeatureSpec( + data=sender_item, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + receiver_cache.get_and_update_features([valid_feature]) + assert valid_feature.data == item + + forged_item = MultiModalKwargsItem( + { + "address": sender_item["address"], + "monotonic_id": sender_item["monotonic_id"], + } + ) + forged_feature = MultiModalFeatureSpec( + data=forged_item, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + with pytest.raises(ValueError, match="SHM handle signature"): + receiver_cache.get_and_update_features([forged_feature]) + + wrong_key = "image_B" + wrong_key_feature = MultiModalFeatureSpec( + data=sender_item, + modality="image", + identifier=wrong_key, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=wrong_key, + ) + with pytest.raises(ValueError, match="SHM handle signature"): + receiver_cache.get_and_update_features([wrong_key_feature]) + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + +def test_shm_sender_handle_survives_engine_msgpack_round_trip(): + vllm_config = VllmConfig( + model_config=ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + mm_processor_cache_type="shm", + mm_shm_cache_max_object_size_mb=1, + mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, + ), + ) + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(1024) + + try: + sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + encoded = MsgpackEncoder().encode(sender_item) + decoded = MsgpackDecoder(MultiModalKwargsItem).decode(encoded) + feature = MultiModalFeatureSpec( + data=decoded, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + + receiver_cache.get_and_update_features([feature]) + + assert feature.data == item + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + +def _small_shm_vllm_config() -> VllmConfig: + return VllmConfig( + model_config=ModelConfig( + model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", + mm_processor_cache_type="shm", + mm_shm_cache_max_object_size_mb=1, + mm_processor_cache_gb=2 * MiB_bytes / GiB_bytes, + ), + ) + + +def _with_elem(item: MultiModalKwargsItem, key: str, data) -> MultiModalKwargsItem: + elems = dict(item) + elems[key] = MultiModalFieldElem(data=data, field=MultiModalBatchedField()) + return MultiModalKwargsItem(elems) + + +def test_shm_sender_validates_client_supplied_handles(): + sender_cache = ShmObjectStoreSenderCache(_small_shm_vllm_config()) + mm_hash = "image_A" + + try: + handle, _ = sender_cache.get_and_update_item( + (MultiModalKwargsItem.dummy(1024), []), mm_hash + ) + sender_cache.validate_input_item(handle, mm_hash) + sender_cache.validate_input_item(MultiModalKwargsItem.dummy(16), mm_hash) + + signature = handle["signature"].data + forged = [ + _with_elem(handle, "signature", [signature[0] ^ 1, *signature[1:]]), + _with_elem(handle, "signature", [signature]), + _with_elem(handle, "address", True), + _with_elem(handle, "address", -1), + _with_elem(handle, "monotonic_id", 2**32), + MultiModalKwargsItem( + {"address": handle["address"], "monotonic_id": handle["monotonic_id"]} + ), + ] + for item in forged: + with pytest.raises(ValueError, match="SHM handle signature"): + sender_cache.validate_input_item(item, mm_hash) + with pytest.raises(ValueError, match="SHM handle signature"): + sender_cache.validate_input_item(handle, "image_B") + + sender_cache.clear_cache() + with pytest.raises(ValueError, match="SHM handle signature"): + sender_cache.validate_input_item(handle, mm_hash) + finally: + sender_cache.close() + + +def test_shm_receiver_rejects_out_of_range_handle_fields(): + vllm_config = _small_shm_vllm_config() + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + + try: + handle, _ = sender_cache.get_and_update_item( + (MultiModalKwargsItem.dummy(1024), []), mm_hash + ) + for key, value in [ + ("address", -1), + ("address", 2**64), + ("monotonic_id", -1), + ("monotonic_id", 2**32), + ]: + with pytest.raises(ValueError, match="SHM handle signature"): + receiver_cache.get_and_update_item( + _with_elem(handle, key, value), mm_hash + ) + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + +def test_shm_handle_issued_before_cache_clear(): + vllm_config = _small_shm_vllm_config() + sender_cache = ShmObjectStoreSenderCache(vllm_config) + receiver_cache = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) + mm_hash = "image_A" + item = MultiModalKwargsItem.dummy(1024) + + def feature(data: MultiModalKwargsItem) -> MultiModalFeatureSpec: + return MultiModalFeatureSpec( + data=data, + modality="image", + identifier=mm_hash, + mm_position=PlaceholderRange(offset=0, length=1), + mm_hash=mm_hash, + ) + + try: + stale_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + sender_cache.clear_cache() + + # A request still draining after the clear resolves to its own data. + draining = feature(stale_item) + receiver_cache.get_and_update_features([draining]) + assert draining.data == item + + sender_cache.get_and_update_item( + (MultiModalKwargsItem.dummy(2048), []), + mm_hash, + ) + + with pytest.raises(ValueError, match="modified or is invalid"): + receiver_cache.get_and_update_features([feature(stale_item)]) + finally: + receiver_cache._shm_cache.close() + sender_cache.close() + + def test_processor_cache_shared_across_loras(): """Test that processor cache uses mm_hash to share data across LoRAs.""" model_config = ModelConfig( diff --git a/vllm/distributed/device_communicators/shm_object_storage.py b/vllm/distributed/device_communicators/shm_object_storage.py index bc4f1b6e51..3da8c5dcf6 100644 --- a/vllm/distributed/device_communicators/shm_object_storage.py +++ b/vllm/distributed/device_communicators/shm_object_storage.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import hashlib +import hmac import pickle +import secrets from abc import ABC, abstractmethod from collections.abc import Callable, Iterable from contextlib import contextmanager, suppress @@ -97,6 +100,9 @@ class SingleWriterShmRingBuffer: - Reader synchronization handled by is_free_fn callback - Writer handles garbage collection (free_buf) based on reader feedback + Shared Memory Layout: + `[32-byte handle-signing secret][ring buffer chunks...]` + Memory Layout per Buffer Chunk: `[4-byte monotonic_id][4-byte chunk_size][actual_data...]` ^metadata_start ^data_start @@ -119,6 +125,8 @@ class SingleWriterShmRingBuffer: self.SIZE_NBYTES = 4 # 4 bytes for id, 4 bytes for buffer size self.MD_SIZE = self.ID_NBYTES + self.SIZE_NBYTES + self.AUTH_SECRET_BYTES = 32 + self.DATA_OFFSET = self.AUTH_SECRET_BYTES self.monotonic_id_end = 0 self.monotonic_id_start = 0 self.data_buffer_start = 0 @@ -129,8 +137,11 @@ class SingleWriterShmRingBuffer: # we are creating a buffer self.metadata: dict[int, int] = {} # monotonic_id -> start address self.shared_memory = shared_memory.SharedMemory( - create=True, size=self.data_buffer_size, name=name + create=True, + size=self.data_buffer_size + self.DATA_OFFSET, + name=name, ) + self._init_auth_secret() else: # we are opening an existing buffer # fix to https://stackoverflow.com/q/62748654/9191338 @@ -146,7 +157,9 @@ class SingleWriterShmRingBuffer: # so the shared memory block size may be larger or equal # to the requested size. The size parameter is ignored # when attaching to an existing block. - assert self.shared_memory.size >= self.data_buffer_size + assert ( + self.shared_memory.size >= self.data_buffer_size + self.DATA_OFFSET + ) logger.debug( "Shared memory created/opened with name: %s, size: %d", @@ -164,11 +177,25 @@ class SingleWriterShmRingBuffer: """Clear the ring buffer.""" assert self.is_writer, "Only the writer can clear the buffer." self.metadata.clear() - self.monotonic_id_end = 0 - self.monotonic_id_start = 0 + # Keep IDs increasing so handles issued before the clear (e.g. for + # requests still draining) fail the ID check once their chunk is + # reused, instead of resolving to a different object. + self.monotonic_id_start = self.monotonic_id_end self.data_buffer_start = 0 self.data_buffer_end = 0 + def get_auth_secret(self) -> bytes: + """Return the current handle-signing secret.""" + assert self.shared_memory.buf is not None, "Buffer has been closed" + return bytes(self.shared_memory.buf[: self.AUTH_SECRET_BYTES]) + + def _init_auth_secret(self) -> None: + assert self.is_writer, "Only the writer can set the auth secret." + assert self.shared_memory.buf is not None, "Buffer has been closed" + self.shared_memory.buf[: self.AUTH_SECRET_BYTES] = secrets.token_bytes( + self.AUTH_SECRET_BYTES + ) + def close(self) -> None: """Close the shared memory.""" if hasattr(self, "shared_memory"): @@ -219,13 +246,14 @@ class SingleWriterShmRingBuffer: # first 4 bytes as the monotonic id buf_idx = self.data_buffer_end % self.data_buffer_size - self.shared_memory.buf[buf_idx : buf_idx + self.ID_NBYTES] = self.int2byte( - self.monotonic_id_end + physical_idx = self.DATA_OFFSET + buf_idx + self.shared_memory.buf[physical_idx : physical_idx + self.ID_NBYTES] = ( + self.int2byte(self.monotonic_id_end) ) # next 4 bytes as the size of the data buffer - self.shared_memory.buf[buf_idx + self.ID_NBYTES : buf_idx + self.MD_SIZE] = ( - self.int2byte(size) - ) + self.shared_memory.buf[ + physical_idx + self.ID_NBYTES : physical_idx + self.MD_SIZE + ] = self.int2byte(size) # record metadata self.metadata[self.monotonic_id_end % self.ID_MAX] = self.data_buffer_end @@ -240,14 +268,19 @@ class SingleWriterShmRingBuffer: def access_buf(self, address: int): assert self.shared_memory.buf is not None, "Buffer has been closed" buf_idx = address % self.data_buffer_size + physical_idx = self.DATA_OFFSET + buf_idx # read metadata - metadata_buff = self.shared_memory.buf[buf_idx : buf_idx + self.MD_SIZE] + metadata_buff = self.shared_memory.buf[ + physical_idx : physical_idx + self.MD_SIZE + ] id = self.byte2int(metadata_buff[: self.ID_NBYTES]) size = self.byte2int(metadata_buff[self.ID_NBYTES : self.MD_SIZE]) # yield the data buffer and metadata - data_buff = self.shared_memory.buf[buf_idx + self.MD_SIZE : buf_idx + size] + data_buff = self.shared_memory.buf[ + physical_idx + self.MD_SIZE : physical_idx + size + ] with ( memoryview(data_buff) as data_view, ): @@ -423,7 +456,8 @@ class SingleWriterShmObjectStorage: Architecture: - Single writer process can put(key, value) objects - - Multiple reader processes can get(address, monotonic_id) objects + - Multiple reader processes can get(address, monotonic_id, signature, key) + objects - Built on SingleWriterShmRingBuffer for efficient shared memory management - Thread-safe operations with reader synchronization via locks @@ -439,6 +473,13 @@ class SingleWriterShmObjectStorage: - Automatic Cleanup: Garbage collection happens transparently during allocation + Handle Signatures: + - Handles are signed with an HMAC of (key, address, monotonic_id), keyed by + a random secret stored at the start of the shared memory + - The writer signs the handles it issues and rejects stale or forged + handles supplied by requests + - Readers verify the signature in get/touch before reading shared memory + Memory Layout per Object: `[4-byte reference_count][metadata_size][serialized_object_data]` @@ -521,6 +562,76 @@ class SingleWriterShmObjectStorage: else: raise ValueError(f"Unsupported data type for serialization: {type(data)}") + def _validate_monotonic_id( + self, + address: int, + monotonic_id: int, + buf_metadata: tuple[int, int], + ) -> None: + if buf_metadata[0] != monotonic_id: + raise ValueError( + f"Data for address:id '{address}:{monotonic_id}'" + " has been modified or is invalid." + ) + + def _verify_reader_signature( + self, + address: int, + monotonic_id: int, + signature: list[int] | None, + key: str | None, + ) -> None: + if signature is None or key is None: + raise ValueError("Missing SHM handle signature for cache key.") + if ( + not isinstance(signature, list) + or len(signature) != hashlib.sha256().digest_size + or any(type(byte) is not int or not 0 <= byte <= 255 for byte in signature) + or not isinstance(key, str) + ): + raise ValueError("Invalid SHM handle signature for cache key.") + + expected_signature = self._make_signature(key, address, monotonic_id) + if not hmac.compare_digest(bytes(signature), expected_signature): + raise ValueError("Invalid SHM handle signature for cache key.") + + def verify_signature( + self, + key: str | None, + address: int, + monotonic_id: int, + signature: list[int] | None, + ) -> None: + """Verify that a handle was issued for the given key. + + For writers: the handle must also be the key's current entry + For readers: only the signature is checked + + Args: + key: String key the handle belongs to + address: Address of the object + monotonic_id: Monotonic ID of the object + signature: Signature issued with the handle + + """ + if self.is_writer and ( + key is None or self.key_index.get(key) != (address, monotonic_id) + ): + raise ValueError("Invalid SHM handle signature for cache key.") + self._verify_reader_signature(address, monotonic_id, signature, key) + + def _make_signature(self, key: str, address: int, monotonic_id: int) -> bytes: + payload = ( + hashlib.sha256(key.encode("utf-8")).digest() + + address.to_bytes(8, "little", signed=False) + + monotonic_id.to_bytes(4, "little", signed=False) + ) + return hmac.new( + self.ring_buffer.get_auth_secret(), + payload, + hashlib.sha256, + ).digest() + def increment_writer_flag(self, id: int) -> None: """Set the in-use flag for the writer.""" self.writer_flag[id] = self.writer_flag.get(id, 0) + 1 @@ -555,6 +666,11 @@ class SingleWriterShmObjectStorage: self.increment_writer_flag(monotonic_id) return address, monotonic_id + def get_signature(self, key: str) -> list[int]: + """Sign the handle of a cached object so readers can verify it.""" + address, monotonic_id = self.key_index[key] + return list(self._make_signature(key, address, monotonic_id)) + def put(self, key: str, value: Any) -> tuple[int, int]: """Store a key-value pair in the object storage. Attempts to free max_object_size bytes using FIFO order @@ -605,15 +721,20 @@ class SingleWriterShmObjectStorage: self.id_index[monotonic_id] = key return address, monotonic_id - def get(self, address: int, monotonic_id: int) -> Any: + def get( + self, + address: int, + monotonic_id: int, + signature: list[int] | None = None, + key: str | None = None, + ) -> Any: + if not self.is_writer: + # Reject forged handles before dereferencing the supplied address. + self.verify_signature(key, address, monotonic_id, signature) + # Read data from buffer with self.ring_buffer.access_buf(address) as (data_view, buf_metadata): - # check id from metadata - if buf_metadata[0] != monotonic_id: - raise ValueError( - f"Data for address:id '{address}:{monotonic_id}'" - " has been modified or is invalid." - ) + self._validate_monotonic_id(address, monotonic_id, buf_metadata) obj = self.ser_de.deserialize(data_view[self.flag_bytes :]) @@ -633,6 +754,7 @@ class SingleWriterShmObjectStorage: key: str, address: int = 0, monotonic_id: int = 0, + signature: list[int] | None = None, ) -> None: """Touch an existing cached item to update its eviction status. @@ -643,6 +765,7 @@ class SingleWriterShmObjectStorage: key: String key of the object to touch address: Address of the object (only for readers) monotonic_id: Monotonic ID of the object (only for readers) + signature: Server-issued handle signature (only for readers) """ if self._reader_lock is None: @@ -652,10 +775,13 @@ class SingleWriterShmObjectStorage: # Writer side: increment writer_flag to raise eviction threshold self.increment_writer_flag(monotonic_id) else: + # Reject forged handles before dereferencing the supplied address. + self.verify_signature(key, address, monotonic_id, signature) with ( self._reader_lock, - self.ring_buffer.access_buf(address) as (data_view, _), + self.ring_buffer.access_buf(address) as (data_view, buf_metadata), ): + self._validate_monotonic_id(address, monotonic_id, buf_metadata) reader_count = self.ring_buffer.byte2int(data_view[: self.flag_bytes]) # NOTE(Long): diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 8b0fbff99b..ade103bbbc 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -39,6 +39,7 @@ from vllm.inputs import EngineInput, TokensPrompt, mm_input from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.multimodal.inputs import ( + MultiModalKwargsItem, MultiModalKwargsItems, PlaceholderRange, ) @@ -106,6 +107,23 @@ class ServingTokens(GenerateBaseServing): else getattr(mc, "override_generation_config", {}).get("max_new_tokens") ) + def _validate_mm_cache_handles( + self, + mm_kwargs: dict[str, list[MultiModalKwargsItem | None]], + mm_hashes: dict[str, list[str]], + ) -> ErrorResponse | None: + cache = self.online_renderer.renderer.mm_processor_cache + if cache is None: + return None + try: + for modality, items in mm_kwargs.items(): + for mm_hash, item in zip(mm_hashes[modality], items, strict=True): + if item is not None: + cache.validate_input_item(item, mm_hash) + except ValueError as error: + return self.create_error_response(error) + return None + async def serve_tokens( self, request: GenerateRequest, @@ -201,6 +219,8 @@ class ServingTokens(GenerateBaseServing): # Deserialize full tensor data and optional metadata-only data. # Metadata-only items are valid when ec_transfer_params is set. mm_kwargs = mm_kwargs_from_features(features) + if error := self._validate_mm_cache_handles(mm_kwargs, features.mm_hashes): + return error engine_input = mm_input( prompt_token_ids=request.token_ids, diff --git a/vllm/multimodal/cache/base.py b/vllm/multimodal/cache/base.py index 1b9b757917..d2aaeb50e3 100644 --- a/vllm/multimodal/cache/base.py +++ b/vllm/multimodal/cache/base.py @@ -356,6 +356,14 @@ class BaseMultiModalProcessorCache( """Close the underlying cache, if needed.""" pass + def validate_input_item( + self, + mm_item: MultiModalKwargsItem, + mm_hash: str, + ) -> None: + """Validate externally supplied cache metadata before engine handoff.""" + return None + @abstractmethod def touch_sender_cache_item(self, mm_hash: str) -> None: """Update the cache eviction order for a multi-modal item. diff --git a/vllm/multimodal/cache/shm.py b/vllm/multimodal/cache/shm.py index dddf3494cf..ab84dcf6af 100644 --- a/vllm/multimodal/cache/shm.py +++ b/vllm/multimodal/cache/shm.py @@ -4,7 +4,7 @@ from collections.abc import Sequence from multiprocessing.synchronize import Lock as LockType -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, TypeAlias from typing_extensions import override @@ -37,6 +37,29 @@ if TYPE_CHECKING: logger = init_logger(__name__) +ShmHandle: TypeAlias = tuple[int, int, list[int]] + + +def _get_shm_handle(mm_item: MultiModalKwargsItem) -> ShmHandle | None: + if "address" not in mm_item: + return None + if "monotonic_id" not in mm_item or "signature" not in mm_item: + raise ValueError("Missing SHM handle signature for cache key.") + + address = mm_item["address"].data + monotonic_id = mm_item["monotonic_id"].data + signature = mm_item["signature"].data + if ( + type(address) is not int + or not 0 <= address < 2**64 + or type(monotonic_id) is not int + or not 0 <= monotonic_id < 2**32 + or not isinstance(signature, list) + ): + raise ValueError("Invalid SHM handle signature for cache key.") + + return address, monotonic_id, signature + class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): """The cache which is used on P0 when SHM caching is enabled. @@ -98,8 +121,12 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): self._total += 1 address, monotonic_id = self._shm_cache.get_cached(mm_hash) + signature = self._shm_cache.get_signature(mm_hash) prompt_updates = self._p0_cache[mm_hash] - return self.address_as_item(address, monotonic_id), prompt_updates + return ( + self.address_as_item(address, monotonic_id, signature), + prompt_updates, + ) assert mm_item is not None, f"Expected a cached item for {mm_hash=}" item, prompt_updates = mm_item @@ -108,12 +135,16 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): try: address, monotonic_id = self._shm_cache.put(mm_hash, item) + signature = self._shm_cache.get_signature(mm_hash) # Try to remove dangling items if p0 cache is too large. if len(self._p0_cache) >= 2 * len(self._shm_cache.key_index): self.remove_dangling_items() self._p0_cache[mm_hash] = prompt_updates - return self.address_as_item(address, monotonic_id), prompt_updates + return ( + self.address_as_item(address, monotonic_id, signature), + prompt_updates, + ) except ValueError as e: # `put` raises ValueError either for an oversize item or for a # duplicate key (concurrent insert); the latter is benign so we @@ -143,6 +174,21 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): Increments writer_flag on sender side.""" self._shm_cache.touch(mm_hash) + @override + def validate_input_item( + self, + mm_item: MultiModalKwargsItem, + mm_hash: str, + ) -> None: + if (handle := _get_shm_handle(mm_item)) is not None: + address, monotonic_id, signature = handle + self._shm_cache.verify_signature( + mm_hash, + address, + monotonic_id, + signature, + ) + @override def clear_cache(self) -> None: self._shm_cache.clear() @@ -171,6 +217,7 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): self, address: int, monotonic_id: int, + signature: list[int], ) -> MultiModalKwargsItem: addr_elem = MultiModalFieldElem( data=address, @@ -181,7 +228,18 @@ class ShmObjectStoreSenderCache(BaseMultiModalProcessorCache): field=MultiModalBatchedField(), ) - return MultiModalKwargsItem({"address": addr_elem, "monotonic_id": id_elem}) + signature_elem = MultiModalFieldElem( + data=signature, + field=MultiModalBatchedField(), + ) + + return MultiModalKwargsItem( + { + "address": addr_elem, + "monotonic_id": id_elem, + "signature": signature_elem, + } + ) class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): @@ -235,10 +293,9 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): mm_hash: str, ) -> MultiModalKwargsItem: assert mm_item is not None, f"Expected an address item for {mm_hash=}" - if "address" in mm_item: - address = cast(int, mm_item["address"].data) - monotonic_id = cast(int, mm_item["monotonic_id"].data) - return self._shm_cache.get(address, monotonic_id) + if (handle := _get_shm_handle(mm_item)) is not None: + address, monotonic_id, signature = handle + return self._shm_cache.get(address, monotonic_id, signature, mm_hash) return mm_item @@ -251,10 +308,14 @@ class ShmObjectStoreReceiverCache(BaseMultiModalReceiverCache): """Touch the item in shared memory cache to prevent eviction. Increments reader_count on receiver side.""" assert mm_item is not None - if "address" in mm_item: - address = cast(int, mm_item["address"].data) - monotonic_id = cast(int, mm_item["monotonic_id"].data) - self._shm_cache.touch(mm_hash, address=address, monotonic_id=monotonic_id) + if (handle := _get_shm_handle(mm_item)) is not None: + address, monotonic_id, signature = handle + self._shm_cache.touch( + mm_hash, + address=address, + monotonic_id=monotonic_id, + signature=signature, + ) @override def clear_cache(self) -> None: From a4c4bb418e051e57b4a87084d757d5710cc66129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20P=C3=A9rez=20de=20Algaba?= <124347725+jperezdealgaba@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:16:31 +0200 Subject: [PATCH 114/248] [Security] Bump remaining Dependabot packages (excl. ignored) (#59315) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Pérez de Algaba Co-authored-by: Cursor --- requirements/build/cpu.txt | 2 +- requirements/build/cuda.txt | 2 +- requirements/build/rock.txt | 2 +- requirements/build/rocm.txt | 2 +- requirements/build/rust.txt | 2 +- requirements/build/tpu.txt | 2 +- requirements/common.txt | 6 +-- requirements/cpu.txt | 2 +- requirements/rocm.txt | 2 +- requirements/test/cpu.txt | 100 ++++++++++++++++++++++++------------ requirements/test/cuda.in | 24 +++++++++ requirements/test/cuda.txt | 100 ++++++++++++++++++++++++------------ requirements/test/rocm.in | 23 +++++++++ requirements/test/rocm.txt | 82 +++++++++++++++++++---------- requirements/test/xpu.in | 20 ++++++++ requirements/test/xpu.txt | 49 +++++++++++++++--- requirements/xpu.txt | 2 +- rust/Cargo.lock | 81 ++++++++++++++++++++--------- rust/Cargo.toml | 6 +-- 19 files changed, 372 insertions(+), 137 deletions(-) diff --git a/requirements/build/cpu.txt b/requirements/build/cpu.txt index 5a03960e56..7f2f4c7914 100644 --- a/requirements/build/cpu.txt +++ b/requirements/build/cpu.txt @@ -1,7 +1,7 @@ cmake>=3.26.1 ninja packaging>=24.2 -setuptools==77.0.3 # this version can reuse CMake build dir +setuptools==78.1.1 # this version can reuse CMake build dir; >=78.1.1 for GHSA-5rjg-fvgr-3xxf setuptools-scm>=8 setuptools-rust>=1.9.0 torch==2.13.0+cpu; platform_machine == "x86_64" or platform_machine == "s390x" or platform_machine == "aarch64" diff --git a/requirements/build/cuda.txt b/requirements/build/cuda.txt index d9d68022b3..3cc83287bc 100644 --- a/requirements/build/cuda.txt +++ b/requirements/build/cuda.txt @@ -2,7 +2,7 @@ cmake>=3.26.1 ninja packaging>=24.2 -setuptools>=77.0.3,<81.0.0 +setuptools>=78.1.1,<81.0.0 setuptools-scm>=8 setuptools-rust>=1.9.0 torch==2.13.0 diff --git a/requirements/build/rock.txt b/requirements/build/rock.txt index ae4697e612..5ca5ada2af 100644 --- a/requirements/build/rock.txt +++ b/requirements/build/rock.txt @@ -9,7 +9,7 @@ torchaudio==2.11.0+rocm7.14.0 triton==3.7.1+git0263a6a6.rocm7.14.0 cmake>=3.26.1,<4 packaging>=26.2 -setuptools>=77.0.3,<80.0.0 +setuptools>=78.1.1,<80.0.0 setuptools-scm>=8 setuptools-rust>=1.9.0 wheel diff --git a/requirements/build/rocm.txt b/requirements/build/rocm.txt index aedfa66ab0..3c9b6c1796 100644 --- a/requirements/build/rocm.txt +++ b/requirements/build/rocm.txt @@ -9,7 +9,7 @@ torchaudio==2.11.0+rocm7.14.0 triton==3.7.1+git0263a6a6 cmake>=3.26.1,<4 packaging>=26.2 -setuptools>=77.0.3,<80.0.0 +setuptools>=78.1.1,<80.0.0 setuptools-scm>=8 setuptools-rust>=1.9.0 wheel diff --git a/requirements/build/rust.txt b/requirements/build/rust.txt index f507deb803..93dba7b378 100644 --- a/requirements/build/rust.txt +++ b/requirements/build/rust.txt @@ -1,5 +1,5 @@ # Dependencies for building Rust artifacts through setuptools-rust. -setuptools>=77.0.3,<81.0.0 +setuptools>=78.1.1,<81.0.0 setuptools-scm>=9.2.0 setuptools-rust>=1.9.0 wheel diff --git a/requirements/build/tpu.txt b/requirements/build/tpu.txt index 1a791a633f..d9f03cb104 100644 --- a/requirements/build/tpu.txt +++ b/requirements/build/tpu.txt @@ -1,7 +1,7 @@ --extra-index-url https://download.pytorch.org/whl/cpu cmake>=3.26.1 ninja -setuptools>=77.0.3,<81.0.0 +setuptools>=78.1.1,<81.0.0 setuptools-scm>=8 setuptools-rust>=1.9.0 torch==2.13.0+cpu diff --git a/requirements/common.txt b/requirements/common.txt index 669efc4900..eec3c050bc 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -3,7 +3,7 @@ cachetools psutil sentencepiece # Required for LLaMA tokenizer. numpy -requests >= 2.26.0 +requests >= 2.33.0 tqdm blake3 py-cpuinfo @@ -28,7 +28,7 @@ outlines_core == 0.2.14 lark == 1.2.2 xgrammar == 0.2.8; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le" typing_extensions >= 4.10 -filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317 +filelock >= 3.20.3 # need to contain https://github.com/tox-dev/filelock/pull/317; GHSA floor partial-json-parser # used for parsing partial JSON outputs jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation pyzmq >= 25.0.0 @@ -37,7 +37,7 @@ mistral_common[image] >= 1.11.6 opencv-python-headless >= 4.13.0 # required for video IO pyyaml six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12 -setuptools>=77.0.3,<81.0.0; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12 +setuptools>=78.1.1,<81.0.0; python_version > '3.11' # used by triton; >=78.1.1 for GHSA-5rjg-fvgr-3xxf (83.x needed for GHSA-h35f-9h28-mq5c deferred) einops # Required for Qwen2-VL. compressed-tensors == 0.17.0 # required for compressed-tensors depyf==0.20.0 # required for profiling and debugging with compilation config diff --git a/requirements/cpu.txt b/requirements/cpu.txt index b4867a0ff3..191ed26ae0 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -1,7 +1,7 @@ # Common dependencies -r common.txt -setuptools==77.0.3 # this version can reuse CMake build dir +setuptools==78.1.1 # this version can reuse CMake build dir; >=78.1.1 for GHSA-5rjg-fvgr-3xxf numba == 0.65.0 # Required for N-gram speculative decoding diff --git a/requirements/rocm.txt b/requirements/rocm.txt index 85b196e401..cd7a114721 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -13,7 +13,7 @@ peft pytest-asyncio tensorizer==2.10.1 packaging>=24.2 -setuptools>=77.0.3,<80.0.0 +setuptools>=78.1.1,<80.0.0 setuptools-scm>=8 setuptools-rust>=1.9.0 runai-model-streamer[s3,gcs,azure]==0.15.7 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index df685200f3..11fc2991ed 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -33,8 +33,9 @@ annotated-types==0.7.0 # via pydantic anthropic==0.112.0 # via -r requirements/test/../common.txt -anyio==4.14.1 +anyio==4.14.2 # via + # -r requirements/test/cuda.in # anthropic # httpx # httpx2 @@ -71,8 +72,10 @@ azure-storage-blob==12.28.0 # via runai-model-streamer-azure backoff==2.2.1 # via -r requirements/test/cuda.in -black==24.10.0 - # via datamodel-code-generator +black==26.5.1 + # via + # -r requirements/test/cuda.in + # datamodel-code-generator blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.0.0 @@ -147,8 +150,9 @@ coverage==7.10.6 # via pytest-cov cramjam==2.9.0 # via fastparquet -cryptography==46.0.5 +cryptography==50.0.1 # via + # -r requirements/test/cuda.in # azure-identity # azure-storage-blob # msal @@ -239,17 +243,20 @@ fastrlock==0.8.2 # via cupy-cuda12x fastsafetensors==0.3.3 # via -r requirements/test/cuda.in -filelock==3.16.1 +filelock==4.0.7 # via # -r requirements/test/../common.txt # blobfile # datasets # huggingface-hub + # python-discovery # ray # torch # virtualenv -fonttools==4.55.0 - # via matplotlib +fonttools==4.66.1 + # via + # -r requirements/test/cuda.in + # matplotlib frozendict==2.4.6 # via einx frozenlist==1.5.0 @@ -314,16 +321,20 @@ h11==0.16.0 # httpcore # httpcore2 # uvicorn -h2==4.3.0 - # via httpx +h2==4.4.1 + # via + # -r requirements/test/cuda.in + # httpx harfile==0.5.0 # via schemathesis hf-xet==1.6.0 # via huggingface-hub hiredis==3.0.0 # via tensorizer -hpack==4.1.0 - # via h2 +hpack==4.2.0 + # via + # -r requirements/test/cuda.in + # h2 html2text==2025.4.15 # via gpt-oss httpcore==1.0.9 @@ -345,7 +356,9 @@ httpx==0.27.2 # perceptron # schemathesis httpx2==2.12.0 - # via mcp + # via + # -r requirements/test/cuda.in + # mcp huggingface-hub==1.31.0 # via # -r requirements/test/../common.txt @@ -468,13 +481,16 @@ lm-format-enforcer==0.11.3 # via -r requirements/test/../common.txt loguru==0.7.3 # via compressed-tensors -lxml==5.3.0 +lxml==6.1.3 # via + # -r requirements/test/cuda.in # blobfile # gpt-oss # sacrebleu -mako==1.3.10 - # via alembic +mako==1.4.3 + # via + # -r requirements/test/cuda.in + # alembic markdown-it-py==3.0.0 # via rich markupsafe==3.0.1 @@ -505,14 +521,15 @@ more-itertools==10.5.0 # via lm-eval mpmath==1.3.0 # via sympy -msal==1.34.0 +msal==1.39.0 # via # azure-identity # msal-extensions msal-extensions==1.3.1 # via azure-identity -msgpack==1.1.0 +msgpack==1.2.3 # via + # -r requirements/test/cuda.in # librosa # ray msgspec==0.21.1 @@ -649,8 +666,10 @@ opentelemetry-semantic-conventions-ai==0.4.13 # via -r requirements/test/../common.txt optuna==3.6.1 # via genai-perf -orjson==3.11.5 - # via genai-perf +orjson==3.12.0 + # via + # -r requirements/test/cuda.in + # genai-perf oss-harmony==0.0.11 # via -r requirements/test/../common.txt outlines-core==0.2.14 @@ -677,6 +696,7 @@ packaging==24.2 # statsmodels # transformers # typepy + # virtualenv pandas==2.2.3 # via # datasets @@ -687,7 +707,7 @@ pandas==2.2.3 # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt -pathspec==0.12.1 +pathspec==1.1.1 # via black pathvalidate==3.2.1 # via pytablewriter @@ -769,12 +789,14 @@ py-cpuinfo==9.0.0 # via -r requirements/test/../common.txt py-spy==0.4.0 # via ray -pyarrow==23.0.0 +pyarrow==25.0.1 # via + # -r requirements/test/cuda.in # datasets # genai-perf -pyasn1==0.6.1 +pyasn1==0.6.4 # via + # -r requirements/test/cuda.in # pyasn1-modules # rsa pyasn1-modules==0.4.2 @@ -821,13 +843,17 @@ pydantic-extra-types==2.10.5 # fastapi # mistral-common pydantic-settings==2.14.2 - # via fastapi -pygments==2.18.0 # via + # -r requirements/test/cuda.in + # fastapi +pygments==2.21.0 + # via + # -r requirements/test/cuda.in # pytest # rich -pyjwt==2.11.0 +pyjwt==2.15.1 # via + # -r requirements/test/cuda.in # mcp # msal pyparsing==3.2.0 @@ -872,6 +898,8 @@ python-dateutil==2.9.0.post0 # matplotlib # pandas # typepy +python-discovery==1.6.1 + # via virtualenv python-dotenv==1.2.2 # via # pydantic-settings @@ -880,10 +908,13 @@ python-json-logger==4.1.0 # via -r requirements/test/../common.txt python-multipart==0.0.32 # via + # -r requirements/test/cuda.in # fastapi # mcp python-rapidjson==1.20 # via tritonclient +pytokens==0.4.1 + # via black pytrec-eval-terrier==0.5.7 # via mteb pytz==2024.2 @@ -933,7 +964,7 @@ regex==2026.2.28 # sacrebleu # tiktoken # transformers -requests==2.32.3 +requests==2.34.2 # via # -r requirements/test/../common.txt # azure-core @@ -1036,7 +1067,7 @@ sentry-sdk==2.63.0 # via fastapi-cloud-cli setproctitle==1.3.7 # via -r requirements/test/../common.txt -setuptools==77.0.3 +setuptools==80.10.2 # via # -r requirements/test/../common.txt # model-hosting-container-standards @@ -1284,8 +1315,9 @@ typing-inspection==0.4.2 # pydantic-settings tzdata==2024.2 # via pandas -urllib3==2.2.3 +urllib3==2.8.0 # via + # -r requirements/test/cuda.in # blobfile # botocore # docker @@ -1305,8 +1337,10 @@ uvloop==0.22.1 # via uvicorn vector-quantize-pytorch==1.21.2 # via -r requirements/test/cuda.in -virtualenv==20.31.2 - # via ray +virtualenv==21.14.0 + # via + # -r requirements/test/cuda.in + # ray vocos==0.1.0 # via -r requirements/test/cuda.in watchfiles==1.2.0 @@ -1317,8 +1351,10 @@ wcwidth==0.2.13 # via ftfy websockets==16.0 # via uvicorn -werkzeug==3.1.3 - # via schemathesis +werkzeug==3.1.9 + # via + # -r requirements/test/cuda.in + # schemathesis word2number==1.1 # via lm-eval wrapt==1.17.2 diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 87d67fa9ff..d96bff8804 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -80,3 +80,27 @@ datasets>=3.3.0,<=3.6.0 openpyxl # required for perf comparison excel report plotly # required for perf comparison html report + +# Dependabot GHSA floors for transitive test deps (excl. dependabot.yml ignores: +# torch/torchvision/xformers/lm-format-enforcer/compressed-tensors/ray/lm-eval). +# Skipped (no patched release yet): sqlitedict, accelerate, lmcache, nltk GHSA-8mgp-746c-j5xp. +urllib3>=2.7.0 +cryptography>=50.0.0 +pyjwt>=2.13.0 +pyasn1>=0.6.4 +python-multipart>=0.0.31 +anyio>=4.14.2 +httpx2>=2.12.0 +werkzeug>=3.1.6 +mako>=1.3.12 +black>=26.3.1 +msgpack>=1.2.1 +lxml>=6.1.0 +pyarrow>=23.0.1 +orjson>=3.11.6 +hpack>=4.2.0 +h2>=4.4.1 +pydantic-settings>=2.14.2 +virtualenv>=20.36.1 +fonttools>=4.60.2 +pygments>=2.20.0 diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index a3ff938b55..cebfd726d5 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -36,8 +36,9 @@ anthropic==0.112.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -anyio==4.14.1 +anyio==4.14.2 # via + # -r requirements/test/cuda.in # anthropic # httpx # httpx2 @@ -76,8 +77,10 @@ azure-storage-blob==12.28.0 # via runai-model-streamer-azure backoff==2.2.1 # via -r requirements/test/cuda.in -black==24.10.0 - # via datamodel-code-generator +black==26.5.1 + # via + # -r requirements/test/cuda.in + # datamodel-code-generator blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.0.0 @@ -154,8 +157,9 @@ coverage==7.10.6 # via pytest-cov cramjam==2.9.0 # via fastparquet -cryptography==46.0.5 +cryptography==50.0.1 # via + # -r requirements/test/cuda.in # azure-identity # azure-storage-blob # msal @@ -257,18 +261,21 @@ fastsafetensors==0.3.3 # via # -c requirements/cuda.txt # -r requirements/test/cuda.in -filelock==3.16.1 +filelock==4.0.7 # via # -c requirements/common.txt # -r requirements/test/../common.txt # blobfile # datasets # huggingface-hub + # python-discovery # ray # torch # virtualenv -fonttools==4.55.0 - # via matplotlib +fonttools==4.66.1 + # via + # -r requirements/test/cuda.in + # matplotlib frozendict==2.4.6 # via einx frozenlist==1.5.0 @@ -333,16 +340,20 @@ h11==0.16.0 # httpcore # httpcore2 # uvicorn -h2==4.3.0 - # via httpx +h2==4.4.1 + # via + # -r requirements/test/cuda.in + # httpx harfile==0.5.0 # via schemathesis hf-xet==1.6.0 # via huggingface-hub hiredis==3.0.0 # via tensorizer -hpack==4.1.0 - # via h2 +hpack==4.2.0 + # via + # -r requirements/test/cuda.in + # h2 html2text==2025.4.15 # via gpt-oss httpcore==1.0.9 @@ -364,7 +375,9 @@ httpx==0.27.2 # perceptron # schemathesis httpx2==2.12.0 - # via mcp + # via + # -r requirements/test/cuda.in + # mcp huggingface-hub==1.31.0 # via # -c requirements/common.txt @@ -497,13 +510,16 @@ lm-format-enforcer==0.11.3 # -r requirements/test/../common.txt loguru==0.7.3 # via compressed-tensors -lxml==5.3.0 +lxml==6.1.3 # via + # -r requirements/test/cuda.in # blobfile # gpt-oss # sacrebleu -mako==1.3.10 - # via alembic +mako==1.4.3 + # via + # -r requirements/test/cuda.in + # alembic markdown-it-py==3.0.0 # via rich markupsafe==3.0.1 @@ -539,14 +555,15 @@ more-itertools==10.5.0 # via lm-eval mpmath==1.3.0 # via sympy -msal==1.34.0 +msal==1.39.0 # via # azure-identity # msal-extensions msal-extensions==1.3.1 # via azure-identity -msgpack==1.1.0 +msgpack==1.2.3 # via + # -r requirements/test/cuda.in # librosa # ray msgspec==0.21.1 @@ -733,8 +750,10 @@ opentelemetry-semantic-conventions-ai==0.4.13 # -r requirements/test/../common.txt optuna==3.6.1 # via genai-perf -orjson==3.11.5 - # via genai-perf +orjson==3.12.0 + # via + # -r requirements/test/cuda.in + # genai-perf oss-harmony==0.0.11 # via # -c requirements/common.txt @@ -765,6 +784,7 @@ packaging==24.2 # statsmodels # transformers # typepy + # virtualenv pandas==2.2.3 # via # datasets @@ -775,7 +795,7 @@ pandas==2.2.3 # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt -pathspec==0.12.1 +pathspec==1.1.1 # via black pathvalidate==3.2.1 # via pytablewriter @@ -862,12 +882,14 @@ py-cpuinfo==9.0.0 # via -r requirements/test/../common.txt py-spy==0.4.0 # via ray -pyarrow==23.0.0 +pyarrow==25.0.1 # via + # -r requirements/test/cuda.in # datasets # genai-perf -pyasn1==0.6.1 +pyasn1==0.6.4 # via + # -r requirements/test/cuda.in # pyasn1-modules # rsa pyasn1-modules==0.4.2 @@ -915,13 +937,17 @@ pydantic-extra-types==2.10.5 # fastapi # mistral-common pydantic-settings==2.14.2 - # via fastapi -pygments==2.18.0 # via + # -r requirements/test/cuda.in + # fastapi +pygments==2.21.0 + # via + # -r requirements/test/cuda.in # pytest # rich -pyjwt==2.11.0 +pyjwt==2.15.1 # via + # -r requirements/test/cuda.in # mcp # msal pyparsing==3.2.0 @@ -966,6 +992,8 @@ python-dateutil==2.9.0.post0 # matplotlib # pandas # typepy +python-discovery==1.6.1 + # via virtualenv python-dotenv==1.2.2 # via # pydantic-settings @@ -974,10 +1002,13 @@ python-json-logger==4.1.0 # via -r requirements/test/../common.txt python-multipart==0.0.32 # via + # -r requirements/test/cuda.in # fastapi # mcp python-rapidjson==1.20 # via tritonclient +pytokens==0.4.1 + # via black pytrec-eval-terrier==0.5.7 # via mteb pytz==2024.2 @@ -1028,7 +1059,7 @@ regex==2026.2.28 # sacrebleu # tiktoken # transformers -requests==2.32.3 +requests==2.34.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt @@ -1133,7 +1164,7 @@ sentry-sdk==2.63.0 # via fastapi-cloud-cli setproctitle==1.3.7 # via -r requirements/test/../common.txt -setuptools==77.0.3 +setuptools==80.10.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt @@ -1395,8 +1426,9 @@ typing-inspection==0.4.2 # pydantic-settings tzdata==2024.2 # via pandas -urllib3==2.2.3 +urllib3==2.8.0 # via + # -r requirements/test/cuda.in # blobfile # botocore # docker @@ -1416,8 +1448,10 @@ uvloop==0.22.1 # via uvicorn vector-quantize-pytorch==1.21.2 # via -r requirements/test/cuda.in -virtualenv==20.31.2 - # via ray +virtualenv==21.14.0 + # via + # -r requirements/test/cuda.in + # ray vocos==0.1.0 # via -r requirements/test/cuda.in watchfiles==1.2.0 @@ -1428,8 +1462,10 @@ wcwidth==0.2.13 # via ftfy websockets==16.0 # via uvicorn -werkzeug==3.1.3 - # via schemathesis +werkzeug==3.1.9 + # via + # -r requirements/test/cuda.in + # schemathesis word2number==1.1 # via lm-eval wrapt==1.17.2 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index 42c2b38d0f..d8baae6c5c 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -80,6 +80,29 @@ datasets>=3.3.0,<=3.6.0 openpyxl # required for perf comparison excel report plotly # required for perf comparison html report +# Dependabot GHSA floors for transitive test deps (excl. dependabot.yml ignores). +# Skipped (no patched release yet): sqlitedict, accelerate, lmcache, nltk GHSA-8mgp-746c-j5xp. +urllib3>=2.7.0 +cryptography>=50.0.0 +pyjwt>=2.13.0 +pyasn1>=0.6.4 +python-multipart>=0.0.31 +anyio>=4.14.2 +httpx2>=2.12.0 +werkzeug>=3.1.6 +mako>=1.3.12 +black>=26.3.1 +msgpack>=1.2.1 +lxml>=6.1.0 +pyarrow>=23.0.1 +orjson>=3.11.6 +hpack>=4.2.0 +h2>=4.4.1 +pydantic-settings>=2.14.2 +virtualenv>=20.36.1 +fonttools>=4.60.2 +pygments>=2.20.0 + # ROCm-specific extras (not in CUDA cuda.in) rapidfuzz # torchgeo also pulled in `lightning` transitively; disabled for the same diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 41234d29fd..3714afaf9d 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -33,8 +33,9 @@ anthropic==0.93.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -anyio==4.13.0 +anyio==4.14.2 # via + # -r requirements/test/rocm.in # anthropic # httpx # httpx2 @@ -76,7 +77,9 @@ backoff==2.2.1 bitsandbytes==0.50.0 # via -r requirements/test/rocm.in black==26.3.1 - # via datamodel-code-generator + # via + # -r requirements/test/rocm.in + # datamodel-code-generator blake3==1.0.8 # via -r requirements/test/../common.txt blobfile==3.0.0 @@ -107,7 +110,7 @@ certifi==2026.2.25 # httpx # requests # sentry-sdk -cffi==1.17.1 +cffi==2.1.1 # via # cryptography # soundfile @@ -156,8 +159,9 @@ coverage==7.13.5 # via pytest-cov cramjam==2.11.0 # via fastparquet -cryptography==46.0.0 +cryptography==50.0.1 # via + # -r requirements/test/rocm.in # azure-identity # azure-storage-blob # google-auth @@ -258,7 +262,9 @@ filelock==3.25.2 # torch # virtualenv fonttools==4.62.1 - # via matplotlib + # via + # -r requirements/test/rocm.in + # matplotlib frozendict==2.4.7 # via einx frozenlist==1.8.0 @@ -326,16 +332,20 @@ h11==0.16.0 # httpcore # httpcore2 # uvicorn -h2==4.3.0 - # via httpx +h2==4.4.1 + # via + # -r requirements/test/rocm.in + # httpx harfile==0.5.0 # via schemathesis hf-xet==1.6.0 # via huggingface-hub hiredis==3.3.1 # via tensorizer -hpack==4.1.0 - # via h2 +hpack==4.2.0 + # via + # -r requirements/test/rocm.in + # h2 html2text==2025.4.15 # via gpt-oss httpcore==1.0.9 @@ -357,7 +367,9 @@ httpx==0.27.2 # perceptron # schemathesis httpx2==2.12.0 - # via mcp + # via + # -r requirements/test/rocm.in + # mcp huggingface-hub==1.31.0 # via # -c requirements/common.txt @@ -492,13 +504,16 @@ logistro==2.0.1 # kaleido loguru==0.7.3 # via compressed-tensors -lxml==6.0.2 +lxml==6.1.3 # via + # -r requirements/test/rocm.in # blobfile # gpt-oss # sacrebleu -mako==1.3.10 - # via alembic +mako==1.4.3 + # via + # -r requirements/test/rocm.in + # alembic markdown-it-py==4.0.0 # via rich markupsafe==3.0.3 @@ -538,14 +553,15 @@ more-itertools==10.8.0 # lm-eval mpmath==1.3.0 # via sympy -msal==1.35.1 +msal==1.39.0 # via # azure-identity # msal-extensions msal-extensions==1.3.1 # via azure-identity -msgpack==1.1.2 +msgpack==1.2.3 # via + # -r requirements/test/rocm.in # librosa # ray msgspec==0.21.0 @@ -705,6 +721,7 @@ optuna==3.6.1 # via genai-perf orjson==3.11.7 # via + # -r requirements/test/rocm.in # genai-perf # kaleido oss-harmony==0.0.11 @@ -843,10 +860,13 @@ py-spy==0.4.1 # via ray pyarrow==23.0.1 # via + # -r requirements/test/rocm.in # datasets # genai-perf -pyasn1==0.6.3 - # via pyasn1-modules +pyasn1==0.6.4 + # via + # -r requirements/test/rocm.in + # pyasn1-modules pyasn1-modules==0.4.2 # via google-auth pybase64==1.4.3 @@ -889,14 +909,18 @@ pydantic-extra-types==2.11.1 # via # fastapi # mistral-common -pydantic-settings==2.13.1 - # via fastapi -pygments==2.19.2 +pydantic-settings==2.15.0 # via + # -r requirements/test/rocm.in + # fastapi +pygments==2.21.0 + # via + # -r requirements/test/rocm.in # pytest # rich -pyjwt==2.12.1 +pyjwt==2.15.1 # via + # -r requirements/test/rocm.in # mcp # msal pyparsing==3.3.2 @@ -949,8 +973,9 @@ python-dotenv==1.2.2 # uvicorn python-json-logger==4.1.0 # via -r requirements/test/../common.txt -python-multipart==0.0.26 +python-multipart==0.0.32 # via + # -r requirements/test/rocm.in # fastapi # mcp python-rapidjson==1.23 @@ -1007,7 +1032,7 @@ regex==2026.2.28 # sacrebleu # tiktoken # transformers -requests==2.32.5 +requests==2.34.2 # via # -c requirements/common.txt # -r requirements/test/../common.txt @@ -1347,8 +1372,9 @@ typing-inspection==0.4.2 # mcp # pydantic # pydantic-settings -urllib3==2.6.3 +urllib3==2.8.0 # via + # -r requirements/test/rocm.in # blobfile # botocore # docker @@ -1369,7 +1395,9 @@ uvloop==0.22.1 vector-quantize-pytorch==1.28.0 # via -r requirements/test/rocm.in virtualenv==21.2.0 - # via ray + # via + # -r requirements/test/rocm.in + # ray vocos==0.1.0 # via -r requirements/test/rocm.in watchfiles==1.1.1 @@ -1381,7 +1409,9 @@ wcwidth==0.6.0 websockets==16.0 # via uvicorn werkzeug==3.1.6 - # via schemathesis + # via + # -r requirements/test/rocm.in + # schemathesis word2number==1.1 # via lm-eval wrapt==2.1.2 diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index a5e7470d6d..28fdf8aacd 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -61,3 +61,23 @@ opentelemetry-semantic-conventions-ai>=0.4.1 cohere_melody>=0.11.1 # required for cohere command reasoning/tool parser tests cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import cohere.types) + +# Dependabot GHSA floors for transitive test deps (excl. dependabot.yml ignores). +urllib3>=2.7.0 +cryptography>=50.0.0 +pyjwt>=2.13.0 +pyasn1>=0.6.4 +python-multipart>=0.0.31 +anyio>=4.14.2 +httpx2>=2.12.0 +werkzeug>=3.1.6 +mako>=1.3.12 +msgpack>=1.2.1 +lxml>=6.1.0 +pyarrow>=23.0.1 +orjson>=3.11.6 +hpack>=4.2.0 +h2>=4.4.1 +pydantic-settings>=2.14.2 +fonttools>=4.60.2 +pygments>=2.20.0 diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index b2a6eb6da1..d6aa92f210 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -33,6 +33,7 @@ anthropic==0.122.0 # -r requirements/test/../common.txt anyio==4.14.2 # via + # -r requirements/test/xpu.in # anthropic # httpx # httpx2 @@ -130,6 +131,7 @@ coverage==7.15.4 # via pytest-cov cryptography==50.0.0 # via + # -r requirements/test/xpu.in # azure-identity # azure-storage-blob # google-auth @@ -219,7 +221,9 @@ filelock==3.32.3 # modelscope # torch fonttools==4.64.0 - # via matplotlib + # via + # -r requirements/test/xpu.in + # matplotlib frozenlist==1.8.0 # via # aiohttp @@ -270,17 +274,23 @@ h11==0.16.0 # httpcore # httpcore2 # uvicorn +h2==4.4.1 + # via -r requirements/test/xpu.in harfile==0.5.0 # via schemathesis hf-xet==1.6.0 # via huggingface-hub hiredis==3.4.1 # via tensorizer +hpack==4.2.0 + # via + # -r requirements/test/xpu.in + # h2 html2text==2025.4.15 # via gpt-oss httpcore==1.0.9 # via httpx -httpcore2==2.10.0 +httpcore2==2.13.1 # via httpx2 httptools==0.8.0 # via uvicorn @@ -293,8 +303,9 @@ httpx==0.28.1 # fastapi-cloud-cli # huggingface-hub # model-hosting-container-standards -httpx2==2.10.0 +httpx2==2.13.1 # via + # -r requirements/test/xpu.in # mcp # openai huggingface-hub==1.31.0 @@ -311,6 +322,8 @@ huggingface-hub==1.31.0 # transformers humanize==4.16.0 # via runai-model-streamer +hyperframe==6.1.0 + # via h2 hypothesis==6.165.10 # via # hypothesis-graphql @@ -441,14 +454,18 @@ loguru==0.7.3 # via compressed-tensors lxml==6.1.1 # via + # -r requirements/test/xpu.in # blobfile # gpt-oss # sacrebleu +mako==1.4.3 + # via -r requirements/test/xpu.in markdown-it-py==4.2.0 # via rich markupsafe==3.0.3 # via # jinja2 + # mako # werkzeug matplotlib==3.11.1 # via -r requirements/test/xpu.in @@ -495,7 +512,9 @@ msal==1.38.0 msal-extensions==1.3.1 # via azure-identity msgpack==1.2.1 - # via librosa + # via + # -r requirements/test/xpu.in + # librosa msgspec==0.21.1 # via -r requirements/test/../common.txt mteb==2.19.4 @@ -638,6 +657,8 @@ opentelemetry-semantic-conventions-ai==0.5.1 # -c requirements/common.txt # -r requirements/test/../common.txt # -r requirements/test/xpu.in +orjson==3.12.0 + # via -r requirements/test/xpu.in oss-harmony==0.0.11 # via # -c requirements/common.txt @@ -730,9 +751,13 @@ psutil==7.2.2 py-cpuinfo==9.0.0 # via -r requirements/test/../common.txt pyarrow==25.0.1 - # via datasets + # via + # -r requirements/test/xpu.in + # datasets pyasn1==0.6.4 - # via pyasn1-modules + # via + # -r requirements/test/xpu.in + # pyasn1-modules pyasn1-modules==0.4.2 # via google-auth pybase64==1.5.0 @@ -774,15 +799,19 @@ pydantic-extra-types==2.11.1 # fastapi # mistral-common pydantic-settings==2.15.0 - # via fastapi + # via + # -r requirements/test/xpu.in + # fastapi pyelftools==0.33 # via triton-xpu pygments==2.21.0 # via + # -r requirements/test/xpu.in # pytest # rich pyjwt==2.13.0 # via + # -r requirements/test/xpu.in # mcp # msal pyparsing==3.3.2 @@ -832,6 +861,7 @@ python-json-logger==4.2.0 # via -r requirements/test/../common.txt python-multipart==0.0.32 # via + # -r requirements/test/xpu.in # fastapi # mcp pytrec-eval-terrier==0.5.10 @@ -1193,6 +1223,7 @@ umf==1.1.0 # torch urllib3==2.7.0 # via + # -r requirements/test/xpu.in # blobfile # botocore # docker @@ -1218,7 +1249,9 @@ wcwidth==0.8.3 websockets==17.0.1 # via uvicorn werkzeug==3.1.8 - # via schemathesis + # via + # -r requirements/test/xpu.in + # schemathesis word2number==1.1 # via lm-eval xgrammar==0.2.8 diff --git a/requirements/xpu.txt b/requirements/xpu.txt index fd5b3214f0..d9d0f3080f 100644 --- a/requirements/xpu.txt +++ b/requirements/xpu.txt @@ -8,7 +8,7 @@ cmake>=3.26.1 packaging>=24.2 setuptools-scm>=8 setuptools-rust>=1.9.0 -setuptools>=77.0.3,<81.0.0 +setuptools>=78.1.1,<81.0.0 wheel jinja2>=3.1.6 datasets # for benchmark scripts diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6c1a8fe727..99878a52ab 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -379,7 +379,7 @@ dependencies = [ "getrandom 0.2.17", "instant", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "tokio", ] @@ -493,6 +493,15 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bstr" version = "1.12.1" @@ -2174,10 +2183,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-sys 0.61.2", ] [[package]] @@ -2191,6 +2202,21 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -3039,7 +3065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -3379,9 +3405,9 @@ checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" dependencies = [ "libc", "once_cell", @@ -3393,18 +3419,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" dependencies = [ "libc", "pyo3-build-config", @@ -3412,9 +3438,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -3424,22 +3450,21 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.117", ] [[package]] name = "pythonize" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b79f670c9626c8b651c0581011b57b6ba6970bb69faf01a7c4c0cfc81c43f95" +checksum = "6ec376e1216e0c929a74964ce2020012a1a39f32d80e78aa688721219ea7fb89" dependencies = [ "pyo3", "serde", @@ -3475,9 +3500,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3899,9 +3924,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -4276,15 +4301,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.18.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.13.0", + "jiff", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -4295,9 +4322,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.18.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -4973,6 +5000,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tls-listener" version = "0.11.2" @@ -5549,7 +5582,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e0d3708daa..39f4ab1a57 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -78,8 +78,8 @@ prometheus-client-derive-encode = "0.5.0" prost = "0.14.3" prost-types = "0.14.3" protox = "0.9.1" -pyo3 = "0.28.3" -pythonize = "0.28.0" +pyo3 = "0.29.0" +pythonize = "0.29.0" rand = "0.9.2" rand_distr = "0.5.1" rayon = "1.11.0" @@ -97,7 +97,7 @@ serde_default = "0.2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } serde_repr = "0.1.20" serde_tuple = "1.1.3" -serde_with = "3.18.0" +serde_with = "3.21.0" serial_test = { version = "3.2.0", features = ["file_locks"] } sha2 = "0.10.9" smallvec = "1.15.1" From df5668e5c935990cd9595bad2c3559b821092a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Ganczarenko?= Date: Wed, 30 Sep 2026 14:19:33 +0200 Subject: [PATCH 115/248] [XPU] Dispatch nn.LayerNorm to fused SYCL kernel via CustomOp (#57172) Signed-off-by: Michal Ganczarenko Co-authored-by: Claude Sonnet 5 Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- vllm/_xpu_ops.py | 66 ++++++++++++++ vllm/model_executor/layers/layernorm.py | 89 +++++++++++++++---- vllm/model_executor/models/deepseek_v2.py | 2 +- vllm/model_executor/models/gpt2.py | 7 +- vllm/model_executor/models/gpt_j.py | 5 +- vllm/model_executor/models/nemotron.py | 55 +++++++++--- vllm/model_executor/models/opt.py | 8 +- vllm/model_executor/models/siglip.py | 12 +-- .../models/transformers/base.py | 4 + .../models/transformers/utils.py | 29 ++++++ vllm/models/deepseek_v32/attention.py | 2 +- vllm/models/glm5next/common/attention.py | 2 +- vllm/models/hy_v4/nvidia/attention.py | 2 +- 13 files changed, 237 insertions(+), 46 deletions(-) diff --git a/vllm/_xpu_ops.py b/vllm/_xpu_ops.py index d25ef6b258..1f00bc681d 100644 --- a/vllm/_xpu_ops.py +++ b/vllm/_xpu_ops.py @@ -135,6 +135,46 @@ def _fused_add_gemma_rms_norm_impl( torch.ops._C.fused_add_gemma_rms_norm(input, residual, weight, epsilon) +def _layer_norm_impl( + out: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + epsilon: float, +) -> None: + torch.ops._C.layer_norm(out, input, weight, bias, epsilon) + + +def _fused_add_layer_norm_impl( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + epsilon: float, +) -> None: + torch.ops._C.fused_add_layer_norm(input, residual, weight, bias, epsilon) + + +def _nemotron_layer_norm_impl( + out: torch.Tensor, + input: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + epsilon: float, +) -> None: + torch.ops._C.nemotron_layer_norm(out, input, weight, bias, epsilon) + + +def _fused_add_nemotron_layer_norm_impl( + input: torch.Tensor, + residual: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + epsilon: float, +) -> None: + torch.ops._C.fused_add_nemotron_layer_norm(input, residual, weight, bias, epsilon) + + def _gdn_attention_core_xpu_impl( core_attn_out: torch.Tensor, z: torch.Tensor, @@ -1268,6 +1308,32 @@ class xpu_ops: mutates_args=["input", "residual"], ) + if hasattr(torch.ops._C, "layer_norm"): + direct_register_custom_op( + op_name="xpu_layer_norm", + op_func=_layer_norm_impl, + mutates_args=["out"], + ) + + direct_register_custom_op( + op_name="xpu_fused_add_layer_norm", + op_func=_fused_add_layer_norm_impl, + mutates_args=["input", "residual"], + ) + + if hasattr(torch.ops._C, "nemotron_layer_norm"): + direct_register_custom_op( + op_name="xpu_nemotron_layer_norm", + op_func=_nemotron_layer_norm_impl, + mutates_args=["out"], + ) + + direct_register_custom_op( + op_name="xpu_fused_add_nemotron_layer_norm", + op_func=_fused_add_nemotron_layer_norm_impl, + mutates_args=["input", "residual"], + ) + direct_register_custom_op( op_name="xpu_ops_deepseek_scaling_rope", op_func=_xpu_ops_deepseek_scaling_rope_impl, diff --git a/vllm/model_executor/layers/layernorm.py b/vllm/model_executor/layers/layernorm.py index 551cd05d3b..266a346a64 100644 --- a/vllm/model_executor/layers/layernorm.py +++ b/vllm/model_executor/layers/layernorm.py @@ -211,6 +211,79 @@ class GemmaRMSNorm(CustomOp): return out +@CustomOp.register("layer_norm") +class LayerNorm(CustomOp): + """Standard (mean-centered) LayerNorm. + + Drop-in for a bare `nn.LayerNorm` that dispatches to a fused XPU kernel + when one is available, and to the native implementation otherwise. + + Normalization runs in the wider of the input and parameter dtypes and the + result is cast back to the input dtype, so `dtype=torch.float32` gives an + fp32 reduction for a lower-precision input. + """ + + def __init__( + self, + hidden_size: int, + eps: float = 1e-5, + elementwise_affine: bool = True, + bias: bool = True, + dtype: torch.dtype | None = None, + ) -> None: + super().__init__() + self.normalized_shape = (hidden_size,) + self.eps = eps + weight_dtype = dtype or torch.get_default_dtype() + self.weight: nn.Parameter | None = None + self.bias: nn.Parameter | None = None + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(hidden_size, dtype=weight_dtype)) + if bias: + self.bias = nn.Parameter(torch.zeros(hidden_size, dtype=weight_dtype)) + + def _compute_dtype(self, x: torch.Tensor) -> torch.dtype: + if self.weight is None: + return x.dtype + return torch.promote_types(x.dtype, self.weight.dtype) + + def forward_native(self, x: torch.Tensor) -> torch.Tensor: + compute_dtype = self._compute_dtype(x) + if compute_dtype != x.dtype: + return F.layer_norm( + x.to(compute_dtype), + self.normalized_shape, + self.weight, + self.bias, + self.eps, + ).type_as(x) + return F.layer_norm(x, self.normalized_shape, self.weight, self.bias, self.eps) + + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor: + return self.forward_native(x) + + def forward_xpu(self, x: torch.Tensor) -> torch.Tensor: + import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_layer_norm + + if ( + self.weight is None + or self.bias is None + or not hasattr(torch.ops._C, "layer_norm") + # The kernel reduces in the input dtype, so a widened reduction + # takes the native path. + or self._compute_dtype(x) != x.dtype + ): + return self.forward_native(x) + # empty_like preserves x's strides, but the kernel requires a + # contiguous out (unlike x, which it can handle non-contiguous). + out = torch.empty(x.shape, device=x.device, dtype=x.dtype) + torch.ops.vllm.xpu_layer_norm(out, x, self.weight, self.bias, self.eps) + return out + + def extra_repr(self) -> str: + return f"hidden_size={self.normalized_shape[0]}, eps={self.eps}" + + # --8<-- [start:rms_norm_gated] @CustomOp.register("rms_norm_gated") class RMSNormGated(CustomOp): @@ -350,19 +423,3 @@ class RMSNormGated(CustomOp): self, x: torch.Tensor, z: torch.Tensor | None = None ) -> torch.Tensor: return self.forward_cuda(x, z) - - -class LayerNorm(nn.Module): - """Layer Normalization.""" - - def __init__(self, dim: int, eps: float = 1e-6): - super().__init__() - self.dim = dim - self.eps = eps - self.weight = nn.Parameter(torch.ones(dim, dtype=torch.float32)) - self.bias = nn.Parameter(torch.zeros(dim, dtype=torch.float32)) - - def forward(self, x: torch.Tensor): - return F.layer_norm( - x.float(), (self.dim,), self.weight, self.bias, self.eps - ).type_as(x) diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index d5728e0c33..01010a13fe 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -715,7 +715,7 @@ class Indexer(nn.Module): disable_tp=True, prefix=f"{prefix}.wk_weights_proj", ) - self.k_norm = LayerNorm(self.head_dim, eps=1e-6) + self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32) self.softmax_scale = self.head_dim**-0.5 self.scale_fmt = "ue8m0" diff --git a/vllm/model_executor/models/gpt2.py b/vllm/model_executor/models/gpt2.py index c95fe51848..f5aa444025 100644 --- a/vllm/model_executor/models/gpt2.py +++ b/vllm/model_executor/models/gpt2.py @@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import ( ) from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.layernorm import LayerNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -156,11 +157,11 @@ class GPT2Block(nn.Module): hidden_size = config.hidden_size inner_dim = config.n_inner if config.n_inner is not None else 4 * hidden_size - self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.ln_1 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon) self.attn = GPT2Attention( config, cache_config, quant_config, prefix=f"{prefix}.attn" ) - self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon) + self.ln_2 = LayerNorm(hidden_size, eps=config.layer_norm_epsilon) self.mlp = GPT2MLP(inner_dim, config, quant_config, prefix=f"{prefix}.mlp") def forward( @@ -212,7 +213,7 @@ class GPT2Model(nn.Module): lambda prefix: GPT2Block(config, cache_config, quant_config, prefix=prefix), prefix=f"{prefix}.h", ) - self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states"], config.n_embd ) diff --git a/vllm/model_executor/models/gpt_j.py b/vllm/model_executor/models/gpt_j.py index 04fe9ba549..8b54dfcbae 100644 --- a/vllm/model_executor/models/gpt_j.py +++ b/vllm/model_executor/models/gpt_j.py @@ -31,6 +31,7 @@ from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.layernorm import LayerNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -163,7 +164,7 @@ class GPTJBlock(nn.Module): ): super().__init__() inner_dim = 4 * config.n_embd if config.n_inner is None else config.n_inner - self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) + self.ln_1 = LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) self.attn = GPTJAttention( config, cache_config, quant_config, prefix=f"{prefix}.attn" ) @@ -206,7 +207,7 @@ class GPTJModel(nn.Module): lambda prefix: GPTJBlock(config, cache_config, quant_config, prefix=prefix), prefix=f"{prefix}.h", ) - self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) + self.ln_f = LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon) self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( ["hidden_states"], config.n_embd ) diff --git a/vllm/model_executor/models/nemotron.py b/vllm/model_executor/models/nemotron.py index 97199c1841..a271432813 100644 --- a/vllm/model_executor/models/nemotron.py +++ b/vllm/model_executor/models/nemotron.py @@ -33,6 +33,7 @@ from torch import nn from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.linear import ( @@ -77,23 +78,22 @@ def _cast_if_autocast_enabled(device_type: str, *args): ) -class NemotronLayerNorm1P(nn.LayerNorm): - def __init__( - self, - normalized_shape: int | list[int] | torch.Size, - eps: float = 1e-5, - elementwise_affine: bool = True, - bias: bool = True, - device=None, - dtype=None, - ): - super().__init__(normalized_shape, eps, elementwise_affine, bias, device, dtype) +@CustomOp.register("nemotron_layer_norm") +class NemotronLayerNorm1P(CustomOp): + """LayerNorm variant used by Nemotron: `x * (1 + w)` instead of `x * w`.""" - def forward( + def __init__(self, hidden_size: int, eps: float = 1e-5) -> None: + super().__init__() + self.normalized_shape = (hidden_size,) + self.eps = eps + self.weight = nn.Parameter(torch.zeros(hidden_size)) + self.bias = nn.Parameter(torch.zeros(hidden_size)) + + def forward_native( self, x: torch.Tensor, residual: torch.Tensor | None = None, - ) -> torch.Tensor: + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: if residual is not None: x = x + residual residual = x @@ -105,6 +105,35 @@ class NemotronLayerNorm1P(nn.LayerNorm): x = torch.nn.functional.layer_norm(*args) return x if residual is None else (x, residual) + def forward_cuda( + self, + x: torch.Tensor, + residual: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + return self.forward_native(x, residual) + + def forward_xpu( + self, + x: torch.Tensor, + residual: torch.Tensor | None = None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + import vllm._xpu_ops # noqa: F401 registers torch.ops.vllm.xpu_nemotron_layer_norm + + if not hasattr(torch.ops._C, "nemotron_layer_norm"): + return self.forward_native(x, residual) + if residual is not None: + if not hasattr(torch.ops._C, "fused_add_nemotron_layer_norm"): + return self.forward_native(x, residual) + torch.ops.vllm.xpu_fused_add_nemotron_layer_norm( + x, residual, self.weight, self.bias, self.eps + ) + return x, residual + # empty_like preserves x's strides, but the kernel requires a + # contiguous out (unlike x, which it can handle non-contiguous). + out = torch.empty(x.shape, device=x.device, dtype=x.dtype) + torch.ops.vllm.xpu_nemotron_layer_norm(out, x, self.weight, self.bias, self.eps) + return out + class NemotronMLP(nn.Module): def __init__( diff --git a/vllm/model_executor/models/opt.py b/vllm/model_executor/models/opt.py index 4726c72b41..4c9e5ffca4 100644 --- a/vllm/model_executor/models/opt.py +++ b/vllm/model_executor/models/opt.py @@ -32,6 +32,7 @@ from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.layernorm import LayerNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -143,7 +144,7 @@ class OPTDecoderLayer(nn.Module): ) self.do_layer_norm_before = config.do_layer_norm_before - self.self_attn_layer_norm = nn.LayerNorm( + self.self_attn_layer_norm = LayerNorm( self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine ) self.fc1 = ColumnParallelLinear( @@ -161,7 +162,7 @@ class OPTDecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.fc2", ) - self.final_layer_norm = nn.LayerNorm( + self.final_layer_norm = LayerNorm( self.embed_dim, elementwise_affine=config.layer_norm_elementwise_affine ) @@ -246,8 +247,9 @@ class OPTDecoder(nn.Module): # keep backward compatibility with checkpoints that have been fine-tuned # before transformers v4.20.1 # see https://github.com/facebookresearch/metaseq/pull/164 + self.final_layer_norm: LayerNorm | None if config.do_layer_norm_before and not config._remove_final_layer_norm: - self.final_layer_norm = nn.LayerNorm( + self.final_layer_norm = LayerNorm( config.hidden_size, elementwise_affine=config.layer_norm_elementwise_affine, ) diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index ab7146c598..bcaaf11f0d 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv2dLayer +from vllm.model_executor.layers.layernorm import LayerNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -467,13 +468,13 @@ class SiglipEncoderLayer(nn.Module): prefix=f"{prefix}.self_attn", attn_cls=attn_cls, ) - self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) + self.layer_norm1 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps) self.mlp = SiglipMLP( config, quant_config=quant_config, prefix=f"{prefix}.mlp", ) - self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_eps) + self.layer_norm2 = LayerNorm(self.embed_dim, eps=config.layer_norm_eps) def forward( self, @@ -573,7 +574,7 @@ class SiglipTextTransformer(nn.Module): attn_cls=EncoderOnlyAttention, ) - self.final_layer_norm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + self.final_layer_norm = LayerNorm(embed_dim, eps=config.layer_norm_eps) self.head = nn.Linear(embed_dim, config.projection_size) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: @@ -616,7 +617,7 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module): self.attention = torch.nn.MultiheadAttention( config.hidden_size, config.num_attention_heads, batch_first=True ) - self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.layernorm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.mlp = SiglipMLP( config=config, quant_config=quant_config, @@ -685,8 +686,9 @@ class SiglipVisionTransformer(nn.Module): if require_post_norm is None: require_post_norm = len(self.encoder.layers) == num_hidden_layers + self.post_layernorm: LayerNorm | None if require_post_norm: - self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + self.post_layernorm = LayerNorm(embed_dim, eps=config.layer_norm_eps) else: self.post_layernorm = None diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index ec0f474ee3..4e53f1fe18 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -71,6 +71,7 @@ from vllm.model_executor.models.transformers.utils import ( named_state, replace_conv_class, replace_embedding_class, + replace_layernorm_class, replace_linear_class, ) from vllm.model_executor.models.utils import ( @@ -466,6 +467,7 @@ class Base( - `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d` - Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding` - RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm` + - `nn.LayerNorm` with vLLM's `LayerNorm` """ tp_plan = self.model.tp_plan or {} @@ -556,6 +558,8 @@ class Base( new_module = replace_embedding_class( child_module, self.quant_config, prefix=qual_name ) + elif isinstance(child_module, nn.LayerNorm): + new_module = replace_layernorm_class(child_module) elif child_module_fusers := fusers[child_module]: for fuser in child_module_fusers: register_fusion(fuser, qual_name, child_module) diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py index 0443f390c4..8007cc789e 100644 --- a/vllm/model_executor/models/transformers/utils.py +++ b/vllm/model_executor/models/transformers/utils.py @@ -28,6 +28,7 @@ from torch import nn from vllm.logger import init_logger from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer +from vllm.model_executor.layers.layernorm import LayerNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, ReplicatedLinear, @@ -187,6 +188,34 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv: ) +def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module: + """Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's + `LayerNorm`. + + Args: + layernorm: `nn.LayerNorm` to be replaced. + + Returns: + The new `LayerNorm`. If the layernorm is not supported (a subclass with + its own behavior, multi-dim `normalized_shape`, or not both + elementwise-affine and biased), returns the original module unchanged. + + """ + if ( + type(layernorm) is not nn.LayerNorm + or len(layernorm.normalized_shape) != 1 + or not layernorm.elementwise_affine + or layernorm.bias is None + ): + return layernorm + + return LayerNorm( + layernorm.normalized_shape[0], + eps=layernorm.eps, + dtype=layernorm.weight.dtype, + ) + + def attrsetter(attr: str) -> Callable[[object, object], None]: """Set a possibly nested attribute, like the inverse of attrgetter.""" parent, _, name = attr.rpartition(".") diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index f7d0e76e54..4e63076414 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -84,7 +84,7 @@ class DeepseekV32Indexer(nn.Module): disable_tp=True, prefix=f"{prefix}.wk_weights_proj", ) - self.k_norm = LayerNorm(self.head_dim, eps=1e-6) + self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32) self.softmax_scale = self.head_dim**-0.5 self.scale_fmt = "ue8m0" diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index f39c3a7cd0..2e17424f1e 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -276,7 +276,7 @@ class Indexer(nn.Module): disable_tp=True, prefix=f"{prefix}.wk_weights_proj", ) - self.k_norm = LayerNorm(self.head_dim, eps=1e-6) + self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32) self.softmax_scale = self.head_dim**-0.5 # Hadamard-128 rotation of the indexer query is fused with the FP8 diff --git a/vllm/models/hy_v4/nvidia/attention.py b/vllm/models/hy_v4/nvidia/attention.py index 0e7fb61173..1e9be2e3bf 100644 --- a/vllm/models/hy_v4/nvidia/attention.py +++ b/vllm/models/hy_v4/nvidia/attention.py @@ -166,7 +166,7 @@ class Indexer(nn.Module): disable_tp=True, prefix=f"{prefix}.wk_weights_proj", ) - self.k_norm = LayerNorm(self.head_dim, eps=1e-6) + self.k_norm = LayerNorm(self.head_dim, eps=1e-6, dtype=torch.float32) self.softmax_scale = self.head_dim**-0.5 self.scale_fmt = "ue8m0" From 93dd80f63639453591e68891614729fb0bd44855 Mon Sep 17 00:00:00 2001 From: almayne Date: Wed, 30 Sep 2026 13:23:42 +0100 Subject: [PATCH 116/248] [CPU] Conv1d optimised kernel for aarch64 (#54093) Signed-off-by: Anna Mayne Co-authored-by: Li, Jiang --- cmake/cpu_extension.cmake | 1 + csrc/cpu/sgl-kernels/conv.cpp | 26 +- csrc/cpu/sgl-kernels/conv_arm.h | 126 +++++++++ csrc/cpu/torch_bindings.cpp | 49 ++-- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py | 256 +++++++++++------- .../test_cpu_unquantized_gemm_dispatch.py | 53 +++- vllm/_custom_ops.py | 8 +- .../layers/mamba/ops/cpu/gdn_attention.py | 116 ++++---- vllm/model_executor/layers/utils.py | 30 +- 9 files changed, 460 insertions(+), 205 deletions(-) create mode 100644 csrc/cpu/sgl-kernels/conv_arm.h diff --git a/cmake/cpu_extension.cmake b/cmake/cpu_extension.cmake index 664e309e94..730b963c23 100644 --- a/cmake/cpu_extension.cmake +++ b/cmake/cpu_extension.cmake @@ -524,6 +524,7 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) "csrc/cpu/cpu_tanhf_neon.hpp" ${VLLM_EXT_SRC}) if (ARM_BF16_FOUND) + set(VLLM_EXT_SRC "csrc/cpu/sgl-kernels/conv.cpp" ${VLLM_EXT_SRC}) if (ARM_I8MM_FOUND) set(VLLM_EXT_SRC "csrc/cpu/cpu_fused_moe_int8.cpp" ${VLLM_EXT_SRC}) endif() diff --git a/csrc/cpu/sgl-kernels/conv.cpp b/csrc/cpu/sgl-kernels/conv.cpp index 38079ba950..46aff3314a 100644 --- a/csrc/cpu/sgl-kernels/conv.cpp +++ b/csrc/cpu/sgl-kernels/conv.cpp @@ -66,6 +66,10 @@ struct tinygemm_kernel { } }; +#if defined(ARM_BF16_SUPPORT) +#include "conv_arm.h" +#endif + #if defined(CPU_CAPABILITY_AVX512) template struct tinygemm_kernel { @@ -543,16 +547,24 @@ at::Tensor causal_conv1d_weight_pack(const at::Tensor& weight) { int64_t dim = weight.size(0); int64_t width = weight.size(1); - constexpr int64_t BLOCK_N = block_size_n(); + constexpr int64_t BLOCK_N = +#if defined(ARM_BF16_SUPPORT) + 8;// 8 channel blocks for microtile +#else + block_size_n(); +#endif TORCH_CHECK(width == 4, "causal_conv1d_weight_pack: support only width of 4"); - TORCH_CHECK(dim % BLOCK_N == 0, "causal_conv1d_weight_pack: invalid dim size ", dim); + TORCH_CHECK( + dim % block_size_n() == 0, + "causal_conv1d_weight_pack: invalid dim size ", + dim); const int64_t N = dim, K2 = width >> 1; const int64_t NB = div_up(N, BLOCK_N); auto packed_weight = at::empty_like(weight); AT_DISPATCH_REDUCED_FLOATING_TYPES(weight.scalar_type(), "causal_conv1d_fwd_kernel_impl", [&] { - // cast to float32 as vnni size is 2 + // Copy each pair as an opaque 32-bit word. This is not a bf16->fp32 conversion. const float* w_data = reinterpret_cast(weight.data_ptr()); float* packed_data = reinterpret_cast(packed_weight.data_ptr()); @@ -642,9 +654,9 @@ at::Tensor causal_conv1d_fwd_cpu( const std::optional& has_initial_state, bool silu_activation, int64_t pad_slot_id, - bool is_vnni) { + bool is_weight_packed) { CHECK_CONTIGUOUS(weight); - auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight); + auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight); const bool is_var_seqlen = query_start_loc.has_value(); const int64_t input_ndim = is_var_seqlen ? 2 : 3; @@ -755,10 +767,10 @@ at::Tensor causal_conv1d_update_cpu( const std::optional& num_accepted_tokens, const std::optional& conv_state_indices, int64_t pad_slot_id, - bool is_vnni) { + bool is_weight_packed) { CHECK_CONTIGUOUS(x); CHECK_CONTIGUOUS(weight); - auto packed_w = is_vnni ? weight : causal_conv1d_weight_pack(weight); + auto packed_w = is_weight_packed ? weight : causal_conv1d_weight_pack(weight); TORCH_CHECK( x.dim() == 2 || x.dim() == 3, diff --git a/csrc/cpu/sgl-kernels/conv_arm.h b/csrc/cpu/sgl-kernels/conv_arm.h new file mode 100644 index 0000000000..9ad78c51fd --- /dev/null +++ b/csrc/cpu/sgl-kernels/conv_arm.h @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +#pragma once + +#if defined(ARM_BF16_SUPPORT) + +template +struct tinygemm_kernel { + static inline bfloat16x8_t load_bf16x8(const at::BFloat16* ptr) { + return vld1q_bf16(reinterpret_cast(ptr)); + } + + // Input layout: A[m][channel] + // Weight layout: B[N/MICRO_N, K/2, MICRO_N, 2] + // The packed weights are reused across all M positions. + static inline void tinygemm_neon_8( + const at::BFloat16* __restrict__ A, const at::BFloat16* __restrict__ B, + at::BFloat16* __restrict__ C, const at::BFloat16* __restrict__ bias, + const at::BFloat16* __restrict__ conv_states, bool has_initial_state, + int64_t M, int64_t lda, bool is_first_token) { + bfloat16x8_t va[4]; + // Load 4 pairs of kernel weights into 4 registers + const bfloat16x8_t vb0 = load_bf16x8(B); // channels 0-3, taps 0-1 + const bfloat16x8_t vb1 = load_bf16x8(B + 8); // channels 4-7, taps 0-1 + const bfloat16x8_t vb2 = load_bf16x8(B + 16); // channels 0-3, taps 2-3 + const bfloat16x8_t vb3 = load_bf16x8(B + 24); // channels 4-7, taps 2-3 + + // Load the input data. A single register holds the values for a + // single token across all 8 channels for this microtile. + auto load_a = [&](int64_t m) { + if (m < 0 && is_first_token) { + if (!has_initial_state) { + return vreinterpretq_bf16_u16(vdupq_n_u16(0)); + } + return load_bf16x8(conv_states + (m + K - 1) * lda); + } + return load_bf16x8(A + m * lda); + }; + + // Load the previous inputs from the cached state if available. + va[1] = load_a(-3); + va[2] = load_a(-2); + va[3] = load_a(-1); + + // Iterate over the sequence block, updating the previous conv inputs + // and reading the next token's input data (8 channels). + for (int64_t m = 0; m < M; ++m) { + va[0] = va[1]; + va[1] = va[2]; + va[2] = va[3]; + va[3] = load_a(m); + + float32x4_t vc0; + float32x4_t vc1; + if constexpr (has_bias) { + const bfloat16x8_t bias_vec = load_bf16x8(bias); + vc0 = vcvtq_low_f32_bf16(bias_vec); + vc1 = vcvtq_high_f32_bf16(bias_vec); + } else { + vc0 = vdupq_n_f32(0.0f); + vc1 = vdupq_n_f32(0.0f); + } + + // Use the zip operation to match the input layout to the weights layout + bfloat16x8_t va_pair = + vreinterpretq_bf16_u16( // channels 0-3, inputs for taps 0-1 + vzip1q_u16(vreinterpretq_u16_bf16(va[0]), + vreinterpretq_u16_bf16(va[1]))); + vc0 = vbfdotq_f32(vc0, va_pair, + vb0); // 4 independent length-two dot products + va_pair = vreinterpretq_bf16_u16( // channels 4-7, inputs for taps 0-1 + vzip2q_u16(vreinterpretq_u16_bf16(va[0]), + vreinterpretq_u16_bf16(va[1]))); + vc1 = vbfdotq_f32(vc1, va_pair, vb1); + va_pair = vreinterpretq_bf16_u16( // channels 0-3, inputs for taps 2-3 + vzip1q_u16(vreinterpretq_u16_bf16(va[2]), + vreinterpretq_u16_bf16(va[3]))); + // Add to the previous result for the same channels + vc0 = vbfdotq_f32(vc0, va_pair, vb2); + va_pair = vreinterpretq_bf16_u16( // channels 4-7, inputs for taps 2-3 + vzip2q_u16(vreinterpretq_u16_bf16(va[2]), + vreinterpretq_u16_bf16(va[3]))); + vc1 = vbfdotq_f32(vc1, va_pair, vb3); + + using fVec = at::vec::Vectorized; + fVec out0(vc0); + fVec out1(vc1); + if constexpr (has_silu) { + const fVec one(1.0f); + out0 = out0 / (one + out0.neg().fexp_u20()); + out1 = out1 / (one + out1.neg().fexp_u20()); + } + + // Convert the 2 x 4 fp32 results to 1 x 8 bf16 + const bfloat16x8_t out = + vcombine_bf16(vcvt_bf16_f32(static_cast(out0)), + vcvt_bf16_f32(static_cast(out1))); + vst1q_bf16(reinterpret_cast(C + m * lda), out); + } + } + + static inline void apply(const at::BFloat16* __restrict__ A, + const at::BFloat16* __restrict__ B, + at::BFloat16* __restrict__ C, + const at::BFloat16* __restrict__ bias, + const at::BFloat16* __restrict__ conv_states, + bool has_initial_state, int64_t M, int64_t lda, + bool is_first_token) { + static_assert(K == 4); + constexpr int64_t MICRO_N = 8; + static_assert(BLOCK_N % MICRO_N == 0); + + // BLOCK_N is always 32 or 64. M is the block sequence length. + // Input layout: A[m][channel] + // Weight layout: B[N/MICRO_N, K/2, MICRO_N, 2] + // Use an 8 channel microtile and iterate over the 8 channel blocks. + for (int64_t n = 0; n < BLOCK_N; n += MICRO_N) { + tinygemm_neon_8(A + n, B + n * K, C + n, has_bias ? bias + n : nullptr, + conv_states ? conv_states + n : nullptr, + has_initial_state, M, lda, is_first_token); + } + } +}; + +#endif diff --git a/csrc/cpu/torch_bindings.cpp b/csrc/cpu/torch_bindings.cpp index 6bbb06efa6..2cb1be095d 100644 --- a/csrc/cpu/torch_bindings.cpp +++ b/csrc/cpu/torch_bindings.cpp @@ -281,14 +281,14 @@ at::Tensor causal_conv1d_fwd_cpu( const std::optional& query_start_loc, const std::optional& cache_indices, const std::optional& has_initial_state, bool silu_activation, - int64_t pad_slot_id, bool is_vnni); + int64_t pad_slot_id, bool is_weight_packed); at::Tensor causal_conv1d_update_cpu( const at::Tensor& x, const at::Tensor& conv_states, const at::Tensor& weight, const std::optional& bias, bool silu_activation, const std::optional& num_accepted_tokens, const std::optional& conv_state_indices, int64_t pad_slot_id, - bool is_vnni); + bool is_weight_packed); void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input, const std::string& activation); @@ -517,6 +517,33 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { #endif // (defined(__aarch64__) && !defined(__APPLE__)) +#if (defined(__aarch64__) && defined(ARM_BF16_SUPPORT) && \ + !defined(__APPLE__)) || \ + (defined(__AVX512BF16__) && defined(__AVX512F__) && \ + defined(__AVX512VNNI__)) + + // Adapted from sglang: casual_conv1d kernels + ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor"); + ops.impl("causal_conv1d_weight_pack", torch::kCPU, + &causal_conv1d_weight_pack); + + ops.def( + "causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? " + "conv_states, Tensor? query_start_loc," + "Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, " + "int pad_slot_id, bool is_weight_packed) -> " + "Tensor"); + ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu); + + ops.def( + "causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor " + "weight, Tensor? bias, bool silu_activation," + "Tensor? num_accepted_tokens, Tensor? conv_state_indices, int " + "pad_slot_id, " + "bool is_weight_packed) -> Tensor"); + ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu); +#endif + // Layernorm // Apply Root Mean Square (RMS) Normalization to the input tensor. ops.def( @@ -665,24 +692,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) { "-> (Tensor, Tensor)"); ops.impl("quantize_fp8e4m3_vec", torch::kCPU, &quantize_fp8e4m3_vec); - ops.def("causal_conv1d_weight_pack(Tensor weight) -> Tensor"); - ops.impl("causal_conv1d_weight_pack", torch::kCPU, - &causal_conv1d_weight_pack); - ops.def( - "causal_conv1d_fwd_cpu(Tensor x, Tensor weight, Tensor? bias, Tensor? " - "conv_states, Tensor? query_start_loc," - "Tensor? cache_indices, Tensor? has_initial_state, bool silu_activation, " - "int pad_slot_id, bool is_vnni) -> " - "Tensor"); - ops.impl("causal_conv1d_fwd_cpu", torch::kCPU, &causal_conv1d_fwd_cpu); - ops.def( - "causal_conv1d_update_cpu(Tensor x, Tensor(a!) conv_states, Tensor " - "weight, Tensor? bias, bool silu_activation," - "Tensor? num_accepted_tokens, Tensor? conv_state_indices, int " - "pad_slot_id, " - "bool is_vnni) -> Tensor"); - ops.impl("causal_conv1d_update_cpu", torch::kCPU, &causal_conv1d_update_cpu); - // Adapted from sglang: MLA CPU kernels (AMX-only, DeepSeek V2/V3/R1) ops.def( "decode_attention_cpu(Tensor query, Tensor k_buffer, Tensor v_buffer, " diff --git a/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py b/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py index 3b662479b1..f0f0cd1396 100644 --- a/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py +++ b/tests/kernels/mamba/cpu/test_cpu_gdn_ops.py @@ -494,8 +494,10 @@ def _sd_conv_states( return storage.transpose(1, 2) -def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor: - return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight +def _maybe_pack_conv_weight( + weight: torch.Tensor, is_weight_packed: bool +) -> torch.Tensor: + return ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight @torch.inference_mode() @@ -523,7 +525,7 @@ def test_causal_conv1d_update_cpu_preserves_sd_state( bias=None, silu_activation=False, conv_state_indices=state_indices, - is_vnni=False, + is_weight_packed=False, ) assert out is x @@ -666,7 +668,10 @@ def test_spec_aware_nonspec_materializes_state_indices( @torch.inference_mode() +@pytest.mark.parametrize(("has_amx", "has_arm_bf16"), [(True, False), (False, True)]) def test_spec_forward_prepares_native_conv_metadata( + has_amx: bool, + has_arm_bf16: bool, monkeypatch: pytest.MonkeyPatch, ) -> None: block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32) @@ -687,14 +692,17 @@ def test_spec_forward_prepares_native_conv_metadata( ) forwarded_indices = None forwarded_counts = None + forwarded_is_weight_packed = None def causal_conv1d_update_cpu(**kwargs): - nonlocal forwarded_counts, forwarded_indices + nonlocal forwarded_counts, forwarded_indices, forwarded_is_weight_packed forwarded_indices = kwargs["conv_state_indices"] forwarded_counts = kwargs["num_accepted_tokens"] + forwarded_is_weight_packed = kwargs["is_weight_packed"] return kwargs["x"] - monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True) + monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: has_amx) + monkeypatch.setattr(gdn_attention, "is_arm_bf16", lambda: has_arm_bf16) monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False) monkeypatch.setattr( gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu @@ -732,6 +740,7 @@ def test_spec_forward_prepares_native_conv_metadata( assert actual is not None assert actual.is_contiguous() assert actual.dtype == torch.int32 + assert forwarded_is_weight_packed is True torch.testing.assert_close(actual, reference) @@ -792,15 +801,15 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No @pytest.mark.skipif( - not torch.cpu._is_amx_tile_supported(), - reason="requires AMX support", + not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()), + reason="native causal conv1d requires AMX or Arm BF16 support", ) @torch.inference_mode() def test_causal_conv1d_update_cpu_accepts_wide_state() -> None: state_len = CONV_KERNEL - 1 wide_state_len = state_len + 5 batch_size = 3 - is_vnni = True + is_weight_packed = True x, weight, bias = _conv_inputs(batch_size) conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32) @@ -813,7 +822,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None: wide_state[:, :, state_len:].fill_(7) wide_tail = wide_state[:, :, state_len:].clone() - conv_weight = _maybe_pack_conv_weight(weight, is_vnni) + conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed) out_narrow = ops.causal_conv1d_update_cpu( x=x, conv_states=narrow_state, @@ -821,7 +830,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None: bias=bias, silu_activation=True, conv_state_indices=conv_state_indices, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) out_wide = ops.causal_conv1d_update_cpu( x=x, @@ -830,7 +839,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None: bias=bias, silu_activation=True, conv_state_indices=conv_state_indices, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2) @@ -872,11 +881,18 @@ def _ref_causal_conv1d_update_cpu_multi( @pytest.mark.skipif( - not torch.cpu._is_amx_tile_supported(), - reason="requires AMX support", + not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()), + reason="native causal conv1d requires AMX or Arm BF16 support", ) @pytest.mark.parametrize( - ("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"), + ( + "batch_size, " + "seq_len, " + "accepted_counts, " + "has_bias, " + "silu_activation, " + "is_weight_packed" + ), [ (1, 1, [1], False, False, False), (1, 1, [1], True, True, True), @@ -893,7 +909,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python( accepted_counts: list[int], has_bias: bool, silu_activation: bool, - is_vnni: bool, + is_weight_packed: bool, ) -> None: dim = 96 state_len = seq_len + 2 @@ -911,7 +927,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python( ) conv_states = conv_states_ref.clone() - conv_weight = _maybe_pack_conv_weight(weight, is_vnni) + conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed) out = ops.causal_conv1d_update_cpu( x=x, conv_states=conv_states, @@ -919,7 +935,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python( bias=bias, silu_activation=silu_activation, conv_state_indices=conv_state_indices, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, num_accepted_tokens=num_accepted_tokens, ) ref_out = _ref_causal_conv1d_update_cpu_multi( @@ -937,8 +953,8 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python( @pytest.mark.skipif( - not torch.cpu._is_amx_tile_supported(), - reason="requires AMX support", + not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()), + reason="native causal conv1d requires AMX or Arm BF16 support", ) @pytest.mark.parametrize("num_accepted", [0, 17]) @torch.inference_mode() @@ -961,14 +977,15 @@ def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count( bias=None, silu_activation=True, conv_state_indices=torch.tensor([0], dtype=torch.int32), - is_vnni=False, + is_weight_packed=False, num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32), ) @pytest.mark.skipif( - not torch.cpu._is_avx512_bf16_supported(), - reason="causal_conv1d_fwd_cpu requires AVX-512BF16 (Intel Xeon or AMD EPYC)", + not (torch.cpu._is_avx512_bf16_supported() or gdn_attention.is_arm_bf16()), + reason="causal_conv1d_fwd_cpu requires AVX-512BF16" + " (Intel Xeon or AMD EPYC) or Arm BF16 support", ) @pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS) @torch.inference_mode() @@ -977,13 +994,13 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) -> matches the single-call result. Regression test for ``causal_conv1d_fwd_varlen_kernel_impl`` (``conv.cpp``) - ignoring the carried conv state on continued chunks. Runs on any - AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles. + ignoring the carried conv state on continued chunks. Runs on aarch64 cpus with + bf16 support or any AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles. """ state_len = CONV_KERNEL - 1 x, weight, bias = _conv_inputs(total_tokens) - def amx(x_seg, conv_states, has_init): + def native(x_seg, conv_states, has_init): seq = x_seg.shape[0] return ops.causal_conv1d_fwd_cpu( x=x_seg.transpose(0, 1), # [dim, seq]; stride(-2)==1 (view of [seq,dim]) @@ -994,30 +1011,30 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) -> cache_indices=torch.tensor([0], dtype=torch.int32), has_initial_state=torch.tensor([has_init]), silu_activation=True, - is_vnni=False, + is_weight_packed=False, ).contiguous() # conv_state layout passed by the AMX branch: [num_slots, dim, state_len]. cs_full = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype) - out_full = amx(x, cs_full, False) + out_full = native(x, cs_full, False) cs_split = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype) - out1 = amx(x[:split], cs_split, False) - out2 = amx(x[split:], cs_split, True) + out1 = native(x[:split], cs_split, False) + out2 = native(x[split:], cs_split, True) out_split = torch.cat([out1, out2], dim=1) torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2) @pytest.mark.skipif( - not torch.cpu._is_amx_tile_supported(), - reason="requires AMX support", + not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()), + reason="native causal conv1d requires AMX or Arm BF16 support", ) @torch.inference_mode() def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None: state_len = CONV_KERNEL - 1 wide_state_len = state_len + 5 - is_vnni = True + is_weight_packed = True seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5] total_tokens = sum(seq_lens) x, weight, bias = _conv_inputs(total_tokens) @@ -1034,7 +1051,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None: wide_state[:, :, state_len:].fill_(7) wide_tail = wide_state[:, :, state_len:].clone() - conv_weight = _maybe_pack_conv_weight(weight, is_vnni) + conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed) out_narrow = ops.causal_conv1d_fwd_cpu( x=x.transpose(0, 1), weight=conv_weight, @@ -1044,7 +1061,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None: cache_indices=cache_indices, has_initial_state=has_initial_state, silu_activation=True, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) out_wide = ops.causal_conv1d_fwd_cpu( x=x.transpose(0, 1), @@ -1055,7 +1072,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None: cache_indices=cache_indices, has_initial_state=has_initial_state, silu_activation=True, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2) @@ -1100,15 +1117,23 @@ def test_batch_memcpy_cpu_fallback() -> None: # --------------------------------------------------------------------------- -# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any -# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time. +# C++ conv (conv.cpp) uses AArch64BF16 or VDPBF16PS, not AMX tiles, so it can +# run on any AVX-512BF16 CPU; weight is packed on this same predicate at load time. # --------------------------------------------------------------------------- _HAS_AVX512_BF16 = torch.cpu._is_avx512_bf16_supported() _STATE_LEN = CONV_KERNEL - 1 -CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]] +CONV_EQUIV_SEQ_LENS = [ + [1], + [7], + [64], + [65], + [1, 2, 3], + [63, 64, 65], + [128, 129], +] def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"): @@ -1153,10 +1178,10 @@ def _run_prefill_torch(x, weight, bias, seq_lens): return out.transpose(0, 1).contiguous(), conv_states -def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False): +def _run_prefill_cpp(x, weight, bias, seq_lens, is_weight_packed=False): num_seqs = len(seq_lens) - packed_w = ops.causal_conv1d_weight_pack(weight) if is_vnni else weight - if is_vnni: + packed_w = ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight + if is_weight_packed: # C++-branch layout: kv-cache "SD" [slots, state_len, dim] transposed to # [slots, dim, state_len] (a non-contiguous view). conv_state = torch.zeros( @@ -1176,7 +1201,7 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False): cache_indices=torch.arange(num_seqs, dtype=torch.int32), has_initial_state=torch.zeros(num_seqs, dtype=torch.bool), silu_activation=True, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) return out.transpose(0, 1).contiguous(), conv_state @@ -1245,7 +1270,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state( cache_indices=state_indices, has_initial_state=has_initial_state, silu_activation=True, - is_vnni=False, + is_weight_packed=False, ) monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True) out_ds = gdn_attention._causal_conv1d_fwd_cpu( @@ -1257,7 +1282,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state( cache_indices=state_indices, has_initial_state=has_initial_state, silu_activation=True, - is_vnni=False, + is_weight_packed=False, ) torch.testing.assert_close(out_ds, out_sd) @@ -1302,7 +1327,7 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback( bias=None, silu_activation=False, conv_state_indices=state_indices, - is_vnni=False, + is_weight_packed=False, num_accepted_tokens=num_accepted_tokens, ) @@ -1321,7 +1346,8 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback( @pytest.mark.skipif( - not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" + not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()), + reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support", ) @pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS) @torch.inference_mode() @@ -1335,7 +1361,8 @@ def test_conv_cpp_matches_torch(seq_lens): @pytest.mark.skipif( - not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" + not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()), + reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support", ) @pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS) @torch.inference_mode() @@ -1354,89 +1381,114 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens): @pytest.mark.skipif( - not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" + not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()), + reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support", ) @pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS) @torch.inference_mode() -def test_conv_cpp_vnni_packed_matches_torch(seq_lens): - """The exact runtime prefill sequence (VNNI-packed weight + SD-layout - conv_state view + is_vnni=True) must match the torch fallback. Validates - the packing + layout handoff on any AVX-512BF16 CPU.""" +def test_conv_cpp_weight_packed_matches_torch(seq_lens): + """The exact runtime prefill sequence (packed weight + SD-layout + conv_state view + is_weight_packed=True) must match the torch fallback. Validates + the native packing + layout handoff.""" x, weight, bias = _conv_inputs(sum(seq_lens)) out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens) - out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True) - torch.testing.assert_close(out_vnni, out_torch, atol=1e-2, rtol=1e-2) + out_weight_packed, _ = _run_prefill_cpp( + x, weight, bias, seq_lens, is_weight_packed=True + ) + torch.testing.assert_close(out_weight_packed, out_torch, atol=1e-2, rtol=1e-2) @pytest.mark.skipif( - not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" + not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()), + reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support", ) @pytest.mark.parametrize("batch", DECODE_BATCH_SIZES) +@pytest.mark.parametrize( + ("dim", "has_bias", "silu_activation", "is_weight_packed"), + [ + (32, False, False, False), + (32, True, True, True), + (96, False, True, True), + (96, True, False, False), + (8192, False, True, False), + (8192, True, True, True), + ], +) @torch.inference_mode() -def test_conv_update_cpp_matches_torch(batch): - """Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch, - including the in-place conv_state update (the next-step handoff).""" +def test_conv_update_cpp_matches_torch( + batch, dim, has_bias, silu_activation, is_weight_packed +): + """Decode preserves indexed, padded state across successive signed inputs.""" from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( causal_conv1d_update_torch, ) - x = tensor_cache(batch * CONV_DIM, torch.bfloat16).view(batch, CONV_DIM) - weight = tensor_cache(CONV_DIM * CONV_KERNEL, torch.bfloat16).view( - CONV_DIM, CONV_KERNEL - ) - bias = tensor_cache(CONV_DIM, torch.bfloat16) - conv_state = tensor_cache(batch * CONV_DIM * _STATE_LEN, torch.bfloat16).view( - batch, CONV_DIM, _STATE_LEN - ) + torch.manual_seed(0) + weight = torch.randn(dim, CONV_KERNEL, dtype=torch.bfloat16) + bias = torch.randn(dim, dtype=torch.bfloat16) if has_bias else None + conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed) + indices = torch.arange(batch, 0, -1, dtype=torch.int32) + storage_ref = torch.randn(batch + 2, _STATE_LEN + 2, dim, dtype=torch.bfloat16) + storage_cpp = storage_ref.clone() + cs_cpp = storage_cpp[:, :_STATE_LEN].transpose(1, 2) - cs_torch = conv_state.clone() - out_torch = causal_conv1d_update_torch( - x=x.unsqueeze(-1), - conv_state=cs_torch, - weight=weight, - bias=bias, - activation="silu", - ).squeeze(-1) - - cs_cpp = conv_state.clone() - out_cpp = ops.causal_conv1d_update_cpu( - x=x.contiguous(), - conv_states=cs_cpp, - weight=weight, - bias=bias, - silu_activation=True, - conv_state_indices=torch.arange(batch, dtype=torch.int32), - is_vnni=False, - ) - torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2) - torch.testing.assert_close(cs_cpp, cs_torch, atol=1e-2, rtol=1e-2) + for _ in range(3): + x = torch.randn(batch, dim, dtype=torch.bfloat16) + cs_torch = storage_ref[indices, :_STATE_LEN].transpose(1, 2).contiguous() + out_torch = causal_conv1d_update_torch( + x=x.unsqueeze(-1), + conv_state=cs_torch, + weight=weight, + bias=bias, + activation="silu" if silu_activation else None, + ).squeeze(-1) + storage_ref[indices, :_STATE_LEN] = cs_torch.transpose(1, 2) + out_cpp = ops.causal_conv1d_update_cpu( + x=x, + conv_states=cs_cpp, + weight=conv_weight, + bias=bias, + silu_activation=silu_activation, + conv_state_indices=indices, + is_weight_packed=is_weight_packed, + ) + torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2) + torch.testing.assert_close(storage_cpp, storage_ref, atol=0, rtol=0) @pytest.mark.skipif( - not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" + not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()), + reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support", ) @torch.inference_mode() def test_conv_weight_pack_roundtrip_unpacked_matches(): """`causal_conv1d_weight_pack` repacks the (dim, width) weight, so the C++ conv op - with packed weight + is_vnni=True must equal unpacked weight + is_vnni=False. - Guards the spec-decode contract: the runtime VNNI-packs `layer.conv1d.weight` - in place and stashes the original as `_cpu_unpacked_conv_weight` for the torch - spec-decode path, so both must produce identical math. + with packed weight + is_weight_packed=True must equal unpacked weight + + is_weight_packed=False. Guards the spec-decode contract: the runtime packs + `layer.conv1d.weight` in place and stashes the original as + `_cpu_unpacked_conv_weight` for the torch spec-decode path, so both must produce + identical math. """ seq_lens = [7, 64, 65] x, weight, bias = _conv_inputs(sum(seq_lens)) - out_unpacked, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False) - out_packed, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True) - torch.testing.assert_close(out_packed, out_unpacked, atol=1e-2, rtol=1e-2) + out_unpacked, state_unpacked = _run_prefill_cpp( + x, weight, bias, seq_lens, is_weight_packed=False + ) + out_packed, state_packed = _run_prefill_cpp( + x, weight, bias, seq_lens, is_weight_packed=True + ) + torch.testing.assert_close(out_packed, out_unpacked, atol=0, rtol=0) + torch.testing.assert_close(state_packed, state_unpacked, atol=0, rtol=0) @pytest.mark.skipif( - not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16" + not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()), + reason="C++ causal_conv1d requires AVX-512BF16 or AArch64 BF16", ) @torch.inference_mode() def test_spec_decode_unpacked_conv_weight_stash(): - """Spec-decode correctness: AVX-512BF16 CPUs VNNI-pack ``conv1d.weight`` in place - at load time and stash the original ``(dim, width)`` tensor as + """Spec-decode correctness: AVX-512BF16 and AArch64 BF16 CPUs pack ``conv1d.weight`` + in place at load time and stash the original ``(dim, width)`` tensor as ``_cpu_unpacked_conv_weight``; the spec-decode path must use that stash since reading the packed weight directly produces garbage. Verifies the recovered weight equals the original and that torch F.conv1d agrees with the C++ conv @@ -1460,7 +1512,7 @@ def test_spec_decode_unpacked_conv_weight_stash(): # 1. The stash must exist and equal the original (dim, width) weight. assert hasattr(conv, "_cpu_unpacked_conv_weight"), ( "dispatch_cpu_unquantized_gemm did not stash _cpu_unpacked_conv_weight " - "on an AVX-512BF16 CPU" + "on an AArch64-BF16 or AVX-512BF16 CPU" ) stash = conv._cpu_unpacked_conv_weight torch.testing.assert_close(stash, orig_2d, atol=0, rtol=0) @@ -1469,10 +1521,10 @@ def test_spec_decode_unpacked_conv_weight_stash(): recovered = getattr(conv, "_cpu_unpacked_conv_weight", None) assert recovered is not None # conv.weight is now packed; using it directly (the bug) would differ. - packed_weight = conv.weight # [dim, 1, width], VNNI-packed contents + packed_weight = conv.weight # [dim, 1, width], packed contents # 3. Spec-path torch conv with the recovered unpacked weight must match the - # nonspec C++ conv with the packed weight (is_vnni=True). + # nonspec C++ conv with the packed weight (is_weight_packed=True). seq_lens = [7, 65] total = sum(seq_lens) x = tensor_cache(total * CONV_DIM, torch.bfloat16).view(total, CONV_DIM) @@ -1494,7 +1546,7 @@ def test_spec_decode_unpacked_conv_weight_stash(): cache_indices=torch.arange(num_seqs, dtype=torch.int32), has_initial_state=torch.zeros(num_seqs, dtype=torch.bool), silu_activation=True, - is_vnni=True, + is_weight_packed=True, ) .transpose(0, 1) .contiguous() diff --git a/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py b/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py index 014a9ca25d..8017be2eea 100644 --- a/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py +++ b/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py @@ -6,10 +6,61 @@ import pytest import torch from vllm.model_executor.layers import utils -from vllm.platforms import current_platform +from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import set_default_torch_dtype +def test_dispatch_prepacks_arm_bf16_causal_conv(monkeypatch): + monkeypatch.setattr( + current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM + ) + monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True}) + monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False) + + packed = torch.randn(32, 4, dtype=torch.bfloat16) + pack_inputs = [] + + def pack(weight): + pack_inputs.append(weight.clone()) + return packed + + monkeypatch.setattr(utils.ops, "causal_conv1d_weight_pack", pack) + original = torch.randn(32, 1, 4, dtype=torch.bfloat16) + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter(original.clone(), requires_grad=False) + + utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False) + + assert len(pack_inputs) == 1 + # Check causal_conv1d_weight_pack received the expected weights + torch.testing.assert_close(pack_inputs[0], original.view(32, 4)) + # Check we have correctly stashed the unpacked weights + torch.testing.assert_close(layer._cpu_unpacked_conv_weight, original.view(32, 4)) + # Check we have correctly stored the packed weights + assert layer.weight.data_ptr() == packed.data_ptr() + + +def test_dispatch_does_not_pack_3d_expert_weight(monkeypatch): + monkeypatch.setattr( + current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM + ) + monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True}) + monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False) + pack_calls = [] + monkeypatch.setattr( + utils.ops, + "causal_conv1d_weight_pack", + lambda weight: pack_calls.append(weight), + ) + + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter( + torch.randn(8, 32, 4, dtype=torch.bfloat16), requires_grad=False + ) + utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False) + assert pack_calls == [] + + @pytest.fixture(scope="module") def _mock_zentorch_linear_unary(): """Register a mock zentorch_linear_unary op when zentorch is not installed. diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index c21f85c029..51a810650f 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -3666,7 +3666,7 @@ def causal_conv1d_fwd_cpu( cache_indices: torch.Tensor | None, has_initial_state: torch.Tensor | None, silu_activation: bool, - is_vnni: bool, + is_weight_packed: bool, ) -> torch.Tensor: return torch.ops._C.causal_conv1d_fwd_cpu( x, @@ -3678,7 +3678,7 @@ def causal_conv1d_fwd_cpu( has_initial_state, silu_activation, -1, - is_vnni, + is_weight_packed, ) @@ -3689,7 +3689,7 @@ def causal_conv1d_update_cpu( bias: torch.Tensor | None, silu_activation: bool, conv_state_indices: torch.Tensor | None, - is_vnni: bool, + is_weight_packed: bool, num_accepted_tokens: torch.Tensor | None = None, ) -> torch.Tensor: return torch.ops._C.causal_conv1d_update_cpu( @@ -3701,7 +3701,7 @@ def causal_conv1d_update_cpu( num_accepted_tokens, conv_state_indices, -1, - is_vnni, + is_weight_packed, ) diff --git a/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py b/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py index f3f3a33ab1..08b03e31d1 100644 --- a/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py +++ b/vllm/model_executor/layers/mamba/ops/cpu/gdn_attention.py @@ -14,7 +14,6 @@ from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( ) from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import ( causal_conv1d_update_cpu, - causal_conv1d_update_torch, ) from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.torch_utils import ( @@ -56,7 +55,7 @@ def _causal_conv1d_fwd_cpu( cache_indices: torch.Tensor, has_initial_state: torch.Tensor, silu_activation: bool, - is_vnni: bool, + is_weight_packed: bool, ) -> torch.Tensor: if not is_conv_state_dim_first(): return ops.causal_conv1d_fwd_cpu( @@ -68,7 +67,7 @@ def _causal_conv1d_fwd_cpu( cache_indices=cache_indices, has_initial_state=has_initial_state, silu_activation=silu_activation, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) scratch, identity_indices = _stage_ds_conv_states(conv_states, cache_indices) @@ -81,7 +80,7 @@ def _causal_conv1d_fwd_cpu( cache_indices=identity_indices, has_initial_state=has_initial_state, silu_activation=silu_activation, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, ) _scatter_ds_conv_states(conv_states, cache_indices, scratch) return out @@ -95,7 +94,7 @@ def _causal_conv1d_update_cpu( bias: torch.Tensor | None, silu_activation: bool, conv_state_indices: torch.Tensor, - is_vnni: bool, + is_weight_packed: bool, num_accepted_tokens: torch.Tensor | None = None, ) -> torch.Tensor: if not is_conv_state_dim_first(): @@ -106,7 +105,7 @@ def _causal_conv1d_update_cpu( bias=bias, silu_activation=silu_activation, conv_state_indices=conv_state_indices, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, num_accepted_tokens=num_accepted_tokens, ) @@ -118,13 +117,20 @@ def _causal_conv1d_update_cpu( bias=bias, silu_activation=silu_activation, conv_state_indices=identity_indices, - is_vnni=is_vnni, + is_weight_packed=is_weight_packed, num_accepted_tokens=num_accepted_tokens, ) _scatter_ds_conv_states(conv_states, conv_state_indices, scratch) return out +def is_arm_bf16() -> bool: + return ( + current_platform.get_cpu_architecture() == CpuArchEnum.ARM + and torch.cpu.get_capabilities().get("bf16", False) + ) + + def cpu_gdn_attention_core( mixed_qkv: torch.Tensor, b: torch.Tensor, @@ -201,9 +207,10 @@ def _cpu_gdn_attention_nonspec( assert state_indices_tensor is not None assert query_start_loc is not None - # C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any - # AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time. - use_cpp_conv = torch.cpu._is_avx512_bf16_supported() + # C++ conv (conv.cpp) runs on aarch64 with bf16 support and on x86 uses VDPBF16PS, + # not AMX tiles, so it runs on any AVX-512BF16 CPU. The weights are packed on this + # same predicate at load time. + use_cpp_conv = torch.cpu._is_avx512_bf16_supported() or is_arm_bf16() conv_state = layer.kv_cache[0] if use_cpp_conv: @@ -244,28 +251,17 @@ def _cpu_gdn_attention_nonspec( bias=layer.conv1d.bias, silu_activation=(layer.activation == "silu"), conv_state_indices=decode_state_indices, - is_vnni=True, + is_weight_packed=True, ) else: - if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: - decode_conv_state = conv_state[decode_state_indices].contiguous() - decode_mixed_qkv = causal_conv1d_update_torch( - x=decode_mixed_qkv.unsqueeze(-1), - conv_state=decode_conv_state, - weight=conv_weights, - bias=layer.conv1d.bias, - activation=layer.activation, - ).squeeze(-1) - conv_state[decode_state_indices] = decode_conv_state - else: - decode_mixed_qkv = causal_conv1d_update_cpu( - x=decode_mixed_qkv, - conv_state=conv_state, - weight=conv_weights, - bias=layer.conv1d.bias, - activation=layer.activation, - conv_state_indices=decode_state_indices, - ) + decode_mixed_qkv = causal_conv1d_update_cpu( + x=decode_mixed_qkv, + conv_state=conv_state, + weight=conv_weights, + bias=layer.conv1d.bias, + activation=layer.activation, + conv_state_indices=decode_state_indices, + ) query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv) @@ -315,7 +311,7 @@ def _cpu_gdn_attention_nonspec( cache_indices=prefill_state_indices, has_initial_state=prefill_has_initial_state, silu_activation=layer.activation == "silu", - is_vnni=True, + is_weight_packed=True, ).transpose(0, 1) else: prefill_mixed_qkv = causal_conv1d_torch( @@ -383,9 +379,9 @@ def _ssm_state_view(layer) -> torch.Tensor: def _unpacked_conv_weight(layer) -> torch.Tensor: """Return the plain (dim, width) conv weight. - On AMX the conv1d weight is VNNI-packed in place at load time (only usable - by the AMX C++ kernel), so the torch spec-decode path relies on the - un-packed copy stashed by ``dispatch_cpu_unquantized_gemm``. + On AMX and aarch64+bf16 the conv1d weight is packed in place at load time, + so the torch spec-decode path relies on the un-packed copy stashed by + ``dispatch_cpu_unquantized_gemm``. """ w = getattr(layer.conv1d, "_cpu_unpacked_conv_weight", None) if w is not None: @@ -505,13 +501,15 @@ def _spec_forward( seq_starts = spec_qsl_cpu[:-1] seq_lens = spec_qsl_cpu[1:] - spec_qsl_cpu[:-1] + is_amx = torch.cpu._is_amx_tile_supported() + # ---- 1. Convolution (per-sequence rolling buffer) ---- dim = mixed_qkv_spec.size(-1) bias = layer.conv1d.bias silu = layer.activation == "silu" can_use_native_conv = ( - torch.cpu._is_amx_tile_supported() + (is_amx or is_arm_bf16()) and width == 4 and num_spec_decodes > 0 and bool(torch.all(seq_lens == seq_lens[0]).item()) @@ -528,7 +526,7 @@ def _spec_forward( conv_state_indices=spec_state_indices[:num_spec_decodes, 0] .to("cpu", torch.int32) .contiguous(), - is_vnni=True, + is_weight_packed=True, num_accepted_tokens=num_accepted[:num_spec_decodes].to("cpu", torch.int32), ).view_as(mixed_qkv_spec) else: @@ -603,9 +601,9 @@ def _spec_aware_nonspec( assert query_start_loc is not None state_indices_tensor = state_indices_tensor.contiguous() - is_amx = torch.cpu._is_amx_tile_supported() + can_use_native_conv = torch.cpu._is_amx_tile_supported() or is_arm_bf16() - if not is_amx: + if not can_use_native_conv: conv_weights = _unpacked_conv_weight(layer) num_decodes = attn_metadata_i.num_decodes @@ -618,7 +616,8 @@ def _spec_aware_nonspec( decode_b = b[:num_decode_tokens] decode_a = a[:num_decode_tokens] decode_state_indices = state_indices_tensor[:num_decodes] - if is_amx: + + if can_use_native_conv: decode_mixed_qkv = _causal_conv1d_update_cpu( x=decode_mixed_qkv, conv_states=conv_buf, @@ -626,30 +625,19 @@ def _spec_aware_nonspec( bias=layer.conv1d.bias, silu_activation=layer.activation == "silu", conv_state_indices=decode_state_indices, - is_vnni=True, + is_weight_packed=True, ) else: # Only the first ``width-1`` columns hold the real conv state. conv_state_view = conv_buf[:, :, : width - 1] - if current_platform.get_cpu_architecture() == CpuArchEnum.ARM: - decode_conv_state = conv_state_view[decode_state_indices].contiguous() - decode_mixed_qkv = causal_conv1d_update_torch( - x=decode_mixed_qkv.unsqueeze(-1), - conv_state=decode_conv_state, - weight=conv_weights, - bias=layer.conv1d.bias, - activation=layer.activation, - ).squeeze(-1) - conv_state_view[decode_state_indices] = decode_conv_state - else: - decode_mixed_qkv = causal_conv1d_update_cpu( - x=decode_mixed_qkv, - conv_state=conv_state_view, - weight=conv_weights, - bias=layer.conv1d.bias, - activation=layer.activation, - conv_state_indices=decode_state_indices, - ) + decode_mixed_qkv = causal_conv1d_update_cpu( + x=decode_mixed_qkv, + conv_state=conv_state_view, + weight=conv_weights, + bias=layer.conv1d.bias, + activation=layer.activation, + conv_state_indices=decode_state_indices, + ) query, key, value = layer.rearrange_mixed_qkv(decode_mixed_qkv) attn_out = ops.fused_sigmoid_gating_delta_rule_update_cpu( @@ -685,7 +673,8 @@ def _spec_aware_nonspec( prefill_has_initial_state = has_initial_state[ num_decodes : num_decodes + num_prefills ] - if is_amx: + + if can_use_native_conv: prefill_mixed_qkv = _causal_conv1d_fwd_cpu( x=prefill_mixed_qkv.transpose(0, 1), weight=layer.conv1d.weight, @@ -695,7 +684,7 @@ def _spec_aware_nonspec( cache_indices=prefill_state_indices, has_initial_state=prefill_has_initial_state, silu_activation=layer.activation == "silu", - is_vnni=True, + is_weight_packed=True, ).transpose(0, 1) else: prefill_mixed_qkv = causal_conv1d_torch( @@ -757,8 +746,9 @@ def _spec_aware_nonspec_subset( prefill_state_indices = prefill_state_indices.contiguous() is_amx = torch.cpu._is_amx_tile_supported() + can_use_native_conv = is_amx or is_arm_bf16() - if is_amx: + if can_use_native_conv: conv_out = _causal_conv1d_fwd_cpu( x=mixed_qkv.transpose(0, 1), weight=layer.conv1d.weight, @@ -768,7 +758,7 @@ def _spec_aware_nonspec_subset( cache_indices=prefill_state_indices, has_initial_state=has_initial_state, silu_activation=layer.activation == "silu", - is_vnni=True, + is_weight_packed=True, ).transpose(0, 1) else: conv_weights = _unpacked_conv_weight(layer) diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index af8b7d4d5c..d4ee153fb2 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -475,13 +475,27 @@ def dispatch_cpu_unquantized_gemm( # Skip CPU GEMM dispatch for non-2D weights (e.g. MoE 3D expert weights). # These layers are handled by their own specialized methods. if layer.weight.ndim != 2: - # this is not a linear layer - # For now it should be a causal_conv1d op or MoE 3D expert weights - # The C++ causal_conv1d kernels use VDPBF16PS (no AMX tiles), so the - # VNNI weight prepack applies to any AVX-512BF16 CPU, not just AMX - # (e.g. AMD Zen5/Turin). - if torch.cpu._is_avx512_bf16_supported() and hasattr( - ops, "causal_conv1d_weight_pack" + # This is not a linear layer. + # For now it should be a causal_conv1d op or MoE 3D expert weights. + # Causal-conv weights use [dim, 1, width]. + # The C++ causal_conv1d kernels support aarch64 with bf16 and + # x86 CPUs that use VDPBF16PS (no AMX tiles). So the + # weight prepack applies to aarch64 with bf16 and any AVX-512BF16 CPU, + # not just AMX (e.g. AMD Zen5/Turin). + is_causal_conv1d_weight = ( + layer.weight.ndim == 3 + and layer.weight.size(1) == 1 + and layer.weight.size(2) == 4 + and layer.weight.dtype == torch.bfloat16 + ) + is_arm_bf16 = ( + current_platform.get_cpu_architecture() == CpuArchEnum.ARM + and torch.cpu.get_capabilities().get("bf16", False) + ) + if ( + (is_arm_bf16 or torch.cpu._is_avx512_bf16_supported()) + and is_causal_conv1d_weight + and hasattr(ops, "causal_conv1d_weight_pack") ): # prepack conv weight unpacked = ( @@ -493,7 +507,7 @@ def dispatch_cpu_unquantized_gemm( .clone() ) # Stash the un-packed (dim, width) weight so the speculative-decode - # GDN path (which uses torch conv, not the C++ kernel) can use it. + # and unsupported-layout fallbacks can still use ordinary weights. layer._cpu_unpacked_conv_weight = unpacked layer.weight.data = ops.causal_conv1d_weight_pack(unpacked) return From effdea9a5303f02759ba243ee624bf0a9b2576b0 Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Wed, 30 Sep 2026 14:27:12 +0200 Subject: [PATCH 117/248] [MM][Mistral] Add compile support for Pixtral vision encoders (#57168) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jakub Zakrzewski Co-authored-by: Nicolò Lucchesi Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../fullgraph/test_multimodal_compile.py | 76 ++++++++++++++++++- vllm/model_executor/models/pixtral.py | 38 ++++++++-- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/tests/compile/fullgraph/test_multimodal_compile.py b/tests/compile/fullgraph/test_multimodal_compile.py index c5dc6f96b2..8f76a18c69 100644 --- a/tests/compile/fullgraph/test_multimodal_compile.py +++ b/tests/compile/fullgraph/test_multimodal_compile.py @@ -1,10 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from functools import partial + import pytest +from tests.models.utils import dummy_hf_overrides from vllm.compilation.counter import compilation_counter from vllm.config import VllmConfig -from vllm.config.compilation import CompilationMode +from vllm.config.compilation import CompilationMode, CUDAGraphMode from vllm.platforms import current_platform @@ -14,6 +17,77 @@ def test_compile(): assert not vllm_config.compilation_config.compile_mm_encoder +@pytest.mark.forked +@pytest.mark.core_model +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") +@pytest.mark.parametrize( + ("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"), + [ + pytest.param( + "mistralai/Pixtral-12B-2409", + None, + "mistral", + "mistral", + "PixtralForConditionalGeneration", + id="pixtral", + ), + pytest.param( + "mistralai/Mistral-Small-3.1-24B-Instruct-2503", + "mistral-community/pixtral-12b", + "hf", + "auto", + "Mistral3ForConditionalGeneration", + id="mistral3-hf", + ), + ], +) +@pytest.mark.parametrize( + ("compile_mm_encoder", "expected_models"), + [ + # The dummy model has two compiled text components. Enabling encoder + # compilation adds its single vision transformer block. + pytest.param(False, 2, id="disabled"), + pytest.param(True, 3, id="enabled"), + ], +) +def test_pixtral_compilation( + vllm_runner, + monkeypatch, + model: str, + tokenizer: str | None, + config_format: str, + tokenizer_mode: str, + model_arch: str, + compile_mm_encoder: bool, + expected_models: int, +): + """Test compilation for both Pixtral vision encoder implementations.""" + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0") + + with ( + compilation_counter.expect(num_models_seen=expected_models), + vllm_runner( + model, + tokenizer_name=tokenizer, + tokenizer_mode=tokenizer_mode, + config_format=config_format, + load_format="dummy", + hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch), + max_model_len=4096, + limit_mm_per_prompt={"image": 1}, + gpu_memory_utilization=0.8, + attention_backend="FLASH_ATTN", + compilation_config={ + "mode": CompilationMode.VLLM_COMPILE, + "compile_mm_encoder": compile_mm_encoder, + "cudagraph_mode": CUDAGraphMode.NONE, + }, + ) as _, + ): + pass + + # forked needed to workaround https://github.com/vllm-project/vllm/issues/21073 @pytest.mark.forked @pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda") diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 521df62a92..fb10026ad1 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -21,6 +21,10 @@ from transformers.models.pixtral.image_processing_pixtral import ( ) from transformers.models.pixtral.modeling_pixtral import apply_rotary_pos_emb +from vllm.compilation.decorators import ( + should_torch_compile_mm_encoder, + support_torch_compile, +) from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions from vllm.distributed import divide, get_tensor_model_parallel_world_size @@ -829,6 +833,16 @@ class Attention(nn.Module): return out +@support_torch_compile( + dynamic_arg_dims={ + "x": 1, + "freqs_cis": 0, + "cu_seqlens": 0, + "sequence_lengths": 0, + }, + enable_if=should_torch_compile_mm_encoder, + is_encoder=True, +) class TransformerBlock(nn.Module): def __init__( self, @@ -1298,7 +1312,7 @@ class PixtralHFAttention(nn.Module): def forward( self, hidden_states: torch.Tensor, - position_embeddings: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], cu_seqlens: torch.Tensor, max_seqlen: torch.Tensor, sequence_lengths: torch.Tensor | None, @@ -1329,6 +1343,17 @@ class PixtralHFAttention(nn.Module): return attn_output, None +@support_torch_compile( + dynamic_arg_dims={ + "hidden_states": 1, + "rotary_pos_emb_cos": 0, + "rotary_pos_emb_sin": 0, + "cu_seqlens": 0, + "sequence_lengths": 0, + }, + enable_if=should_torch_compile_mm_encoder, + is_encoder=True, +) class PixtralHFTransformerBlock(nn.Module): def __init__( self, @@ -1355,14 +1380,15 @@ class PixtralHFTransformerBlock(nn.Module): def forward( self, hidden_states: torch.Tensor, - position_embeddings: torch.Tensor, + rotary_pos_emb_cos: torch.Tensor, + rotary_pos_emb_sin: torch.Tensor, cu_seqlens: torch.Tensor, max_seqlen: torch.Tensor, sequence_lengths: torch.Tensor | None, ) -> torch.Tensor: r, _ = self.attention.forward( self.attention_norm(hidden_states), - position_embeddings=position_embeddings, + position_embeddings=(rotary_pos_emb_cos, rotary_pos_emb_sin), cu_seqlens=cu_seqlens, max_seqlen=max_seqlen, sequence_lengths=sequence_lengths, @@ -1403,18 +1429,20 @@ class PixtralHFTransformer(nn.Module): def forward( self, x: torch.Tensor, - position_embeddings: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], cu_seqlens: torch.Tensor, max_seqlen: torch.Tensor, sequence_lengths: torch.Tensor | None, return_all_hidden_states: bool, ) -> torch.Tensor: hidden_states_pool = [x] + rotary_pos_emb_cos, rotary_pos_emb_sin = position_embeddings for layer in self.layers: x = layer( x, - position_embeddings, + rotary_pos_emb_cos, + rotary_pos_emb_sin, cu_seqlens, max_seqlen, sequence_lengths, From b22494cc0cb4bd9db4a62fb107d92429a4a3249d Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:41:53 +0100 Subject: [PATCH 118/248] [Docs] Reinstate docs build gate for PRs (#59411) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .github/mergify.yml | 1 - docs/contributing/README.md | 3 +++ docs/pre_run_check.sh | 12 +++--------- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/mergify.yml b/.github/mergify.yml index 1267f0264d..707936b93c 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -5,7 +5,6 @@ pull_request_rules: - label != stale - -closed - or: - - files~=^[^/]+\.md$ - files~=^docs/ - files~=^examples/ actions: diff --git a/docs/contributing/README.md b/docs/contributing/README.md index d0c2e25315..89f29afce0 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -135,6 +135,9 @@ API_AUTONAV_EXCLUDE=vllm mkdocs serve # API ref off (~15 seconds) Once you see a `Serving on http://
:/` line in the logs, the live preview is ready! Open that address in your browser to see it — `http://127.0.0.1:8000/` by default, or whichever host/port you passed to `-a`. +On pull requests, Read the Docs builds a documentation preview only if the PR changes files that affect the docs (e.g. `docs/`, `examples/` or `mkdocs.yaml`) and the `pre-commit` checks have not failed. +Changes to `vllm/` alone do not trigger a build, so if your PR changes the API or CLI reference, add the `build-docs` label to force one. + For additional features and advanced configurations, refer to the: - [MkDocs documentation](https://www.mkdocs.org/) diff --git a/docs/pre_run_check.sh b/docs/pre_run_check.sh index d7cf214993..04f4a642ef 100644 --- a/docs/pre_run_check.sh +++ b/docs/pre_run_check.sh @@ -1,4 +1,3 @@ -# shellcheck disable=SC2317 # code after the exit/exit-183 gate below is intentionally dead pending re-enablement (see line comment) if [ "$READTHEDOCS_VERSION_TYPE" != "external" ]; then echo "Not a PR build (version type=$READTHEDOCS_VERSION_TYPE); skipping pre-run-check gate." exit 0 @@ -11,24 +10,19 @@ if [ -n "$GITHUB_TOKEN" ]; then CURL_AUTH=(-H "Authorization: Bearer $GITHUB_TOKEN") fi -# Docs builds are now manually enabled via the 'build-docs' label. +# The 'build-docs' label forces a docs build, bypassing the checks below. echo "Checking for the 'build-docs' label on PR #${READTHEDOCS_VERSION_NAME}..." LABELS=$(curl -sS "${CURL_AUTH[@]}" "https://api.github.com/repos/vllm-project/vllm/issues/${READTHEDOCS_VERSION_NAME}/labels" | python3 -c "import sys, json; print('\n'.join(l.get('name', '') for l in json.load(sys.stdin)))") if printf '%s\n' "$LABELS" | grep -qx "build-docs"; then - echo "PR has the 'build-docs' label; continuing build." + echo "PR has the 'build-docs' label; forcing build." exit 0 -else - echo "PR does not have the 'build-docs' label; cancelling build." - # See https://docs.readthedocs.com/platform/latest/guides/build/skip-build.html for info on exit code - exit 183 fi -# Everything below this line is effectively disabled as a temporary measure. echo "Checking for changes to docs-affecting files vs origin/main..." DOCS_PATHS=( docs/ # Actual docs content examples/ # Examples are rendered in docs - vllm/ # API & CLI reference + # vllm/ # API & CLI reference (too broad, use 'build-docs' label) requirements/test/cuda.txt # CLI reference (see docs/mkdocs/gen_files/generate_argparse.py) mkdocs.yaml # Affects build process .readthedocs.yaml # Affects build process From f28a5081629377c36cd219a35070555b72279109 Mon Sep 17 00:00:00 2001 From: JulienDarve <86800349+JulienDarve@users.noreply.github.com> Date: Wed, 30 Sep 2026 05:49:25 -0700 Subject: [PATCH 119/248] [Feature][Rust Frontend] Add Shutdown control RPC (#59316) Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: Julien Darve Signed-off-by: Bugen Zhao --- docs/usage/security.md | 2 +- rust/proto/README.md | 2 +- rust/proto/control.proto | 8 +++ rust/src/cmd/src/cli.rs | 6 ++ rust/src/cmd/src/cli/tests.rs | 26 ++++++++ .../examples/external_engine_openai_qwen.rs | 1 + rust/src/server/src/config.rs | 3 + rust/src/server/src/grpc/control.rs | 20 ++++++ rust/src/server/src/grpc/tests.rs | 62 +++++++++++++++++-- rust/src/server/src/lib.rs | 8 ++- 10 files changed, 129 insertions(+), 9 deletions(-) diff --git a/docs/usage/security.md b/docs/usage/security.md index a6c9867755..2b09ee5266 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -375,7 +375,7 @@ An attacker who can reach the gRPC port can: 1. **Run arbitrary inference** via the `Generate` and `GenerateStream` RPCs without any credentials 2. **Mutate engine state** by pausing generation, sleeping the engine, or initiating configured RL weight updates through the `Control` service 3. **Consume GPU and compute resources** by submitting unbounded generation requests -4. **Cause Denial of Service** by exploiting bugs in the gRPC interface that can crash vLLM. +4. **Stop a managed engine** through `Control.Shutdown`, or cause denial of service by exploiting bugs in the gRPC interface. ### Recommendations diff --git a/rust/proto/README.md b/rust/proto/README.md index e792cf79e2..d312e27aca 100644 --- a/rust/proto/README.md +++ b/rust/proto/README.md @@ -16,7 +16,7 @@ the schemas on pull requests, including on forks. ```toml [dependencies] -vllm-proto = "0.3" +vllm-proto = "0.4" ``` For example, import `vllm_proto::inference_client::InferenceClient` or diff --git a/rust/proto/control.proto b/rust/proto/control.proto index ef3218d35c..e42f958582 100644 --- a/rust/proto/control.proto +++ b/rust/proto/control.proto @@ -9,6 +9,10 @@ service Control { rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {} rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {} + // Request managed-engine shutdown using the configured shutdown timeout. + // Returns FAILED_PRECONDITION when this server does not manage the engine process. + rpc Shutdown (ShutdownRequest) returns (ShutdownResponse) {} + // Request lifecycle. rpc Abort (AbortRequest) returns (AbortResponse) {} @@ -86,6 +90,10 @@ message ModelInfo { string tool_call_parser = 25; } +message ShutdownRequest {} +// Acknowledges acceptance, not completion; shutdown may close the connection first. +message ShutdownResponse {} + message AbortRequest { repeated string request_ids = 1; } diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index f57d43a75f..1fab1c93d1 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -557,6 +557,8 @@ impl SharedRuntimeArgs { disable_log_stats: self.disable_log_stats, grpc_port: self.grpc_port, shutdown_timeout, + // The engine is launched and supervised by another process. + manages_engine: false, keep_alive_timeout, profiler, } @@ -572,6 +574,7 @@ impl SharedRuntimeArgs { engine_count: usize, local_input_address: Option, local_output_address: Option, + manages_engine: bool, ) -> Config { let ready_timeout = self.ready_timeout(); let shutdown_timeout = self.shutdown_timeout(); @@ -615,6 +618,7 @@ impl SharedRuntimeArgs { disable_log_stats: self.disable_log_stats, grpc_port: self.grpc_port, shutdown_timeout, + manages_engine, keep_alive_timeout, profiler, } @@ -793,6 +797,8 @@ impl ServeArgs { self.managed_engine.data_parallel_size, local_input_address, local_output_address, + // `--data-parallel-size-local 0` runs the frontend without a local engine. + self.managed_engine.data_parallel_size_local != Some(0), ) } diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index dac00ac983..0802d2062a 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -1889,6 +1889,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { disable_log_stats: false, grpc_port: None, shutdown_timeout: 0ns, + manages_engine: true, keep_alive_timeout: 5s, profiler: None, } @@ -1983,6 +1984,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { disable_log_stats: false, grpc_port: None, shutdown_timeout: 0ns, + manages_engine: true, keep_alive_timeout: 5s, profiler: None, } @@ -2011,6 +2013,29 @@ fn frontend_args_reject_legacy_handshake_flags() { assert!(error.to_string().contains("--handshake-address")); } +#[test] +fn serve_frontend_config_does_not_manage_engine_without_local_engines() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--data-parallel-address", + "10.99.48.128", + "--data-parallel-size", + "2", + "--data-parallel-size-local", + "0", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string()); + + assert!(!config.manages_engine); +} + #[test] fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() { let cli = Cli::try_parse_from([ @@ -2098,6 +2123,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present disable_log_stats: false, grpc_port: None, shutdown_timeout: 0ns, + manages_engine: false, keep_alive_timeout: 5s, profiler: None, } diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 3de227babd..cb5fed4437 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -87,6 +87,7 @@ async fn main() -> Result<()> { disable_log_stats: false, grpc_port: None, shutdown_timeout: Duration::ZERO, + manages_engine: false, keep_alive_timeout: Duration::from_secs(5), profiler: None, }; diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index a9a77a9a9e..b307f8b3ac 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -277,6 +277,9 @@ pub struct Config { pub grpc_port: Option, /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown. pub shutdown_timeout: Duration, + /// Whether the caller manages the engine process and shuts it down when + /// the shutdown token is cancelled. Enables the gRPC `Control.Shutdown` RPC. + pub manages_engine: bool, /// Maximum idle time on a keep-alive HTTP connection before the server /// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s). pub keep_alive_timeout: Duration, diff --git a/rust/src/server/src/grpc/control.rs b/rust/src/server/src/grpc/control.rs index 680ebdfbda..d1b5ac4fe0 100644 --- a/rust/src/server/src/grpc/control.rs +++ b/rust/src/server/src/grpc/control.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use serde_json::Value as JsonValue; use thiserror_ext::AsReport as _; use tokio::sync::Mutex; +use tokio_util::sync::CancellationToken; use tonic::{Code, Request, Response, Status}; use vllm_engine_core_client::EngineCoreClient; use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse; @@ -23,6 +24,7 @@ pub(crate) type ControlGrpcService = ControlServer; pub struct ControlServiceImpl { state: Arc, rl_lock: Mutex<()>, + engine_shutdown: Option, } impl ControlServiceImpl { @@ -30,9 +32,15 @@ impl ControlServiceImpl { Self { state, rl_lock: Mutex::new(()), + engine_shutdown: None, } } + pub fn with_engine_shutdown(mut self, shutdown: Option) -> Self { + self.engine_shutdown = shutdown; + self + } + fn ready(&self) -> &EngineCoreReadyResponse { self.state.engine_core_client().ready_response() } @@ -179,6 +187,18 @@ fn list_loras_status(error: LoraDisabledError) -> Status { #[tonic::async_trait] impl pb::control_server::Control for ControlServiceImpl { + async fn shutdown( + &self, + _request: Request, + ) -> Result, Status> { + let shutdown = self.engine_shutdown.as_ref().ok_or_else(|| { + Status::failed_precondition("this frontend does not manage the engine process") + })?; + tracing::info!("engine shutdown requested via gRPC"); + shutdown.cancel(); + Ok(Response::new(pb::ShutdownResponse {})) + } + async fn get_server_info( &self, _request: Request, diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 21b240c4b1..9f01886062 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -366,6 +366,7 @@ where engine_id, default_ready_response(), backend, + None, move |dealer, push| { boxed_test_future(async move { let add = recv_engine_message(dealer).await; @@ -387,6 +388,7 @@ async fn setup_grpc_service_with_engine_script( engine_id: impl Into, ready: EngineCoreReadyResponse, backend: Arc, + engine_shutdown: Option, script: F, ) -> ( InferenceServer, @@ -425,7 +427,7 @@ where ( InferenceServer::new(InferenceServiceImpl::new(state.clone())) .max_decoding_message_size(crate::DEFAULT_REQUEST_BODY_LIMIT_BYTES), - ControlServer::new(ControlServiceImpl::new(state)), + ControlServer::new(ControlServiceImpl::new(state).with_engine_shutdown(engine_shutdown)), engine_health, engine_task, ) @@ -1920,6 +1922,7 @@ async fn control_reports_server_and_model_info() { b"engine-grpc-info".to_vec(), ready, Arc::new(FakeTextBackend), + None, |_, _| boxed_test_future(async {}), ) .await; @@ -1987,6 +1990,7 @@ async fn control_lora_lifecycle_selects_adapter_for_generation() { b"engine-grpc-lora".to_vec(), ready, Arc::new(FakeTextBackend), + None, |dealer, push| { boxed_test_future(async move { reply_utility_bool(dealer, push, "add_lora", true).await; @@ -2111,6 +2115,7 @@ async fn control_list_loras_requires_lora_enabled_engine() { b"engine-grpc-lora-disabled".to_vec(), default_ready_response(), Arc::new(FakeTextBackend), + None, |_, _| boxed_test_future(async move {}), ) .await; @@ -2143,6 +2148,7 @@ async fn control_forwards_weight_update_without_pause_guard() { b"engine-grpc-rl".to_vec(), ready, Arc::new(FakeTextBackend), + None, |dealer, push| { boxed_test_future(async move { let frames = recv_engine_message(dealer).await; @@ -2392,9 +2398,9 @@ async fn grpc_health_transitions_to_not_serving_when_engine_becomes_unhealthy() #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn grpc_health_watch_closes_on_graceful_shutdown() { +async fn grpc_shutdown_requires_managed_engine() { let (inference_service, control_service, engine_health, _engine_task) = setup_grpc_service( - b"engine-grpc-health-shutdown", + b"engine-grpc-external-shutdown", default_stream_output_specs(), ) .await; @@ -2406,6 +2412,47 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() { shutdown.clone(), ) .await; + let mut control_client = ControlClient::new(channel.clone()); + + let status = control_client + .shutdown(pb::ShutdownRequest {}) + .await + .expect_err("external engine shutdown must be rejected"); + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert!(!shutdown.is_cancelled()); + + let health = HealthClient::new(channel) + .check(HealthCheckRequest { + service: "vllm.Inference".to_string(), + }) + .await + .expect("server remains healthy after rejected shutdown") + .into_inner(); + assert_eq!(health.status, HealthServingStatus::Serving as i32); + server_task.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn grpc_shutdown_closes_health_watch_and_server() { + let shutdown = tokio_util::sync::CancellationToken::new(); + let (inference_service, control_service, engine_health, _engine_task) = + setup_grpc_service_with_engine_script( + b"engine-grpc-health-shutdown", + default_ready_response(), + Arc::new(FakeTextBackend), + Some(shutdown.clone()), + |_, _| boxed_test_future(async {}), + ) + .await; + let (channel, server_task) = start_grpc_test_server( + inference_service, + control_service, + engine_health, + shutdown.child_token(), + ) + .await; + let mut control_client = ControlClient::new(channel.clone()); let mut health_client = HealthClient::new(channel); let mut stream = health_client .watch(HealthCheckRequest { @@ -2426,7 +2473,14 @@ async fn grpc_health_watch_closes_on_graceful_shutdown() { "unexpected initial health status for vllm.Inference" ); - shutdown.cancel(); + control_client + .shutdown(pb::ShutdownRequest {}) + .await + .expect("accept managed engine shutdown"); + assert!( + shutdown.is_cancelled(), + "engine owner must receive shutdown" + ); let update = tokio::time::timeout(Duration::from_secs(2), stream.message()) .await diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index d5f9e1ea55..6a36491fd9 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -247,9 +247,11 @@ where let engine_health = state.engine_core_client().subscribe_health(); health_reporter.set_serving::().await; health_reporter.set_serving::().await; - let control_service = - grpc::ControlGrpcService::new(grpc::ControlServiceImpl::new(state.clone())) - .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES); + let control_service = grpc::ControlGrpcService::new( + grpc::ControlServiceImpl::new(state.clone()) + .with_engine_shutdown(config.manages_engine.then(|| shutdown.clone())), + ) + .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES); let inference_service = grpc::InferenceGrpcService::new(grpc::InferenceServiceImpl::new(state.clone())) .max_decoding_message_size(DEFAULT_REQUEST_BODY_LIMIT_BYTES); From 5463fe4962785cdc3383477bf3af6533e7647dfd Mon Sep 17 00:00:00 2001 From: Varun Sundar Rabindranath Date: Wed, 30 Sep 2026 09:11:03 -0400 Subject: [PATCH 120/248] [KV-Offloading][TP] : Expand replicated_layout detection to multi-group MLA (#57652) --- tests/v1/core/test_kv_cache_utils.py | 97 ++++ .../unit/offloading_connector/test_config.py | 484 +++++++++++++++--- .../kv_connector/v1/offloading/config.py | 16 +- .../layers/attention/mla_attention.py | 1 + vllm/model_executor/models/deepseek_v2.py | 1 + vllm/models/deepseek_v4/attention.py | 2 + vllm/models/deepseek_v4/compressor.py | 1 + vllm/v1/attention/backends/mla/sparse_swa.py | 1 + vllm/v1/core/kv_cache_utils.py | 29 +- vllm/v1/kv_cache_interface.py | 12 + 10 files changed, 553 insertions(+), 91 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index fb82d28c36..0c56e89d59 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -52,6 +52,7 @@ from vllm.v1.core.kv_cache_utils import ( hash_block_tokens, init_none_hash, is_kv_cache_spec_uniform, + kv_cache_groups_tp_replicas, make_block_hash_with_group_id, tensor_data, to_event_extra_keys, @@ -3274,6 +3275,7 @@ def new_mla_spec(cache_dtype_str=None, block_size: int = 16): return MLAAttentionSpec( block_size=block_size, num_kv_heads=1, + max_tp_shards=1, head_size=576, dtype=torch.float32, cache_dtype_str=cache_dtype_str, @@ -3284,6 +3286,7 @@ def new_swa_mla_spec(head_size=576, sliding_window=128, model_version=None): return SlidingWindowMLASpec( block_size=16, num_kv_heads=1, + max_tp_shards=1, head_size=head_size, dtype=torch.float32, sliding_window=sliding_window, @@ -4637,3 +4640,97 @@ def test_trailing_layer_fallback_requires_exact_partition(): _annotate_eagle_groups(config, specs, trimmed, use_trailing_layer_fallback=True) assert not any(g.is_eagle_group for g in trimmed) + + +_GQA_SPEC = FullAttentionSpec( + block_size=16, num_kv_heads=8, head_size=64, dtype=torch.float32 +) +_GQA_SWA_SPEC = SlidingWindowSpec( + block_size=16, num_kv_heads=8, head_size=64, dtype=torch.float32, sliding_window=128 +) + + +@pytest.mark.parametrize( + "specs,expected", + [ + pytest.param({"l.0": new_mla_spec(), "l.1": new_mla_spec()}, 4, id="mla"), + pytest.param( + { + "l.0": new_mla_spec(), + "l.1": new_mla_spec(), + "s.0": new_swa_mla_spec(), + "s.1": new_swa_mla_spec(), + }, + 4, + id="mla-and-swa-mla", + ), + pytest.param({"l.0": _GQA_SPEC, "l.1": _GQA_SPEC}, 1, id="gqa"), + pytest.param( + {"l.0": new_mla_spec(), "l.1": _GQA_SPEC}, + 1, + id="mla-and-gqa-uniform-group", + ), + pytest.param( + { + "l.0": new_mla_spec(), + "l.1": new_mla_spec(), + "s.0": _GQA_SWA_SPEC, + "s.1": _GQA_SWA_SPEC, + }, + 1, + id="mla-and-gqa-swa", + ), + ], +) +def test_kv_tp_replicas(monkeypatch, specs, expected): + """Resolved per layer before scheduler flattening hides mixed groups.""" + from vllm.config import ParallelConfig + from vllm.platforms import current_platform + + monkeypatch.setattr(current_platform, "device_count", lambda: 4) + parallel_config = ParallelConfig(tensor_parallel_size=4) + vllm_config = VllmConfig( + model_config=ModelConfig(max_model_len=16), parallel_config=parallel_config + ) + vllm_config.cache_config.kv_cache_layout = "LBNHC" + mem = sum(s.page_size_bytes for s in specs.values()) * 10 + configs = get_kv_cache_configs(vllm_config, [specs], [mem]) + assert configs[0].kv_tp_replicas == expected + scheduler = generate_scheduler_kv_cache_config(configs) + assert scheduler.kv_tp_replicas == expected + + +@pytest.mark.parametrize( + "specs,tp_size,dcp_size,expected", + [ + pytest.param([new_mla_spec()], 8, 1, 8, id="mla"), + pytest.param([new_mla_spec()], 8, 2, 1, id="mla-dcp"), + pytest.param( + [replace(new_mla_spec(), max_tp_shards=None)], 8, 1, 1, id="mla-unset" + ), + pytest.param([replace(_GQA_SPEC, max_tp_shards=2)], 8, 1, 4, id="gqa-partial"), + pytest.param([replace(_GQA_SPEC, max_tp_shards=8)], 4, 1, 1, id="gqa-sharded"), + pytest.param( + [new_mla_spec(), replace(_GQA_SWA_SPEC, max_tp_shards=4)], + 8, + 1, + 2, + id="mla-and-gqa-partial", + ), + pytest.param( + [ + HiddenStateCacheSpec( + block_size=16, num_kv_heads=1, head_size=64, dtype=torch.float32 + ) + ], + 8, + 1, + 1, + id="hidden-state", + ), + ], +) +def test_kv_cache_groups_tp_replicas(specs, tp_size, dcp_size, expected): + """Replicas are the gcd of each layer's tp_size // max_tp_shards.""" + groups = [KVCacheGroupSpec([f"l.{i}"], spec) for i, spec in enumerate(specs)] + assert kv_cache_groups_tp_replicas(groups, tp_size, dcp_size) == expected diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_config.py b/tests/v1/kv_connector/unit/offloading_connector/test_config.py index fe0d258462..237b376004 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_config.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_config.py @@ -17,7 +17,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( SchedulerOffloadConfig, ) from vllm.platforms import current_platform -from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config +from vllm.v1.core.kv_cache_utils import ( + generate_scheduler_kv_cache_config, + kv_cache_groups_tp_replicas, +) from vllm.v1.kv_cache_interface import ( CircularBufferSpec, FullAttentionSpec, @@ -164,6 +167,7 @@ def _mla_spec( return MLAAttentionSpec( block_size=block_size, num_kv_heads=1, + max_tp_shards=1, head_size=head_size, dtype=dtype, tokens_per_state=tokens_per_state, @@ -321,7 +325,184 @@ def _replicated_layout( config.parallel_config.nnodes = nnodes if world_size is not None: config.parallel_config.world_size = world_size - return build_offloading_config(config, kv_cache_config).replicated_layout + kv_cache_config.kv_tp_replicas = kv_cache_groups_tp_replicas( + kv_cache_config.kv_cache_groups, + config.parallel_config.tensor_parallel_size, + config.parallel_config.decode_context_parallel_size, + ) + + # Scheduler and Worker offload config + worker_offload_config = build_offloading_config(config, kv_cache_config) + scheduler_offload_config = build_offloading_config( + config, generate_scheduler_kv_cache_config([kv_cache_config]) + ) + # Verify that the Worker and Scheduler agree on replication. + # and worker-bytes-per-block + assert ( + worker_offload_config.replicated_layout + == scheduler_offload_config.replicated_layout + ) + assert ( + worker_offload_config.worker_kv_bytes_per_block + == scheduler_offload_config.worker_kv_bytes_per_block + ) + + return worker_offload_config.replicated_layout + + +_SWA_MLA_PAGE = SlidingWindowMLASpec( + block_size=16, + num_kv_heads=1, + max_tp_shards=1, + head_size=512, + dtype=torch.float32, + sliding_window=128, +).page_size_bytes + + +def _make_swa_mla_kv_cache_config( + num_blocks: int = 4, +) -> KVCacheConfig: + """Single SlidingWindowMLASpec group — DSV4 sliding-window layers.""" + spec = SlidingWindowMLASpec( + block_size=16, + num_kv_heads=1, + max_tp_shards=1, + head_size=512, + dtype=torch.float32, + sliding_window=128, + ) + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[ + KVCacheTensor( + size=_SWA_MLA_PAGE * num_blocks, + layers=["layer"], + layer_stride=_SWA_MLA_PAGE * num_blocks, + block_stride=_SWA_MLA_PAGE, + ) + ], + kv_cache_groups=[KVCacheGroupSpec(["layer"], spec)], + ) + + +def _make_dsv4_flash_kv_cache_config(num_blocks: int = 4) -> KVCacheConfig: + """Minimal fake of DeepSeek-V4-Flash's five UniformTypeKVCacheSpecs groups. + + The real model has the following groups (layer counts from runtime logs); + 2 per group is enough to exercise the structural properties that govern + replicated_layout: + group[0]: 22 SlidingWindowMLASpec layers, sliding_window=128 + group[1]: 21 SlidingWindowMLASpec layers, sliding_window=128 + group[2]: 62 MLAAttentionSpec layers (MLA + indexer.k_cache) + group[3]: 42 SlidingWindowMLASpec layers, sliding_window=8 + group[4]: 20 SlidingWindowMLASpec layers, sliding_window=128 + All inner specs have num_kv_heads=1 → every group is TP-replicated. + """ + block_size = 16 + dtype = torch.float32 + head_size = 512 + + def _swa_group(tag: str, sw: int) -> KVCacheGroupSpec: + names = [f"{tag}_0", f"{tag}_1"] + spec = SlidingWindowMLASpec( + block_size=block_size, + num_kv_heads=1, + max_tp_shards=1, + head_size=head_size, + dtype=dtype, + sliding_window=sw, + ) + return KVCacheGroupSpec( + names, + UniformTypeKVCacheSpecs( + block_size=block_size, + kv_cache_specs={name: spec for name in names}, + ), + ) + + def _mla_group(tag: str) -> KVCacheGroupSpec: + names = [f"{tag}_0", f"{tag}_1"] + spec = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + max_tp_shards=1, + head_size=head_size, + dtype=dtype, + ) + return KVCacheGroupSpec( + names, + UniformTypeKVCacheSpecs( + block_size=block_size, + kv_cache_specs={name: spec for name in names}, + ), + ) + + groups = [ + _swa_group("swa_128a", 128), # group[0]: SWA-MLA, sw=128 + _swa_group("swa_128b", 128), # group[1]: SWA-MLA, sw=128 + _mla_group("mla"), # group[2]: MLA + indexer layers + _swa_group("swa_8", 8), # group[3]: SWA-MLA, sw=8 + _swa_group("swa_128c", 128), # group[4]: SWA-MLA, sw=128 + ] + # All specs share the same page size (num_kv_heads=1, head_size=512, fp32). + page = MLAAttentionSpec( + block_size=block_size, num_kv_heads=1, head_size=head_size, dtype=dtype + ).page_size_bytes + all_layers = [name for g in groups for name in g.layer_names] + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[ + KVCacheTensor( + size=page * len(all_layers) * num_blocks, + layers=all_layers, + layer_stride=page * num_blocks, + block_stride=page, + ) + ], + kv_cache_groups=groups, + ) + + +def _make_dsv3_2_kv_cache_config(num_blocks: int = 4) -> KVCacheConfig: + """Minimal fake of DeepSeek-V3.2-Exp's single UniformTypeKVCacheSpecs group. + + The real model has 1 group (layer counts from runtime logs): + group[0]: 122 MLAAttentionSpec layers (self_attn.attn + indexer.k_cache) + All inner specs have num_kv_heads=1 → the group is TP-replicated. + + GLM-5.3 has the same topology (one UniformType group of MLAAttentionSpec + layers, num_kv_heads=1) with 99 layers instead of 122, so this config + covers both models. + """ + block_size = 16 + names = [f"mla_{i}" for i in range(4)] # 4 layers; real model has 122 + spec = MLAAttentionSpec( + block_size=block_size, + num_kv_heads=1, + max_tp_shards=1, + head_size=512, + dtype=torch.float32, + ) + group = KVCacheGroupSpec( + names, + UniformTypeKVCacheSpecs( + block_size=block_size, + kv_cache_specs={name: spec for name in names}, + ), + ) + return KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[ + KVCacheTensor( + size=spec.page_size_bytes * len(names) * num_blocks, + layers=names, + layer_stride=spec.page_size_bytes * num_blocks, + block_stride=spec.page_size_bytes, + ) + ], + kv_cache_groups=[group], + ) @pytest.mark.parametrize("packed", [False, True]) @@ -588,35 +769,64 @@ def test_replicated_layout_enabled_for_pure_mla_tp_mp_single_node( ) +@pytest.mark.parametrize("world_size", [2, 4]) +def test_replicated_layout_enabled_for_swa_mla(world_size: int): + """SlidingWindowMLASpec groups are also replicated and qualify.""" + assert _replicated_layout( + _make_swa_mla_kv_cache_config(), + tensor_parallel_size=world_size, + ) + + +@pytest.mark.parametrize("world_size", [2, 4, 8]) +def test_replicated_layout_dsv4_flash(world_size: int): + """DeepSeek-V4-Flash: all 5 UniformType groups are MLA/SWA-MLA with + num_kv_heads=1, so replicated_layout must be True under TP/mp single-node.""" + assert _replicated_layout( + _make_dsv4_flash_kv_cache_config(), + tensor_parallel_size=world_size, + ) + + +@pytest.mark.parametrize("world_size", [2, 4, 8]) +def test_replicated_layout_dsv3_2(world_size: int): + """DeepSeek-V3.2 / GLM-5.3: single UniformType group of MLAAttentionSpec + layers (122 and 99 respectively), num_kv_heads=1 → replicated_layout must + be True under TP/mp single-node.""" + assert _replicated_layout( + _make_dsv3_2_kv_cache_config(), + tensor_parallel_size=world_size, + ) + + +def test_replicated_layout_enabled_for_uniform_wrapper_mla(): + """UniformTypeKVCacheSpecs wrapping MLA layers of different sizes qualifies: + each layer is replicated and the page accounting sums correctly.""" + mla_layers = ["layer0", "layer1"] + spec = UniformTypeKVCacheSpecs( + block_size=16, + kv_cache_specs={"layer0": _mla_spec(), "layer1": _mla_spec(head_size=256)}, + ) + total_page = spec.page_size_bytes + num_blocks = 4 + kv_cache_config = KVCacheConfig( + num_blocks=num_blocks, + kv_cache_tensors=[ + KVCacheTensor( + size=total_page * num_blocks, + layers=mla_layers, + layer_stride=total_page * num_blocks, + block_stride=total_page, + ) + ], + kv_cache_groups=[KVCacheGroupSpec(mla_layers, spec)], + ) + assert _replicated_layout(kv_cache_config) + + @pytest.mark.parametrize( ("kv_cache_config", "case"), [ - ( - KVCacheConfig( - num_blocks=4, - kv_cache_tensors=[ - KVCacheTensor( - size=_MLA_PAGE * 4, - layers=["layer"], - layer_stride=_MLA_PAGE * 4, - block_stride=_MLA_PAGE, - ) - ], - kv_cache_groups=[ - KVCacheGroupSpec( - ["layer"], - SlidingWindowMLASpec( - block_size=16, - num_kv_heads=1, - head_size=512, - dtype=torch.float32, - sliding_window=128, - ), - ) - ], - ), - "sliding-window-mla", - ), ( KVCacheConfig( num_blocks=4, @@ -642,40 +852,6 @@ def test_replicated_layout_enabled_for_pure_mla_tp_mp_single_node( ), "hidden-state", ), - ( - # One group, two page sizes: layer1's run starts past layer0's region. - KVCacheConfig( - num_blocks=4, - kv_cache_tensors=[ - KVCacheTensor( - size=(_MLA_PAGE + _HALF_MLA_PAGE) * 4, - layers=["layer0"], - layer_stride=_MLA_PAGE * 4, - block_stride=_MLA_PAGE, - ), - KVCacheTensor( - size=(_MLA_PAGE + _HALF_MLA_PAGE) * 4, - layers=["layer1"], - layer_stride=_HALF_MLA_PAGE * 4, - block_stride=_HALF_MLA_PAGE, - offset=_MLA_PAGE * 4, - ), - ], - kv_cache_groups=[ - KVCacheGroupSpec( - ["layer0", "layer1"], - UniformTypeKVCacheSpecs( - block_size=16, - kv_cache_specs={ - "layer0": _mla_spec(), - "layer1": _mla_spec(head_size=256), - }, - ), - ) - ], - ), - "uniform-wrapper", - ), ( # Overlaid groups with different page sizes: a block is a window of # the largest group's packing. @@ -726,6 +902,84 @@ def test_replicated_layout_enabled_for_pure_mla_tp_mp_single_node( ), "mla-mamba-hybrid", ), + ], + ids=[ + "hidden-state", + "mla-full-hybrid", + "mla-mamba-hybrid", + ], +) +def test_replicated_layout_excludes_unproven_cache_shapes( + kv_cache_config: KVCacheConfig, + case: str, +): + assert not _replicated_layout(kv_cache_config), case + + +@pytest.mark.parametrize( + ("kv_cache_config", "case"), + [ + ( + KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[ + KVCacheTensor( + size=_MLA_PAGE * 4, + layers=["layer"], + layer_stride=_MLA_PAGE * 4, + block_stride=_MLA_PAGE, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer"], + SlidingWindowMLASpec( + block_size=16, + num_kv_heads=1, + max_tp_shards=1, + head_size=512, + dtype=torch.float32, + sliding_window=128, + ), + ) + ], + ), + "sliding-window-mla", + ), + ( + # One group, two page sizes: layer1's run starts past layer0's region. + KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[ + KVCacheTensor( + size=(_MLA_PAGE + _HALF_MLA_PAGE) * 4, + layers=["layer0"], + layer_stride=_MLA_PAGE * 4, + block_stride=_MLA_PAGE, + ), + KVCacheTensor( + size=(_MLA_PAGE + _HALF_MLA_PAGE) * 4, + layers=["layer1"], + layer_stride=_HALF_MLA_PAGE * 4, + block_stride=_HALF_MLA_PAGE, + offset=_MLA_PAGE * 4, + ), + ], + kv_cache_groups=[ + KVCacheGroupSpec( + ["layer0", "layer1"], + UniformTypeKVCacheSpecs( + block_size=16, + kv_cache_specs={ + "layer0": _mla_spec(), + "layer1": _mla_spec(head_size=256), + }, + ), + ) + ], + ), + "uniform-wrapper", + ), ( KVCacheConfig( num_blocks=4, @@ -748,31 +1002,54 @@ def test_replicated_layout_enabled_for_pure_mla_tp_mp_single_node( ], ids=[ "sliding-window-mla", - "hidden-state", "uniform-wrapper", - "mla-full-hybrid", - "mla-mamba-hybrid", "multi-group-mla", ], ) -def test_replicated_layout_excludes_unproven_cache_shapes( +def test_replicated_layout_supported_cache_shapes( kv_cache_config: KVCacheConfig, case: str, ): - assert not _replicated_layout(kv_cache_config), case + assert _replicated_layout(kv_cache_config), case -def test_replicated_layout_rejects_bare_mla_with_mixed_page_accounting(): +@pytest.mark.parametrize( + ("main_spec", "add_spec", "expected"), + [ + ( + _mla_spec(head_size=512), + _mla_spec(head_size=128, dtype=torch.uint8), # indexer.k_cache spec + True, + ), + ( + _mla_spec(head_size=512), + _full_attention_spec(), + False, + ), + ], + ids=["uniform-mla-indexer", "uniform-mla-fa"], +) +def test_replicated_layout_for_dsa_style_groups( + main_spec: KVCacheSpec, add_spec: KVCacheSpec, expected: bool +): + """KV Cache groups with packed-specs inside a UniformTypeKVCacheSpec.""" num_blocks = 4 - main_spec = _mla_spec(head_size=512) - indexer_spec = _mla_spec(head_size=128, dtype=torch.uint8) main_layers = [f"main_{i}" for i in range(61)] - indexer_layers = [f"indexer_{i}" for i in range(61)] - # A DSA-style group: the main pages and the smaller indexer pages are packed one - # after the other, so a block holds more than 61 MLA pages. + add_layers = [f"add_{i}" for i in range(61)] + main_bytes = main_spec.page_size_bytes * len(main_layers) - indexer_bytes = indexer_spec.page_size_bytes * len(indexer_layers) - size = (main_bytes + indexer_bytes) * num_blocks + add_bytes = add_spec.page_size_bytes * len(add_layers) + size = (main_bytes + add_bytes) * num_blocks + + kv_cache_specs = { + **{layer: main_spec for layer in main_layers}, + **{layer: add_spec for layer in add_layers}, + } + layer_names = main_layers + add_layers + group_spec: KVCacheSpec = UniformTypeKVCacheSpecs( + block_size=16, kv_cache_specs=kv_cache_specs + ) + kv_cache_config = KVCacheConfig( num_blocks=num_blocks, kv_cache_tensors=[ @@ -784,16 +1061,16 @@ def test_replicated_layout_rejects_bare_mla_with_mixed_page_accounting(): ), KVCacheTensor( size=size, - layers=indexer_layers, - layer_stride=indexer_spec.page_size_bytes * num_blocks, - block_stride=indexer_spec.page_size_bytes, + layers=add_layers, + layer_stride=add_spec.page_size_bytes * num_blocks, + block_stride=add_spec.page_size_bytes, offset=main_bytes * num_blocks, ), ], - kv_cache_groups=[KVCacheGroupSpec(main_layers + indexer_layers, main_spec)], + kv_cache_groups=[KVCacheGroupSpec(layer_names, group_spec)], ) - assert not _replicated_layout(kv_cache_config) + assert _replicated_layout(kv_cache_config) == expected @pytest.mark.parametrize( @@ -841,6 +1118,7 @@ _SWA_SPEC = SlidingWindowSpec( _SWA_MLA_SPEC = SlidingWindowMLASpec( block_size=16, num_kv_heads=1, + max_tp_shards=1, head_size=576, dtype=torch.float32, sliding_window=128, @@ -904,6 +1182,52 @@ def test_canonical_layout_gate(kv_cache_groups, certified): assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified +@pytest.mark.parametrize( + ("spec"), + [_SWA_MLA_SPEC, _mla_spec(tokens_per_state=2)], + ids=["swa", "compressed-mla"], +) +def test_uncertifiable_canonical_layout_resets_replication(spec: KVCacheSpec): + def _get_offloading_config(canonical: bool): + config = _make_vllm_config( + tensor_parallel_size=2, + extra_config={"canonical_layout": True} if canonical else None, + ) + config.model_config.use_mla = True + + PAGE = spec.page_size_bytes + kv_config = KVCacheConfig( + num_blocks=4, + kv_cache_tensors=[ + KVCacheTensor( + size=PAGE * 4, + layers=["layer"], + layer_stride=PAGE * 4, + block_stride=PAGE, + ) + ], + kv_cache_groups=[ + KVCacheGroupSpec(["layer"], spec), + ], + ) + + kv_config.kv_tp_replicas = kv_cache_groups_tp_replicas( + kv_config.kv_cache_groups, + config.parallel_config.tensor_parallel_size, + config.parallel_config.decode_context_parallel_size, + ) + + return build_offloading_config(config, kv_config) + + offload_config = _get_offloading_config(canonical=False) + assert offload_config.replicated_layout + assert not offload_config.parallel.is_parallelism_agnostic + + canon_offload_config = _get_offloading_config(canonical=True) + assert not canon_offload_config.replicated_layout + assert not canon_offload_config.parallel.is_parallelism_agnostic + + def test_canonical_layout_certifies_v2_model_runner(): """Canonical bytes are certified per layer against live tensor strides at registration, so the static gate must not depend on the model-runner diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py index e20c1189af..186590b956 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/config.py @@ -157,15 +157,10 @@ def build_offloading_config( ) replicated_layout = ( vllm_config.model_config.use_mla - # Exact type: fail closed on wrappers and sliding-window variants. - and type(single_group_spec) is MLAAttentionSpec - # Page accounting: one MLA page per layer, no packed/mixed rows. - and worker_kv_bytes_per_block > 0 - and worker_kv_bytes_per_block - == single_group_spec.page_size_bytes - * len(kv_cache_config.kv_cache_groups[0].layer_names) - # Safe MVP boundary: TP-only, no other parallel axes. and parallel_config.tensor_parallel_size > 1 + and kv_cache_config.kv_tp_replicas == parallel_config.tensor_parallel_size + and worker_kv_bytes_per_block > 0 + # Safe MVP boundary: TP-only, no other parallel axes. and parallel_config.pipeline_parallel_size == 1 and parallel_config.prefill_context_parallel_size == 1 and parallel_config.decode_context_parallel_size == 1 @@ -233,9 +228,10 @@ def build_offloading_config( and parallel_config.world_size == tp_size ) - if canonical_layout and is_parallelism_agnostic: + if canonical_layout: replicated_layout = ( - all( + is_parallelism_agnostic + and all( type(spec) is MLAAttentionSpec for _, group in selected_groups for spec in iter_layer_specs(group.kv_cache_spec) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 2ae111c1c3..e016e5daff 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1350,6 +1350,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): common_kwargs = dict( block_size=vllm_config.cache_config.block_size, num_kv_heads=1, + max_tp_shards=1, head_size=self.head_size, dtype=kv_cache_dtype, cache_dtype_str=self.kv_cache_dtype, diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 01010a13fe..70d87529f6 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -658,6 +658,7 @@ class DeepseekV32IndexerCache(torch.nn.Module, AttentionLayerBase): return MLAAttentionSpec( block_size=self.cache_config.block_size, num_kv_heads=1, + max_tp_shards=1, head_size=self.head_dim, dtype=self.dtype, cache_role=SparseCacheRole.INDEXER, diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 2315bedfeb..4f756538fb 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -865,6 +865,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): return MLAAttentionSpec( block_size=vllm_config.cache_config.block_size, num_kv_heads=1, + max_tp_shards=1, head_size=self.head_dim, dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype, tokens_per_state=self.compress_ratio, @@ -910,6 +911,7 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase): return MLAAttentionSpec( block_size=self.cache_config.block_size, num_kv_heads=1, + max_tp_shards=1, head_size=self.head_dim, dtype=self.dtype, tokens_per_state=self.compress_ratio, diff --git a/vllm/models/deepseek_v4/compressor.py b/vllm/models/deepseek_v4/compressor.py index e6e0fd959e..ccb864faa2 100644 --- a/vllm/models/deepseek_v4/compressor.py +++ b/vllm/models/deepseek_v4/compressor.py @@ -184,6 +184,7 @@ class CompressorStateCache(torch.nn.Module, AttentionLayerBase): return SlidingWindowMLASpec( # only has one vector instead of K + V block_size=self.block_size, num_kv_heads=1, + max_tp_shards=1, head_size=self.state_dim, dtype=self.dtype, sliding_window=self.sliding_window, diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index b0ef9e65a3..0f5bff4144 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -124,6 +124,7 @@ class DeepseekV4SWACache(torch.nn.Module, AttentionLayerBase): bounded_replay=self.bounded_replay, block_size=self.block_size, num_kv_heads=1, + max_tp_shards=1, head_size=self.head_dim, dtype=self.dtype, sliding_window=self.window_size, diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 7ef71dd471..a82d3e8113 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -9,7 +9,7 @@ import os from collections import defaultdict from collections.abc import Callable, Iterable, Iterator, Sequence from dataclasses import dataclass, field, replace -from functools import partial +from functools import partial, reduce from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload from vllm import envs @@ -2409,6 +2409,26 @@ def get_kv_cache_groups( return groups +def _layer_tp_replicas(spec: KVCacheSpec, tp_size: int, dcp_size: int) -> int: + if not isinstance(spec, AttentionSpec) or spec.max_tp_shards is None: + return 1 + if spec.dcp_sharded and dcp_size > 1: + return 1 + return max(1, tp_size // spec.max_tp_shards) + + +def kv_cache_groups_tp_replicas( + groups: list[KVCacheGroupSpec], tp_size: int, dcp_size: int = 1 +) -> int: + """Consecutive TP ranks holding identical KV for every layer.""" + specs = [spec for g in groups for spec in iter_layer_specs(g.kv_cache_spec)] + if not specs: + return 1 + return reduce( + math.gcd, (_layer_tp_replicas(s, tp_size, dcp_size) for s in specs), tp_size + ) + + def generate_scheduler_kv_cache_config( kv_cache_configs: list[KVCacheConfig], ) -> KVCacheConfig: @@ -2823,6 +2843,13 @@ def get_kv_cache_configs( vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups) ) + for kv_cache_config in kv_cache_configs: + kv_cache_config.kv_tp_replicas = kv_cache_groups_tp_replicas( + kv_cache_config.kv_cache_groups, + vllm_config.parallel_config.tensor_parallel_size, + vllm_config.parallel_config.decode_context_parallel_size, + ) + return kv_cache_configs diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 5791704b2b..cabbbfadc3 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -484,6 +484,9 @@ class HiSparseResidentSpec(KVCacheSpec): @dataclass(frozen=True, kw_only=True) class AttentionSpec(KVCacheSpec): dcp_sharded: bool = True + max_tp_shards: int | None = None + """Distinct shards this cache splits into across TP; TP ranks beyond this + hold replicas. None: unknown, treated as not replicated.""" num_kv_heads: int head_size: int dtype: torch.dtype @@ -618,6 +621,7 @@ class FullAttentionSpec(AttentionSpec): dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, dcp_sharded=specs[0].dcp_sharded, + max_tp_shards=specs[0].max_tp_shards, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -705,6 +709,7 @@ class MLAAttentionSpec(FullAttentionSpec): dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, dcp_sharded=specs[0].dcp_sharded, + max_tp_shards=specs[0].max_tp_shards, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -768,6 +773,7 @@ class RSWASpec(FullAttentionSpec): dtype=base.dtype, kv_quant_mode=base.kv_quant_mode, dcp_sharded=base.dcp_sharded, + max_tp_shards=base.max_tp_shards, page_size_padded=base.page_size_padded, num_head_slots=base.num_head_slots, state_content_bytes=base.state_content_bytes, @@ -960,6 +966,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): bounded_replay_set = set(spec.bounded_replay for spec in specs) block_stride_alignment_set = {spec.block_stride_alignment for spec in specs} assert len({spec.dcp_sharded for spec in specs}) == 1 + assert len({spec.max_tp_shards for spec in specs}) == 1 assert ( len(cache_dtype_str_set) == 1 and len(tokens_per_state_set) == 1 @@ -980,6 +987,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec): head_size=specs[0].head_size, dtype=specs[0].dtype, dcp_sharded=specs[0].dcp_sharded, + max_tp_shards=specs[0].max_tp_shards, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1197,6 +1205,7 @@ class SinkFullAttentionSpec(FullAttentionSpec): dtype=specs[0].dtype, kv_quant_mode=specs[0].kv_quant_mode, dcp_sharded=specs[0].dcp_sharded, + max_tp_shards=specs[0].max_tp_shards, page_size_padded=specs[0].page_size_padded, num_head_slots=specs[0].num_head_slots, state_content_bytes=specs[0].state_content_bytes, @@ -1483,6 +1492,9 @@ class KVCacheConfig: hisparse_shared_host_pool: bool = False """Whether local TP ranks share one physical HiSparse host pool.""" + kv_tp_replicas: int = 1 + """Consecutive TP ranks holding identical KV for every layer (1: none).""" + @cached_property def transfer_group_ids(self) -> tuple[int, ...]: """IDs of cache groups that participate in external KV transfer.""" From 677e3f5d5040db80087fd5693cd3434d1adcde83 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Wed, 30 Sep 2026 21:45:28 +0800 Subject: [PATCH 121/248] [CI/Build] Fix pre-commit (#59426) --- tests/multimodal/test_cache.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index 0416734c24..da6b0369b4 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import multiprocessing as mp from types import SimpleNamespace +from typing import cast from unittest.mock import Mock import numpy as np @@ -796,6 +797,7 @@ def test_shm_receiver_requires_sender_handle_for_current_cache_key(): try: sender_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + assert sender_item is not None valid_feature = MultiModalFeatureSpec( data=sender_item, @@ -897,10 +899,12 @@ def test_shm_sender_validates_client_supplied_handles(): handle, _ = sender_cache.get_and_update_item( (MultiModalKwargsItem.dummy(1024), []), mm_hash ) + assert handle is not None + sender_cache.validate_input_item(handle, mm_hash) sender_cache.validate_input_item(MultiModalKwargsItem.dummy(16), mm_hash) - signature = handle["signature"].data + signature = cast(list[int], handle["signature"].data) forged = [ _with_elem(handle, "signature", [signature[0] ^ 1, *signature[1:]]), _with_elem(handle, "signature", [signature]), @@ -934,6 +938,8 @@ def test_shm_receiver_rejects_out_of_range_handle_fields(): handle, _ = sender_cache.get_and_update_item( (MultiModalKwargsItem.dummy(1024), []), mm_hash ) + assert handle is not None + for key, value in [ ("address", -1), ("address", 2**64), @@ -967,6 +973,8 @@ def test_shm_handle_issued_before_cache_clear(): try: stale_item, _ = sender_cache.get_and_update_item((item, []), mm_hash) + assert stale_item is not None + sender_cache.clear_cache() # A request still draining after the clear resolves to its own data. From 02a3c6dfc56c8776179a2e1c4fe01df270188f71 Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Wed, 30 Sep 2026 21:50:40 +0800 Subject: [PATCH 122/248] [Test] Skip IPC weight-transfer test on XPU platforms (#59379) Signed-off-by: Chaojun Zhang Co-authored-by: Kunshang Ji --- .../serve/dev/rlhf/state_transitions/test_weight_checker.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py index d2fb97174e..7cb58df9d9 100644 --- a/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py +++ b/tests/entrypoints/serve/dev/rlhf/state_transitions/test_weight_checker.py @@ -102,6 +102,8 @@ class TestSingleGPU: ) def test_ipc_weight_transfer_restores_reset_weights(): """The documented RL flow: reset, then transfer the checkpoint back.""" + if current_platform.is_xpu(): + pytest.skip("IPC weight transfer backend uses CUDA IPC handles") args = ["--weight-transfer-config", '{"backend": "ipc"}'] with ( patch.dict(os.environ, {"VLLM_ALLOW_INSECURE_SERIALIZATION": "1"}), From 2e52a558f04307b3f6f842e937e35aac079121c1 Mon Sep 17 00:00:00 2001 From: Giulio De Pasquale Date: Wed, 30 Sep 2026 15:04:28 +0100 Subject: [PATCH 123/248] [Bugfix][Core] Schedule encoder-only prompts larger than one step (#59029) Signed-off-by: Giulio De Pasquale Co-authored-by: Isotr0py <2037008807@qq.com> --- tests/v1/core/test_scheduler.py | 46 +++++++++++++++++++++++++++++++++ tests/v1/core/utils.py | 3 +++ vllm/v1/core/sched/scheduler.py | 1 + 3 files changed, 50 insertions(+) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 89a73b675c..c8ac4dec64 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -437,6 +437,52 @@ def test_schedule_partial_requests(): assert requests[2].request_id not in output.num_scheduled_tokens +def test_encoder_only_prompt_longer_than_budget_is_chunked(): + """The engine switches chunked prefill off for an encoder-only instance, + but the token budget must still split its prompt across steps instead of + leaving the request waiting forever.""" + scheduler = create_scheduler( + model="llava-hf/llava-1.5-7b-hf", + max_num_batched_tokens=1024, + max_model_len=4096, + mm_encoder_only=True, + ) + # EngineCore turns this off after config validation whenever the instance + # holds no KV cache, which is every encoder-only instance. + scheduler.scheduler_config.enable_chunked_prefill = False + (request,) = create_requests( + num_requests=1, + num_tokens=2500, + mm_positions=[[PlaceholderRange(offset=100, length=600)]], + ) + scheduler.add_request(request) + + def advance(output): + scheduler.update_from_output( + output, + ModelRunnerOutput( + req_ids=[request.request_id], + req_id_to_index={request.request_id: 0}, + sampled_token_ids=[[]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ), + ) + + first = scheduler.schedule() + assert first.num_scheduled_tokens[request.request_id] == 1024 + assert request.request_id in first.scheduled_encoder_inputs + advance(first) + + second = scheduler.schedule() + assert second.num_scheduled_tokens[request.request_id] == 1024 + advance(second) + + third = scheduler.schedule() + assert third.num_scheduled_tokens[request.request_id] == 452 + + @pytest.mark.parametrize("has_running", [True, False]) def test_schedule_prefills_gating(has_running: bool): """DP prefill-balancing gate: when `throttle_prefills` is True, a new diff --git a/tests/v1/core/utils.py b/tests/v1/core/utils.py index 46ca29f67b..49aaca3d7c 100644 --- a/tests/v1/core/utils.py +++ b/tests/v1/core/utils.py @@ -60,6 +60,7 @@ def create_scheduler( long_prefill_token_threshold: int = 0, long_prefill_token_threshold_adaptive: bool = False, disable_chunked_mm_input: bool = False, + mm_encoder_only: bool = False, use_kv_connector: None | bool | str | MockKVConfig = None, kv_role: str = "kv_both", num_blocks: int = 10000, @@ -109,6 +110,8 @@ def create_scheduler( # SchedulerConfig one, so both must agree. max_model_len=max_model_len, ) + if mm_encoder_only: + model_config.multimodal_config.mm_encoder_only = True if use_ec_connector and ec_role == "ec_producer": model_config.multimodal_config = MultiModalConfig() if max_model_len is None: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 24ecc4534d..fc924b876c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1115,6 +1115,7 @@ class Scheduler(SchedulerInterface): if ( not self.scheduler_config.enable_chunked_prefill and num_new_tokens > request_token_budget + and not self.is_mm_encoder_only ): # If chunked_prefill is disabled, # we can stop the scheduling here. From 98b45c1a94db509e75fa9ef268cf5d2ae71875b7 Mon Sep 17 00:00:00 2001 From: Qiu Chunshuo Date: Wed, 30 Sep 2026 07:21:56 -0700 Subject: [PATCH 124/248] [Feat] Support PP with PCP in GPU Model Runner V2 (#59139) Signed-off-by: QiuChunshuo Co-authored-by: Lucas Wilkinson --- tests/v1/worker/test_gpu_model_runner_v2.py | 39 ++++++++++++++++++- .../worker/test_gpu_model_runner_v2_eplb.py | 1 + vllm/v1/worker/gpu/model_runner.py | 2 + vllm/v1/worker/gpu/pcp_manager.py | 7 +++- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 308c9fe9ea..2e0c7271ce 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -3,6 +3,7 @@ import contextlib from types import SimpleNamespace +from unittest.mock import Mock import pytest import torch @@ -18,7 +19,43 @@ from vllm.v1.kv_cache_interface import ( UniformTypeKVCacheSpecs, ) from vllm.v1.worker.gpu.block_table import BlockTables -from vllm.v1.worker.gpu.model_runner import GPUModelRunner +from vllm.v1.worker.gpu.model_runner import ExecuteModelState, GPUModelRunner + + +def test_non_last_pp_rank_uses_global_batch_for_sample_feedback(): + runner = GPUModelRunner.__new__(GPUModelRunner) + runner.is_last_pp_rank = False + local_batch = object() + global_batch = SimpleNamespace(idx_mapping=object()) + runner.pcp_manager = SimpleNamespace( + global_batch=global_batch, + restore_for_sampling=Mock(), + ) + runner.pp_handler = SimpleNamespace(receive=Mock(return_value=False)) + runner.postprocess_num_computed_tokens = Mock() + runner.model_state = SimpleNamespace(postprocess_state=Mock()) + runner.kv_connector = SimpleNamespace(post_forward=Mock(return_value=None)) + runner.eplb = SimpleNamespace(step=Mock()) + runner.execute_model_state = ExecuteModelState( + input_batch=local_batch, + attn_metadata=None, + slot_mappings_by_layer=None, + hidden_states=None, + aux_hidden_states=None, + dp_sync=None, + finished_req_ids=set(), + ec_connector_output=None, + cudagraph_stats=None, + ) + + runner.sample_tokens(None) + + runner.pp_handler.receive.assert_called_once_with(global_batch) + runner.postprocess_num_computed_tokens.assert_called_once_with(global_batch) + runner.model_state.postprocess_state.assert_called_once_with( + global_batch.idx_mapping, 0 + ) + runner.pcp_manager.restore_for_sampling.assert_not_called() def test_qsa_circular_group_uses_custom_slot_mapping(monkeypatch): diff --git a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py index 7c759894fe..bbde3295ec 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2_eplb.py +++ b/tests/v1/worker/test_gpu_model_runner_v2_eplb.py @@ -79,6 +79,7 @@ def _make_runner(**overrides: Any) -> Any: runner.use_aux_hidden_state_outputs = False runner.speculative_config = None runner.speculator = None + runner.pcp_manager = None runner.num_speculative_steps = 0 runner.encoder_cache = None runner.is_pooling_model = False diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 800ae825d4..3bdeda23dd 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -2064,6 +2064,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Non-last PP rank: hidden_states is None because this rank produced # IntermediateTensors instead of final hidden states. Receive the # sampled tokens broadcast from the last rank and update local state. + if self.pcp_manager is not None: + input_batch = self.pcp_manager.global_batch assert self.pp_handler is not None all_decode_next = self.pp_handler.receive(input_batch) # Optimistically update num_computed_tokens for entire batch here. diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index 11ee7522fc..f740e4caf3 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -131,8 +131,6 @@ class PCPManager: if not model_config.use_mla: raise NotImplementedError("MRV2 PCP currently supports MLA models only.") - if parallel_config.pipeline_parallel_size > 1: - raise NotImplementedError("MRV2 PCP does not support PP yet.") if model_config.is_encoder_decoder: raise NotImplementedError( "MRV2 PCP does not support encoder-decoder models yet." @@ -406,6 +404,11 @@ class PCPManager: assert self._input_buffers is not None return self._input_buffers + @property + def global_batch(self) -> InputBatch: + assert self._global_batch is not None + return self._global_batch + def partition_batch( self, input_batch: InputBatch, batch_desc: "BatchExecutionDescriptor" ) -> InputBatch: From eef8b15d7c34019880b5ad8c811f60652aa4a6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20P=C3=A9rez=20de=20Algaba?= <124347725+jperezdealgaba@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:26:16 +0200 Subject: [PATCH 125/248] [Security] Bump pyjwt and rand for remaining GHSAs (#59427) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Juan Pérez de Algaba Co-authored-by: Cursor --- requirements/test/cuda.in | 2 +- requirements/test/rocm.in | 2 +- requirements/test/xpu.in | 2 +- requirements/test/xpu.txt | 2 +- rust/Cargo.lock | 22 +++++++++++----------- rust/Cargo.toml | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index d96bff8804..cb5ceb248e 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -86,7 +86,7 @@ plotly # required for perf comparison html report # Skipped (no patched release yet): sqlitedict, accelerate, lmcache, nltk GHSA-8mgp-746c-j5xp. urllib3>=2.7.0 cryptography>=50.0.0 -pyjwt>=2.13.0 +pyjwt>=2.14.0 pyasn1>=0.6.4 python-multipart>=0.0.31 anyio>=4.14.2 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index d8baae6c5c..e66343d47b 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -84,7 +84,7 @@ plotly # required for perf comparison html report # Skipped (no patched release yet): sqlitedict, accelerate, lmcache, nltk GHSA-8mgp-746c-j5xp. urllib3>=2.7.0 cryptography>=50.0.0 -pyjwt>=2.13.0 +pyjwt>=2.14.0 pyasn1>=0.6.4 python-multipart>=0.0.31 anyio>=4.14.2 diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index 28fdf8aacd..c39e8350dd 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -65,7 +65,7 @@ cohere>=7.0.0 # required for cohere chat v2 api tests (protocol/serving import c # Dependabot GHSA floors for transitive test deps (excl. dependabot.yml ignores). urllib3>=2.7.0 cryptography>=50.0.0 -pyjwt>=2.13.0 +pyjwt>=2.14.0 pyasn1>=0.6.4 python-multipart>=0.0.31 anyio>=4.14.2 diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index d6aa92f210..01a0fda14b 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -809,7 +809,7 @@ pygments==2.21.0 # -r requirements/test/xpu.in # pytest # rich -pyjwt==2.13.0 +pyjwt==2.15.1 # via # -r requirements/test/xpu.in # mcp diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 99878a52ab..d6143f941b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -197,7 +197,7 @@ dependencies = [ "getrandom 0.3.4", "hex", "hmac", - "rand 0.9.2", + "rand 0.9.5", "reqwest 0.12.28", "reqwest-eventsource", "secrecy", @@ -1727,7 +1727,7 @@ dependencies = [ "log", "native-tls", "num_cpus", - "rand 0.9.2", + "rand 0.9.5", "reqwest 0.12.28", "serde", "serde_json", @@ -2843,7 +2843,7 @@ checksum = "7b8d41ed865b7a26b6b2d2a519b774460e5ddc50eeeb4ac2a2409c8817ec2de9" dependencies = [ "bitflags", "chrono", - "rand 0.9.2", + "rand 0.9.5", "schemars 0.8.22", "serde", "serde_json", @@ -3511,9 +3511,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3564,7 +3564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.5", ] [[package]] @@ -5042,7 +5042,7 @@ dependencies = [ "monostate", "onig", "paste", - "rand 0.9.2", + "rand 0.9.5", "rayon", "rayon-cond", "regex", @@ -5443,7 +5443,7 @@ checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "log", - "rand 0.9.2", + "rand 0.9.5", "thiserror 2.0.18", "utf-8", ] @@ -5750,7 +5750,7 @@ dependencies = [ "indicatif", "mimalloc", "parquet", - "rand 0.9.2", + "rand 0.9.5", "rand_distr", "rayon", "reqwest 0.12.28", @@ -5949,7 +5949,7 @@ dependencies = [ "asynk-strim-attr", "clap", "futures", - "rand 0.9.2", + "rand 0.9.5", "rmpv", "serde", "tokio", @@ -6734,7 +6734,7 @@ dependencies = [ "num-traits", "once_cell", "parking_lot", - "rand 0.9.2", + "rand 0.9.5", "regex", "scc 3.6.9", "thiserror 1.0.69", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 39f4ab1a57..f3d3053ed5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -80,7 +80,7 @@ prost-types = "0.14.3" protox = "0.9.1" pyo3 = "0.29.0" pythonize = "0.29.0" -rand = "0.9.2" +rand = "0.9.3" rand_distr = "0.5.1" rayon = "1.11.0" reasoning-parser = "1.2.2" From 643bbef55c37edb5c3fd341aae1e20e026d0e730 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Wed, 30 Sep 2026 22:29:12 +0800 Subject: [PATCH 126/248] [Refactor] Share workspace and model runner init between GPU and XPU workers (#59200) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- vllm/v1/worker/gpu_worker.py | 26 +++++++++++++------------- vllm/v1/worker/xpu_worker.py | 16 +++------------- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 8b7a0260a7..247265b659 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -522,6 +522,10 @@ class Worker(WorkerBase): else: raise RuntimeError(f"Unsupported device type: {self.device_config.device}") + self._init_workspace_and_model_runner() + + def _init_workspace_and_model_runner(self) -> None: + """Set up the workspace manager, build the model runner, report usage.""" # DSpark target and draft CUDA graphs retain workspace views concurrently. num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1 init_workspace_manager( @@ -529,8 +533,13 @@ class Worker(WorkerBase): num_ubatches, _num_workspace_lanes(self.vllm_config, self.use_v2_model_runner), ) + self.model_runner: GPUModelRunner = self._make_model_runner() + if self.rank == 0: + # If usage stat is enabled, collect relevant info. + report_usage_stats(self.vllm_config) - # Construct the model runner + def _make_model_runner(self) -> "GPUModelRunner": + """Construct the platform's model runner; subclasses override this.""" if self.use_v2_model_runner: if self.vllm_config.is_mm_encoder_only: from vllm.v1.worker.mm_encoder_model_runner import ( @@ -542,19 +551,10 @@ class Worker(WorkerBase): ) # HACK(woosuk): This is a temporary fix to avoid type errors. - self.model_runner: GPUModelRunner = GPUModelRunnerV2( # type: ignore - self.vllm_config, self.device - ) - else: - from vllm.v1.worker.gpu_model_runner import ( - GPUModelRunner as GPUModelRunnerV1, - ) + return GPUModelRunnerV2(self.vllm_config, self.device) # type: ignore + from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1 - self.model_runner = GPUModelRunnerV1(self.vllm_config, self.device) - - if self.rank == 0: - # If usage stat is enabled, collect relevant info. - report_usage_stats(self.vllm_config) + return GPUModelRunnerV1(self.vllm_config, self.device) def handle_ft_command(self, ft_request): assert self.worker_sentinel is not None diff --git a/vllm/v1/worker/xpu_worker.py b/vllm/v1/worker/xpu_worker.py index 8a8166c435..f7539d2c20 100644 --- a/vllm/v1/worker/xpu_worker.py +++ b/vllm/v1/worker/xpu_worker.py @@ -10,9 +10,7 @@ from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.utils.mem_utils import MemorySnapshot, format_gib from vllm.utils.torch_utils import set_random_seed -from vllm.v1.utils import report_usage_stats from vllm.v1.worker.gpu_worker import Worker, init_worker_distributed_environment -from vllm.v1.worker.workspace import init_workspace_manager from vllm.v1.worker.xpu_model_runner import ( XPUMMEncoderModelRunner, XPUModelRunner, @@ -214,11 +212,9 @@ class XPUWorker(Worker): "worker requested memory: %sGiB", format_gib(self.requested_memory) ) - # Initialize workspace manager - num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1 - init_workspace_manager(self.device, num_ubatches) + self._init_workspace_and_model_runner() - # Construct the model runner + def _make_model_runner(self): if self.use_v2_model_runner: model_runner = ( XPUMMEncoderModelRunner @@ -227,13 +223,7 @@ class XPUWorker(Worker): ) else: model_runner = XPUModelRunner # type: ignore[assignment] - self.model_runner = model_runner( # type: ignore - self.vllm_config, self.device - ) - - if self.rank == 0: - # If usage stat is enabled, collect relevant info. - report_usage_stats(self.vllm_config) + return model_runner(self.vllm_config, self.device) # type: ignore def shutdown(self) -> None: logger.info( From 2df122e65e87dc12870ce5f0180b49f012e7a53b Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:45:00 +0200 Subject: [PATCH 127/248] [ROCm][Perf][GLM-5.3-Flash] "Fit kpool top-k indices to AITER" with a single Triton kernel (#58008) Signed-off-by: simondanielsson Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../v1/attention/test_rocm_glm5next_sparse.py | 44 +++++++++++ .../backends/mla/rocm_aiter_mla_sparse.py | 79 ++++++++++++++++--- 2 files changed, 110 insertions(+), 13 deletions(-) diff --git a/tests/v1/attention/test_rocm_glm5next_sparse.py b/tests/v1/attention/test_rocm_glm5next_sparse.py index ccc9e5c617..e69686f500 100644 --- a/tests/v1/attention/test_rocm_glm5next_sparse.py +++ b/tests/v1/attention/test_rocm_glm5next_sparse.py @@ -8,6 +8,7 @@ import torch from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( _use_rocm_sparse_triton, @@ -26,6 +27,26 @@ def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexp tl.store(output_ptr, _sparse_kv_row_offset(slot, stride)) +def _fit_kpool_indices_reference( + token_indices: torch.Tensor, topk_tokens: int +) -> torch.Tensor: + history = token_indices[:, :topk_tokens] + tail = token_indices[:, topk_tokens:] + valid_history = (history >= 0).sum(dim=1) + valid_tail = (tail >= 0).sum(dim=1) + keep_history = torch.minimum(valid_history, topk_tokens - valid_tail) + + columns = torch.arange(topk_tokens, device=token_indices.device).unsqueeze(0) + tail_offsets = columns - keep_history.unsqueeze(1) + tail_values = torch.gather( + tail, 1, tail_offsets.clamp(min=0, max=tail.shape[1] - 1) + ) + output = torch.where(columns < keep_history.unsqueeze(1), history, tail_values) + valid_output = columns < (keep_history + valid_tail).unsqueeze(1) + return torch.where(valid_output, output, -1) + + +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required") def test_fit_kpool_indices_preserves_tail_and_best_history(): token_indices = torch.tensor( [ @@ -34,6 +55,7 @@ def test_fit_kpool_indices_preserves_tail_and_best_history(): [-1, -1, -1, -1, -1, -1, -1, -1], ], dtype=torch.int32, + device="cuda", ) fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6) @@ -45,6 +67,28 @@ def test_fit_kpool_indices_preserves_tail_and_best_history(): ] +@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required") +@pytest.mark.parametrize( + ("num_tokens", "topk_tokens", "tail_width"), + [(1, 4, 1), (7, 6, 2), (13, 128, 128), (3, 2048, 128)], +) +def test_fit_kpool_indices_matches_eager_reference(num_tokens, topk_tokens, tail_width): + """The Triton fit must reproduce the eager packing rule exactly.""" + set_random_seed(topk_tokens + num_tokens) + width = topk_tokens + tail_width + token_indices = torch.randint(0, 4096, (num_tokens, width), dtype=torch.int32) + valid = torch.rand((num_tokens, width)) < 0.5 + valid[0] = False + valid[-1] = True + token_indices = torch.where(valid, token_indices, -1).cuda() + + fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=topk_tokens) + + torch.testing.assert_close( + fitted, _fit_kpool_indices_reference(token_indices, topk_tokens) + ) + + def test_fit_kpool_indices_exact_width_is_noop(): token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index c5a672e5d4..e473f85948 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -70,6 +70,50 @@ def _use_rocm_sparse_triton( ) +@triton.jit +def _fit_kpool_indices_kernel( + token_indices_ptr, # int32 [num_tokens, NUM_TOPK_TOKENS + TAIL_WIDTH] + out_ptr, # int32 [num_tokens, NUM_TOPK_TOKENS] + ti_stride0, + ti_stride1, + out_stride0, + out_stride1, + NUM_TOPK_TOKENS: tl.constexpr, + TAIL_WIDTH: tl.constexpr, + BLOCK_T: tl.constexpr, + BLOCK_TAIL: tl.constexpr, +): + row_ptr = token_indices_ptr + tl.program_id(0) * ti_stride0 + columns = tl.arange(0, BLOCK_T) + column_mask = columns < NUM_TOPK_TOKENS + history = tl.load(row_ptr + columns * ti_stride1, mask=column_mask, other=-1) + + tail_columns = tl.arange(0, BLOCK_TAIL) + tail = tl.load( + row_ptr + (NUM_TOPK_TOKENS + tail_columns) * ti_stride1, + mask=tail_columns < TAIL_WIDTH, + other=-1, + ) + + valid_history = tl.sum((history >= 0).to(tl.int32)) + valid_tail = tl.sum((tail >= 0).to(tl.int32)) + keep_history = tl.minimum(valid_history, NUM_TOPK_TOKENS - valid_tail) + + tail_offsets = columns - keep_history + tail_values = tl.load( + row_ptr + (NUM_TOPK_TOKENS + tail_offsets) * ti_stride1, + mask=column_mask & (tail_offsets >= 0) & (tail_offsets < TAIL_WIDTH), + other=-1, + ) + fitted = tl.where(columns < keep_history, history, tail_values) + fitted = tl.where(columns < keep_history + valid_tail, fitted, -1) + tl.store( + out_ptr + tl.program_id(0) * out_stride0 + columns * out_stride1, + fitted, + mask=column_mask, + ) + + def fit_kpool_indices_to_aiter( token_indices: torch.Tensor, topk_tokens: int ) -> torch.Tensor: @@ -79,20 +123,29 @@ def fit_kpool_indices_to_aiter( if token_indices.shape[1] == topk_tokens: return token_indices - history = token_indices[:, :topk_tokens] - tail = token_indices[:, topk_tokens:] - valid_history = (history >= 0).sum(dim=1) - valid_tail = (tail >= 0).sum(dim=1) - keep_history = torch.minimum(valid_history, topk_tokens - valid_tail) - - columns = torch.arange(topk_tokens, device=token_indices.device).unsqueeze(0) - tail_offsets = columns - keep_history.unsqueeze(1) - tail_values = torch.gather( - tail, 1, tail_offsets.clamp(min=0, max=tail.shape[1] - 1) + num_tokens, width = token_indices.shape + tail_width = width - topk_tokens + output = torch.empty( + (num_tokens, topk_tokens), + dtype=token_indices.dtype, + device=token_indices.device, ) - output = torch.where(columns < keep_history.unsqueeze(1), history, tail_values) - valid_output = columns < (keep_history + valid_tail).unsqueeze(1) - return torch.where(valid_output, output, -1) + if num_tokens == 0: + return output + + _fit_kpool_indices_kernel[(num_tokens,)]( + token_indices, + output, + token_indices.stride(0), + token_indices.stride(1), + output.stride(0), + output.stride(1), + NUM_TOPK_TOKENS=topk_tokens, + TAIL_WIDTH=tail_width, + BLOCK_T=triton.next_power_of_2(topk_tokens), + BLOCK_TAIL=triton.next_power_of_2(tail_width), + ) + return output @triton.jit From 4e1182a3a609a5165afd6de99b1269099b5d5999 Mon Sep 17 00:00:00 2001 From: Sergey Shlyapnikov Date: Wed, 30 Sep 2026 19:08:21 +0400 Subject: [PATCH 128/248] [ROCm][MoE] Pad the AITER MoE intermediate size at allocation time, and round the expert-group count to a kernel that exists (#55368) Signed-off-by: Sergei Shliapnikov --- tests/kernels/moe/test_rocm_aiter_moe.py | 251 +++++++++++++++++- .../moe/test_rocm_aiter_num_expert_group.py | 136 ++++++++++ .../layers/fused_moe/oracle/unquantized.py | 42 ++- .../router/fused_topk_bias_router.py | 38 ++- 4 files changed, 452 insertions(+), 15 deletions(-) create mode 100644 tests/kernels/moe/test_rocm_aiter_num_expert_group.py diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index 05692852e1..abd4675351 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -30,7 +30,7 @@ import torch.nn.functional as F from tests.kernels.utils import _assert_deterministic from vllm.platforms import current_platform -from vllm.platforms.rocm import on_gfx942, on_gfx950 +from vllm.platforms.rocm import on_cdna, on_gfx942, on_gfx950 from vllm.utils.torch_utils import set_random_seed pytestmark = pytest.mark.skipif( @@ -1317,3 +1317,252 @@ def test_aiter_fused_moe_mi3xx_fp8_accuracy(): pass_rate=1.0, max_violation_factor=1.5, ) + + +# Weight alignment tests -------------------------------------------------- +# +# AITER's CK 2stages MoE kernel rejects an intermediate size not divisible by +# its tile width (64 at or below inter_dim 192, 128 above). Some model + TP +# splits land on an unaligned size (e.g. 1792 / TP=8 = 224), so the AITER path +# rounds the intermediate dim up in ``maybe_roundup_sizes`` and allocates the +# weights at the padded size. + +ALIGNMENT_HIDDEN = 64 +ALIGNMENT_NUM_EXPERTS = 2 + + +def _make_alignment_moe_config(intermediate: int): + from tests.kernels.moe.utils import make_dummy_moe_config + + return make_dummy_moe_config( + num_experts=ALIGNMENT_NUM_EXPERTS, + hidden_dim=ALIGNMENT_HIDDEN, + intermediate_size=intermediate, + ) + + +def _make_aiter_method(moe_config): + """Build the unquantized method with the backend pinned to AITER. + + ``select_unquantized_moe_backend`` needs a real ROCm + AITER runtime, so + stub it out and set the backend directly. + """ + from unittest.mock import patch + + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + ) + from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( + UnquantizedFusedMoEMethod, + ) + + with patch( + "vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method" + ".select_unquantized_moe_backend", + return_value=(UnquantizedMoeBackend.AITER, None), + ): + return UnquantizedFusedMoEMethod(moe_config) + + +def _roundup(method, moe_config, intermediate): + from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig + + return method.maybe_roundup_sizes( + hidden_size=ALIGNMENT_HIDDEN, + intermediate_size_per_partition=intermediate, + act_dtype=moe_config.in_dtype, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), + ) + + +@pytest.mark.parametrize( + ("intermediate", "expected_padded"), + [ + (64, 64), # already aligned, untouched + (96, 128), + (160, 192), + (192, 192), # must stay 192: it is valid and has tuned configs + (224, 256), # K2-Horizon-375B at TP=8 + (256, 256), + (448, 512), # K2-Horizon-375B at TP=4 + (4096, 4096), + ], +) +def test_aiter_moe_roundup_pads_intermediate( + intermediate, + expected_padded, + default_vllm_config, +): + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + aiter_moe_intermediate_alignment, + ) + + moe_config = _make_alignment_moe_config(intermediate) + method = _make_aiter_method(moe_config) + + hidden, padded = _roundup(method, moe_config, intermediate) + + assert padded == expected_padded + assert hidden == ALIGNMENT_HIDDEN + # Padding must not move a size across the <= 192 threshold, or the + # alignment we picked is not the one the kernel dispatches the padded + # shape to. + assert aiter_moe_intermediate_alignment(padded) == ( + aiter_moe_intermediate_alignment(intermediate) + ) + + +@pytest.mark.parametrize("backend_name", ["TRITON", "FLASHINFER_CUTLASS"]) +def test_aiter_moe_roundup_is_not_applied_to_other_backends( + backend_name, + default_vllm_config, +): + """Backends with no alignment requirement keep the unaligned size.""" + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + UnquantizedMoeBackend, + ) + + intermediate = 224 + moe_config = _make_alignment_moe_config(intermediate) + method = _make_aiter_method(moe_config) + method.unquantized_backend = UnquantizedMoeBackend[backend_name] + + _, padded = _roundup(method, moe_config, intermediate) + + assert padded == intermediate + + +@pytest.mark.parametrize("intermediate", [224, 448]) +def test_aiter_moe_reload_zeroes_intermediate_padding( + intermediate, + monkeypatch, + default_vllm_config, +): + """A reload must re-zero pad lanes left dirty in the weight storage. + + The reload writes back only the logical slices, so the pad lanes hold + garbage (NaN here) that the AITER conversion must zero. Two passes also + cover reloading into storage that already holds a previous conversion. + """ + from unittest.mock import MagicMock + + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + aiter_moe_intermediate_alignment, + ) + + alignment = aiter_moe_intermediate_alignment(intermediate) + padded = -(-intermediate // alignment) * alignment + assert padded != intermediate + + moe_config = _make_alignment_moe_config(intermediate) + assert moe_config.is_act_and_mul + method = _make_aiter_method(moe_config) + # RoutedExperts records the rounded-up size after maybe_roundup_sizes. + moe_config.intermediate_size_per_partition = padded + layer = torch.nn.Module() + layer.moe_config = moe_config + + method.create_weights( + layer=layer, + num_experts=ALIGNMENT_NUM_EXPERTS, + hidden_size=ALIGNMENT_HIDDEN, + intermediate_size_per_partition=padded, + params_dtype=torch.float32, + ) + + # Identity shuffle keeps pad lanes sliceable; both stubs need real ROCm. + monkeypatch.setattr( + "vllm.model_executor.layers.fused_moe.oracle.unquantized" + ".rocm_aiter_ops.shuffle_weights", + lambda w13, w2: (w13, w2), + ) + monkeypatch.setattr( + method, + "_init_moe_kernel", + lambda _layer: setattr(method, "moe_kernel", MagicMock()), + ) + + up = padded # up projection starts one padded block into the fused rows + + for _ in range(2): + gate = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN) + up_proj = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN) + down = torch.randn(ALIGNMENT_NUM_EXPERTS, ALIGNMENT_HIDDEN, intermediate) + + # NaN the pads, then write only the logical slices like the loader. + layer.w13_weight.data.fill_(float("nan")) + layer.w2_weight.data.fill_(float("nan")) + layer.w13_weight.data[:, :intermediate].copy_(gate) + layer.w13_weight.data[:, up : up + intermediate].copy_(up_proj) + layer.w2_weight.data[:, :, :intermediate].copy_(down) + + method.process_weights_after_loading(layer) + + w13 = layer.w13_weight + w2 = layer.w2_weight + assert torch.equal(w13[:, :intermediate], gate) + assert torch.equal(w13[:, up : up + intermediate], up_proj) + assert torch.equal(w2[:, :, :intermediate], down) + assert torch.all(w13[:, intermediate:up] == 0) + assert torch.all(w13[:, up + intermediate :] == 0) + assert torch.all(w2[:, :, intermediate:] == 0) + + +def _aiter_accepts_intermediate(intermediate: int, num_tokens: int) -> bool: + """Run the real AITER CK MoE kernel and report whether it dispatched.""" + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + ActivationMethod, + QuantMethod, + ) + + case = _make_moe_case( + num_tokens=num_tokens, + hidden_dim=1024, + intermediate_dim=intermediate, + num_experts=8, + topk=2, + seed=0, + ) + try: + _run_fused_moe( + case["hidden_states"], + case["w1"], + case["w2"], + case["topk_weights"], + case["topk_ids"], + activation_method=int(ActivationMethod.SILU), + quant_method=int(QuantMethod.NO), + ) + except RuntimeError: + return False + return True + + +@pytest.mark.skipif( + not on_cdna(), + reason="CDNA ROCm only", +) +@pytest.mark.parametrize("intermediate", [192, 224, 256]) +# With topk=2 and 8 experts, these token counts select block_m 32 / 64 / 128 +# respectively (AITER's get_block_size_M, on both 256- and 304-CU parts). +@pytest.mark.parametrize("num_tokens", [1024, 2560, 5120]) +def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens): + """The alignment rule must hold at every reachable ``block_m``. + + Stage 1 checks ``NPerBlock`` and stage 2 ``KPerBlock``; both vary with + ``block_m``, so a size validated at one tile size says nothing about the + others. + """ + from vllm.model_executor.layers.fused_moe.oracle.unquantized import ( + aiter_moe_intermediate_alignment, + ) + + _assert_aiter_supported() + + alignment = aiter_moe_intermediate_alignment(intermediate) + padded = -(-intermediate // alignment) * alignment + + assert _aiter_accepts_intermediate(intermediate, num_tokens) == ( + intermediate % alignment == 0 + ) + assert _aiter_accepts_intermediate(padded, num_tokens) diff --git a/tests/kernels/moe/test_rocm_aiter_num_expert_group.py b/tests/kernels/moe/test_rocm_aiter_num_expert_group.py new file mode 100644 index 0000000000..40a14bb86c --- /dev/null +++ b/tests/kernels/moe/test_rocm_aiter_num_expert_group.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the AITER biased_grouped_topk expert-group count. + +The AITER kernel is only instantiated for ``NUM_GRP`` in {1, 2, 4, 8}, so the +group count derived from ``num_experts`` has to be rounded to a supported +value that still divides ``num_experts``. Grouping is a no-op on this path +(``topk_group == num_expert_group``), so any such value routes identically; +the constraint is purely about which kernel exists. +""" + +import pytest +import torch + +from vllm._aiter_ops import rocm_aiter_ops +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + AITER_MAX_EXPERTS_PER_GROUP as MAX_EXPERTS_PER_GROUP, +) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + AITER_SUPPORTED_NUM_GRP as SUPPORTED_NUM_GRP, +) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + _aiter_can_use_biased_grouped_topk, + _aiter_get_num_expert_group, +) +from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import ( + grouped_topk, +) +from vllm.platforms import current_platform +from vllm.platforms.rocm import on_cdna + +pytestmark = pytest.mark.skipif( + not current_platform.is_rocm(), reason="ROCm-specific tests" +) + + +@pytest.mark.parametrize( + ("num_experts", "expected"), + [ + (8, 1), # fits in a single group + (32, 1), # exactly at the per-group limit + (64, 2), + (128, 4), + (256, 8), + (72, 8), # naive ceil gives 3 -> unsupported, rounds up to 8 + (96, 8), # naive ceil gives 3 -> unsupported, rounds up to 8 + (160, 8), # naive ceil gives 5 -> unsupported, rounds up to 8 + (192, 8), # naive ceil gives 6 -> unsupported, rounds up to 8 + # No supported NUM_GRP divides these within the group-size limit, so + # the naive value is kept and the call site declines the kernel. + (33, 3), + (129, 43), + (257, 257), + (320, 10), + (384, 12), + ], +) +def test_group_count_and_guard(num_experts, expected): + g = _aiter_get_num_expert_group(num_experts) + assert g == expected + # The router asserts both unconditionally; rounding must never break them. + assert num_experts % g == 0 + assert num_experts // g <= MAX_EXPERTS_PER_GROUP + + supported = g in SUPPORTED_NUM_GRP + # Ample topk, so only the membership check can decline. + assert _aiter_can_use_biased_grouped_topk(num_experts, topk=64) == supported + if supported: + assert _aiter_can_use_biased_grouped_topk(num_experts, topk=g) + assert not _aiter_can_use_biased_grouped_topk(num_experts, topk=g - 1) + + +@pytest.mark.skipif(not on_cdna(), reason="CDNA ROCm only") +@pytest.mark.parametrize( + "num_experts", + [ + 64, # g == 2, not rounded -- covers the no-op claim itself + 128, # g == 4, not rounded + 256, # g == 8, not rounded + 96, # naive 3 -> rounded to 8 + 192, # naive 6 -> rounded to 8 + ], +) +def test_rounded_group_count_routes_like_the_reference(num_experts): + """The rounded group count must route identically to ``grouped_topk``. + + Gating is continuous random, so exact score ties -- the one place the two + may legitimately differ -- have measure zero. + """ + torch.manual_seed(num_experts) + device = "cuda" + topk = 8 + num_tokens = 83 # not a multiple of any warp/tile size + g = _aiter_get_num_expert_group(num_experts) + assert _aiter_can_use_biased_grouped_topk(num_experts, topk) + + gating = torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device) + bias = torch.randn(num_experts, dtype=torch.float32, device=device) + + ref_weights, ref_ids = grouped_topk( + hidden_states=torch.empty(num_tokens, 1, device=device), + gating_output=gating, + topk=topk, + renormalize=True, + num_expert_group=g, + topk_group=g, + scoring_func="sigmoid", + e_score_correction_bias=bias, + ) + + weights = torch.empty(num_tokens, topk, dtype=torch.float32, device=device) + ids = torch.empty(num_tokens, topk, dtype=torch.int32, device=device) + rocm_aiter_ops.biased_grouped_topk( + gating, + bias, + weights, + ids, + num_expert_group=g, + topk_group=g, + need_renorm=True, + ) + + # Neither side promises an order within a token's top-k, so sort by expert + # id and carry the weights through the same permutation -- sorting the two + # independently would not catch a weight attached to the wrong expert. + order = ids.argsort(dim=-1) + ref_order = ref_ids.argsort(dim=-1) + torch.testing.assert_close( + ids.gather(1, order), ref_ids.to(torch.int32).gather(1, ref_order) + ) + torch.testing.assert_close( + weights.gather(1, order), + ref_weights.gather(1, ref_order), + atol=1e-4, + rtol=1e-4, + ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 2c84e532cc..25b55670d1 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -371,6 +371,16 @@ def select_unquantized_moe_backend( ) +def aiter_moe_intermediate_alignment(intermediate: int) -> int: + """Intermediate-size alignment required by AITER's CK 2stages MoE kernel. + + AITER dispatches on ``inter_dim <= 192``: below the threshold both stages + use 64-wide tiles, above it at least one stage uses a 128-wide tile, and + CK's ``IsSupportedArgument`` rejects a size not divisible by that width. + """ + return 64 if intermediate <= 192 else 128 + + def unquantized_round_up_hidden_size_and_intermediate_size( backend: UnquantizedMoeBackend, hidden_size: int, @@ -379,9 +389,30 @@ def unquantized_round_up_hidden_size_and_intermediate_size( """Round up dimensions before allocation to satisfy the selected kernel.""" if backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM: intermediate_size = round_up(intermediate_size, 128) + elif backend == UnquantizedMoeBackend.AITER: + intermediate_size = round_up( + intermediate_size, aiter_moe_intermediate_alignment(intermediate_size) + ) return hidden_size, intermediate_size +def _zero_intermediate_padding( + moe_config: FusedMoEConfig, + w13_weight: torch.Tensor, + w2_weight: torch.Tensor, +) -> None: + """Zero intermediate padding; reloads rewrite only the checkpoint slices.""" + unpadded = moe_config.intermediate_size_per_partition_unpadded + assert unpadded is not None + padded = moe_config.intermediate_size_per_partition + if padded <= unpadded: + return + w13_weight[:, unpadded:padded].zero_() + if moe_config.is_act_and_mul: + w13_weight[:, padded + unpadded :].zero_() + w2_weight[:, :, unpadded:].zero_() + + def convert_to_unquantized_kernel_format( unquantized_backend: UnquantizedMoeBackend, moe_config: FusedMoEConfig, @@ -424,6 +455,7 @@ def convert_to_unquantized_kernel_format( return layout.full_gate_weight, layout.full_down_weight if unquantized_backend == UnquantizedMoeBackend.AITER: + _zero_intermediate_padding(moe_config, w13_weight, w2_weight) w13_weight, w2_weight = rocm_aiter_ops.shuffle_weights(w13_weight, w2_weight) w13_weight.is_shuffled = True w2_weight.is_shuffled = True @@ -445,15 +477,7 @@ def convert_to_unquantized_kernel_format( ) moe_config.intermediate_size_per_partition = padded_intermediate - # Reloads only overwrite checkpoint slices. An earlier in-place - # permutation can leave nonzero values in the raw padding slots. - unpadded = moe_config.intermediate_size_per_partition_unpadded - assert unpadded is not None - if padded_intermediate > unpadded: - w13_weight[:, unpadded:padded_intermediate].zero_() - if is_act_and_mul: - w13_weight[:, padded_intermediate + unpadded :].zero_() - w2_weight[:, :, unpadded:].zero_() + _zero_intermediate_padding(moe_config, w13_weight, w2_weight) _cache_permute_indices: dict[torch.Size, torch.Tensor] = {} convert_moe_weights_to_flashinfer_trtllm_block_layout( diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py index 2df66d063a..656b3cc585 100644 --- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py +++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py @@ -19,6 +19,11 @@ from vllm.model_executor.layers.fused_moe.router.dsv4_topk import ( dsv4_topk, ) +# AITER instantiates biased_grouped_topk only for these NUM_GRP values, with at +# most this many experts per group. +AITER_SUPPORTED_NUM_GRP = (1, 2, 4, 8) +AITER_MAX_EXPERTS_PER_GROUP = 32 + def _get_padding_mask(num_tokens: int) -> torch.Tensor | None: if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available(): @@ -104,17 +109,40 @@ def vllm_topk_softplus_sqrt( @functools.lru_cache(maxsize=8) def _aiter_get_num_expert_group(num_experts: int) -> int: - _AITER_MAX_EXPERTS_PER_GROUP = 32 - g = max(1, -(-num_experts // _AITER_MAX_EXPERTS_PER_GROUP)) + g = max(1, -(-num_experts // AITER_MAX_EXPERTS_PER_GROUP)) while num_experts % g != 0: g += 1 assert num_experts % g == 0, f"{num_experts=} not divisible by {g=}" - assert num_experts // g <= _AITER_MAX_EXPERTS_PER_GROUP, ( - f"group size {num_experts // g} exceeds limit {_AITER_MAX_EXPERTS_PER_GROUP}" + # Grouping is a no-op here (topk_group == num_expert_group), so any divisor + # within the limit routes identically and we are free to pick one AITER has + # a kernel for. If none fits, the unsupported value is returned as-is -- + # callers must gate on _aiter_can_use_biased_grouped_topk. + if g not in AITER_SUPPORTED_NUM_GRP: + g = next( + ( + c + for c in sorted(AITER_SUPPORTED_NUM_GRP, reverse=True) + if num_experts % c == 0 + and num_experts // c <= AITER_MAX_EXPERTS_PER_GROUP + ), + g, + ) + assert num_experts // g <= AITER_MAX_EXPERTS_PER_GROUP, ( + f"group size {num_experts // g} exceeds limit {AITER_MAX_EXPERTS_PER_GROUP}" ) return g +def _aiter_can_use_biased_grouped_topk(num_experts: int, topk: int) -> bool: + """Whether AITER has an instantiated kernel for this routing shape. + + Both checks are needed: an unsupported group count can still be small + enough to satisfy ``topk >= g`` (num_experts=33 gives g=3). + """ + g = _aiter_get_num_expert_group(num_experts) + return topk >= g and g in AITER_SUPPORTED_NUM_GRP + + def fused_topk_bias( hidden_states: torch.Tensor, gating_output: torch.Tensor, @@ -229,7 +257,7 @@ def fused_topk_bias( M = hidden_states.size(0) num_experts = gating_output.shape[-1] num_expert_group = _aiter_get_num_expert_group(num_experts) - if topk >= num_expert_group: + if _aiter_can_use_biased_grouped_topk(num_experts, topk): topk_weights = torch.empty( M, topk, dtype=torch.float32, device=hidden_states.device ) From 17e9295dd565d0a2c2fe7404ecd27705aafbf354 Mon Sep 17 00:00:00 2001 From: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:09:11 +0800 Subject: [PATCH 129/248] [BugFix][Multimodal] Pick worst-case DeepSeek-V4 VL dummy image size (#59271) (#59373) Signed-off-by: Yida Weng <84164537+YidaWeng@users.noreply.github.com> Co-authored-by: Grok --- .../processing/test_deepseek_v4_vl.py | 53 +++++++++++++++++++ .../deepseek_v4/common/mm_preprocess.py | 35 +++++++++--- 2 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 tests/models/multimodal/processing/test_deepseek_v4_vl.py diff --git a/tests/models/multimodal/processing/test_deepseek_v4_vl.py b/tests/models/multimodal/processing/test_deepseek_v4_vl.py new file mode 100644 index 0000000000..5358fae970 --- /dev/null +++ b/tests/models/multimodal/processing/test_deepseek_v4_vl.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek-V4 VL processing helpers.""" + +from PIL import Image + +from vllm.models.deepseek_v4.common.mm_preprocess import ( + DeepseekV4VLProcessingInfo, + DeepseekV4VLProcessor, + build_image_block_pad_free, +) +from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config + + +class _ConfigCtx: + def __init__(self, config: DeepseekV4Config) -> None: + self._config = config + + def get_hf_config(self, *args, **kwargs): + return self._config + + +def _feature_counts( + processor: DeepseekV4VLProcessor, width: int, height: int +) -> tuple[int, int]: + out = processor(images=[Image.new("RGB", (width, height))]) + n_llm_h, n_llm_w = out["llm_grid"][0].tolist() + types, _ = build_image_block_pad_free(n_llm_h, n_llm_w) + return len(types), out["patches"].shape[0] + + +def test_get_image_size_with_most_features_beats_wide_real_image(): + """Dummy size must maximize tokens/patches within the budget (#59271). + + A square under-counts because each aligner row pays an IMAGE_NEW_LINE. + """ + config = DeepseekV4Config(vision_n_layers=24) + info = DeepseekV4VLProcessingInfo(_ConfigCtx(config)) + processor = DeepseekV4VLProcessor(config) + + dummy = info.get_image_size_with_most_features() + dummy_features = _feature_counts(processor, dummy.width, dummy.height) + wide_features = _feature_counts(processor, 1932, 336) + + assert dummy == (1932, 336) + assert wide_features <= dummy_features + + +def test_get_image_size_with_most_features_respects_max_wh_ratio(): + config = DeepseekV4Config(vision_n_layers=24) + info = DeepseekV4VLProcessingInfo(_ConfigCtx(config)) + dummy = info.get_image_size_with_most_features() + assert dummy.width / dummy.height <= config.vision_max_wh_ratio diff --git a/vllm/models/deepseek_v4/common/mm_preprocess.py b/vllm/models/deepseek_v4/common/mm_preprocess.py index d28848bb8f..62a5c2070f 100644 --- a/vllm/models/deepseek_v4/common/mm_preprocess.py +++ b/vllm/models/deepseek_v4/common/mm_preprocess.py @@ -354,15 +354,34 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo): hf_config = self.get_hf_config() patch_size = hf_config.vision_patch_size downsample_ratio = hf_config.vision_downsample_ratio - # A square maximizes the ViT patch count (area) within the token - # budget; solve the budget-derived size directly to keep the dummy - # image small. + max_wh_ratio = hf_config.vision_max_wh_ratio + # Every aligner row costs an extra IMAGE_NEW_LINE token, so height is + # more expensive than width and a square is *not* optimal: search the + # LLM grids that fit the budget for the one with the most ViT patches + # (ties broken by the longer sentinel block). budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1) - side = budget * patch_size * downsample_ratio - _, _, best_h, best_w, _ = solve_resize_ratio( - side, side, patch_size, downsample_ratio, budget - ) - return ImageSize(width=best_w, height=best_h) + step = patch_size * downsample_ratio + best_grid = (1, 1) + best_key = (0, 0) + for n_llm_h in range(1, budget): + if ( + grid_tokens(n_llm_h * step, step, patch_size, downsample_ratio)[2] + > budget + ): + break + for n_llm_w in range(1, budget): + if max_wh_ratio is not None and n_llm_w > n_llm_h * max_wh_ratio: + break + num_tokens = grid_tokens( + n_llm_h * step, n_llm_w * step, patch_size, downsample_ratio + )[2] + if num_tokens > budget: + break + key = (n_llm_h * n_llm_w, num_tokens) + if key > best_key: + best_key, best_grid = key, (n_llm_h, n_llm_w) + n_llm_h, n_llm_w = best_grid + return ImageSize(width=n_llm_w * step, height=n_llm_h * step) class DeepseekV4VLDummyInputsBuilder( From fc2c801ad93bcd8d7cd863645ee3ac8533cc85be Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:18:50 +0100 Subject: [PATCH 130/248] Disable mypy `arg-type` and `assignment` checks in tests (#59428) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- pyproject.toml | 5 + tests/benchmarks/test_bfcl_dataset.py | 4 +- ...est_custom_dataset_chat_template_kwargs.py | 4 +- .../passes/distributed/test_async_tp.py | 2 +- .../distributed/test_fusion_all_reduce.py | 4 +- .../distributed/test_sequence_parallelism.py | 2 +- tests/compile/passes/test_pass_manager.py | 2 +- .../passes/test_silu_mul_quant_fusion.py | 2 +- tests/config/test_config_generation.py | 4 +- tests/config/test_multimodal_config.py | 10 +- tests/conftest.py | 2 +- .../test_anthropic_messages_conversion.py | 2 +- .../cohere/test_cohere_chat_message.py | 4 +- tests/entrypoints/cohere/test_protocol.py | 4 +- .../test_api_server_process_manager.py | 6 +- .../llm/test_mm_processor_kwargs.py | 2 +- .../entrypoints/openai/test_render_parity.py | 2 +- .../pooling/embed/test_io_processor.py | 2 +- .../pooling/scoring/test_io_processor_unit.py | 4 +- .../test_jina_ranking_io_processor_unit.py | 6 +- .../entrypoints/pooling/test_io_processor.py | 4 +- .../test_authentication_middleware.py | 2 +- .../serve/tokenize/test_tokenization.py | 4 +- .../test_qwen3_asr_sanitize_prompt.py | 2 +- .../entrypoints/unit_tests/test_chat_utils.py | 4 +- tests/jit_monitor/test_hooks.py | 2 +- .../attention/test_attention_selector.py | 8 +- tests/kernels/attention/test_cpu_attn.py | 2 +- tests/kernels/attention/test_minimax_m3.py | 2 +- ...st_minimax_m3_msa_cutlass_sparse_decode.py | 10 +- .../test_rocm_aiter_mla_causal_verify_mask.py | 2 +- .../test_rocm_aiter_mla_fp8_prefill.py | 4 +- .../attention/test_rocm_aiter_mla_sink.py | 12 +-- ...est_rocm_aiter_mla_sparse_metadata_sync.py | 4 +- .../test_rocm_paged_mxfp4_indexer.py | 6 +- .../attention/test_rocm_triton_attn_dsv4.py | 4 +- .../mamba/test_gdn_forward_core_split.py | 2 +- tests/kernels/moe/test_cutlass_moe.py | 2 +- tests/kernels/moe/test_ocp_mx_moe.py | 2 +- tests/kernels/quantization/test_block_fp8.py | 2 +- .../kernels/quantization/test_cutlass_w4a8.py | 2 +- .../quantization/test_marlin_tile_padding.py | 6 +- .../quantization/test_nvfp4_emulation.py | 2 +- .../layers/test_fused_shared_expert.py | 6 +- .../layers/test_mla_short_prefill_indexer.py | 8 +- .../layers/test_pooler_activations.py | 4 +- .../layers/test_pooler_methods.py | 2 +- .../tensorizer_loader/test_tensorizer.py | 2 +- .../model_loader/test_weight_cache.py | 2 +- .../language/generation/test_mistral.py | 6 +- .../generation/vlm_utils/model_utils.py | 2 +- tests/models/multimodal/pooling/test_clip.py | 2 +- .../pooling/test_llama_nemotron_vl.py | 2 +- .../models/multimodal/pooling/test_siglip.py | 2 +- .../multimodal/processing/test_qwen2_vl.py | 4 +- tests/models/transformers/test_backend.py | 4 +- tests/multimodal/media/test_video.py | 2 +- tests/multimodal/test_audio.py | 2 +- tests/multimodal/test_cache.py | 24 ++--- tests/multimodal/test_processing.py | 8 +- tests/multimodal/test_registry.py | 2 +- tests/multimodal/test_video.py | 2 +- tests/parser/cohere/test_structural_tags.py | 2 +- .../bitsandbytes/test_bitsandbytes.py | 2 +- .../bitsandbytes/test_transformers.py | 4 +- .../gguf/test_gguf_plugin_multimodal.py | 2 +- tests/plugins_tests/test_endpoint_plugins.py | 4 +- .../test_oot_registration_offline.py | 2 +- .../test_stats_logger_plugins.py | 2 +- tests/quantization/test_auto_gptq.py | 4 +- tests/quantization/test_auto_round.py | 12 +-- tests/quantization/test_fp8.py | 4 +- tests/quantization/test_moe_wna16.py | 8 +- tests/quantization/test_online.py | 8 +- tests/quantization/test_quark.py | 4 +- .../test_trtllm_nvfp4_hidden_dim_padding.py | 2 +- tests/quantization/test_turboquant.py | 2 +- .../test_base_thinking_reasoning_parser.py | 2 +- tests/renderers/test_cohere.py | 100 +++++++++--------- tests/renderers/test_completions.py | 2 +- tests/samplers/test_beam_search.py | 6 +- tests/samplers/test_beam_search_online.py | 4 +- tests/standalone_tests/lazy_imports.py | 2 +- tests/test_audio_media_size_precheck.py | 2 +- tests/tokenizers_/test_mistral.py | 10 +- tests/tokenizers_/test_registry.py | 2 +- .../test_ernie45_moe_tool_parser.py | 4 +- tests/tool_parsers/test_jamba_tool_parser.py | 6 +- .../test_qwen3coder_tool_parser.py | 2 +- .../tool_parsers/test_step3p5_tool_parser.py | 2 +- tests/tool_parsers/test_xlam_tool_parser.py | 8 +- .../test_gemma4_responses_adjust_request.py | 2 +- tests/tool_use/test_kimi_k3_tool_parser.py | 2 +- .../processors/test_voxtral.py | 2 +- tests/transformers_utils/test_processor.py | 4 +- tests/utils.py | 2 +- tests/utils_/test_argparse_utils.py | 18 ++-- .../v1/attention/test_mla_prefill_selector.py | 4 +- .../v1/attention/test_sparse_mla_backends.py | 6 +- tests/v1/cudagraph/test_cudagraph_manager.py | 4 +- tests/v1/cudagraph/test_encoder_cudagraph.py | 8 +- tests/v1/distributed/test_eagle_dp.py | 2 +- .../v1/engine/test_startup_watch_processes.py | 4 +- tests/v1/executor/test_executor.py | 4 +- .../unit/offloading_connector/test_metrics.py | 18 ++-- .../unit/test_mooncake_store_worker.py | 2 +- .../kv_connector/unit/test_nixl_connector.py | 2 +- .../unit/test_nixl_push_connector.py | 2 +- .../v1/kv_offload/tiering/p2p/test_manager.py | 82 +++++++------- .../kv_offload/tiering/p2p/test_sessions.py | 30 +++--- tests/v1/sample/test_dry.py | 6 +- tests/v1/sample/test_logprobs.py | 6 +- tests/v1/sample/test_trace_replay_params.py | 2 +- tests/v1/simple_kv_offload/test_worker.py | 6 +- .../spec_decode/test_adaptive_verification.py | 10 +- tests/v1/spec_decode/test_dflash2.py | 16 +-- tests/v1/spec_decode/test_dflash_lookahead.py | 8 +- .../spec_decode/test_dsv4_mtp_local_argmax.py | 4 +- .../spec_decode/test_extract_hidden_states.py | 8 +- tests/v1/spec_decode/test_lilicorr.py | 22 ++-- .../v1/spec_decode/test_llm_base_proposer.py | 2 +- .../test_llm_base_proposer_sampling.py | 2 +- .../test_rejection_sampler_utils.py | 2 +- .../test_backend_xgrammar_lark.py | 4 +- .../test_backend_xgrammar_plamo3.py | 6 +- .../test_structured_output_manager.py | 2 +- tests/v1/structured_output/test_validation.py | 20 ++-- tests/v1/worker/test_attn_utils.py | 10 +- tests/v1/worker/test_gpu_kv_connector.py | 10 +- .../worker/test_gpu_worker_weight_transfer.py | 2 +- .../v1/worker/test_late_interaction_runner.py | 2 +- 131 files changed, 402 insertions(+), 409 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2098e097bf..b2658168d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,11 @@ ignore_missing_imports = true check_untyped_defs = true follow_imports = "silent" +[[tool.mypy.overrides]] +module = "tests.*" +disable_error_code = ["arg-type", "assignment"] +enable_error_code = ["method-assign"] + [tool.pytest.ini_options] markers = [ "slow_test", diff --git a/tests/benchmarks/test_bfcl_dataset.py b/tests/benchmarks/test_bfcl_dataset.py index 222a57a33f..d191922319 100644 --- a/tests/benchmarks/test_bfcl_dataset.py +++ b/tests/benchmarks/test_bfcl_dataset.py @@ -271,7 +271,7 @@ def test_bfcl_prompt_len_includes_tools(tmp_path: Path) -> None: args.num_prompts = 1 with _patch_hf_api(fake_download): - samples = get_samples(args, fake) # type: ignore[arg-type] + samples = get_samples(args, fake) assert len(samples) == 1 assert captured["tools"] is not None, ( @@ -311,7 +311,7 @@ def test_bfcl_prompt_len_falls_back_when_tokenizer_rejects_tools( args.num_prompts = 1 with _patch_hf_api(fake_download): - samples = get_samples(args, _LegacyTokenizer()) # type: ignore[arg-type] + samples = get_samples(args, _LegacyTokenizer()) assert len(samples) == 1 assert samples[0].prompt_len > 0 diff --git a/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py b/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py index 3817fbbb86..149ea6625a 100644 --- a/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py +++ b/tests/benchmarks/test_custom_dataset_chat_template_kwargs.py @@ -58,7 +58,7 @@ def test_chat_template_kwargs_forwarded(tmp_path: Path) -> None: _write_one(jsonl) tok = _RecordingTokenizer() - get_samples(_args(str(jsonl), {"thinking": True}), tok) # type: ignore[arg-type] + get_samples(_args(str(jsonl), {"thinking": True}), tok) assert tok.captured_kwargs == {"thinking": True} @@ -70,6 +70,6 @@ def test_chat_template_kwargs_default_is_noop(tmp_path: Path) -> None: _write_one(jsonl) tok = _RecordingTokenizer() - get_samples(_args(str(jsonl), None), tok) # type: ignore[arg-type] + get_samples(_args(str(jsonl), None), tok) assert tok.captured_kwargs == {} diff --git a/tests/compile/passes/distributed/test_async_tp.py b/tests/compile/passes/distributed/test_async_tp.py index 6601f7a2f0..c0fd4c00a8 100644 --- a/tests/compile/passes/distributed/test_async_tp.py +++ b/tests/compile/passes/distributed/test_async_tp.py @@ -285,7 +285,7 @@ def test_async_tp_pass_requires_full_graph_compilation(): ] async_tp_pass = object.__new__(AsyncTPPass) - async_tp_pass.compilation_config = vllm_config.compilation_config # type: ignore[assignment] + async_tp_pass.compilation_config = vllm_config.compilation_config with pytest.raises( AssertionError, match="AsyncTPPass requires full-graph compilation" diff --git a/tests/compile/passes/distributed/test_fusion_all_reduce.py b/tests/compile/passes/distributed/test_fusion_all_reduce.py index a7b4cf0ed0..1c99cc2533 100644 --- a/tests/compile/passes/distributed/test_fusion_all_reduce.py +++ b/tests/compile/passes/distributed/test_fusion_all_reduce.py @@ -111,7 +111,7 @@ def test_fused_ar_workspace_hidden_dim( target_hidden: int, draft_hidden: int | None, expected: int ): config = _make_workspace_hidden_config(target_hidden, draft_hidden) - assert _fused_ar_workspace_hidden_dim(config) == expected # type: ignore[arg-type] + assert _fused_ar_workspace_hidden_dim(config) == expected def test_fused_ar_workspace_hidden_dim_spec_without_draft(): @@ -120,7 +120,7 @@ def test_fused_ar_workspace_hidden_dim_spec_without_draft(): model_config=SimpleNamespace(get_hidden_size=lambda: 2048), speculative_config=SimpleNamespace(draft_model_config=None), ) - assert _fused_ar_workspace_hidden_dim(config) == 2048 # type: ignore[arg-type] + assert _fused_ar_workspace_hidden_dim(config) == 2048 class TestAllReduceRMSNormModel(torch.nn.Module): diff --git a/tests/compile/passes/distributed/test_sequence_parallelism.py b/tests/compile/passes/distributed/test_sequence_parallelism.py index b5c390e24c..c40d75f675 100644 --- a/tests/compile/passes/distributed/test_sequence_parallelism.py +++ b/tests/compile/passes/distributed/test_sequence_parallelism.py @@ -222,7 +222,7 @@ def test_sequence_parallelism_pass_requires_full_graph_compilation(): ] sequence_parallelism_pass = object.__new__(SequenceParallelismPass) - sequence_parallelism_pass.compilation_config = vllm_config.compilation_config # type: ignore[assignment] + sequence_parallelism_pass.compilation_config = vllm_config.compilation_config sequence_parallelism_pass.min_token_num = 1 with pytest.raises( diff --git a/tests/compile/passes/test_pass_manager.py b/tests/compile/passes/test_pass_manager.py index 9ba9892289..459921bae8 100644 --- a/tests/compile/passes/test_pass_manager.py +++ b/tests/compile/passes/test_pass_manager.py @@ -28,7 +28,7 @@ def test_bad_callable(): pass_manager.configure(config) with pytest.raises(AssertionError): - pass_manager.add(simple_callable) # type: ignore[arg-type] + pass_manager.add(simple_callable) # Pass that inherits from InductorPass diff --git a/tests/compile/passes/test_silu_mul_quant_fusion.py b/tests/compile/passes/test_silu_mul_quant_fusion.py index a397dd9406..fa25bd0f20 100644 --- a/tests/compile/passes/test_silu_mul_quant_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_fusion.py @@ -331,7 +331,7 @@ def test_fusion_silu_and_mul_quant( backend = TestBackend(*passes) model = model_class( hidden_size=hidden_size, - force_kernel=force_kernel, # type: ignore[arg-type] + force_kernel=force_kernel, x=x, dtype=dtype, ) diff --git a/tests/config/test_config_generation.py b/tests/config/test_config_generation.py index a6908f35e5..eea9e8fa4a 100644 --- a/tests/config/test_config_generation.py +++ b/tests/config/test_config_generation.py @@ -133,7 +133,7 @@ def test_get_diff_sampling_param_includes_penalties(): }, ) - diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type] + diff = ModelConfig.get_diff_sampling_param(fake) assert diff["presence_penalty"] == 1.5 assert diff["frequency_penalty"] == 0.5 @@ -161,7 +161,7 @@ def test_generation_config_penalties_survive_diff_sampling_filter(): }, ) - diff = ModelConfig.get_diff_sampling_param(fake) # type: ignore[arg-type] + diff = ModelConfig.get_diff_sampling_param(fake) assert diff["presence_penalty"] == 1.1 assert diff["frequency_penalty"] == 0.7 diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 6c8bead2d5..c1d4cf6454 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -40,18 +40,18 @@ def test_supports_multimodal_inputs(model_id, limit_mm_per_prompt, expected): def test_mm_encoder_attn_backend_str_conversion(): - config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") # type: ignore[arg-type] + config = MultiModalConfig(mm_encoder_attn_backend="FLASH_ATTN") assert config.mm_encoder_attn_backend == AttentionBackendEnum.FLASH_ATTN def test_mm_encoder_attn_backend_invalid(): with pytest.raises(ValueError): - MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type] + MultiModalConfig(mm_encoder_attn_backend="not_a_backend") def test_mm_hasher_algorithm_invalid(): with pytest.raises(ValueError, match="mm_hasher_algorithm"): - MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type] + MultiModalConfig(mm_hasher_algorithm="md5") def test_mm_encoder_attn_backend_hash_updates(): @@ -560,7 +560,7 @@ def _resolve_mm_processor_device( """Run the `auto` resolution and report where the processor ended up.""" mm_config = MultiModalConfig( mm_processor_kwargs={} if device is None else {"device": device}, - mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type] + mm_tensor_ipc=mm_tensor_ipc, ) model_config = MagicMock(spec=ModelConfig) model_config.multimodal_config = mm_config @@ -662,7 +662,7 @@ def _resolve_mm_video_decode_device( the resulting video media IO kwargs.""" mm_config = MultiModalConfig( mm_processor_kwargs={} if device is None else {"device": device}, - mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type] + mm_tensor_ipc=mm_tensor_ipc, media_io_kwargs={} if video_kwargs is None else {"video": dict(video_kwargs)}, ) model_config = MagicMock(spec=ModelConfig) diff --git a/tests/conftest.py b/tests/conftest.py index c1380f61cd..952c328f91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1163,7 +1163,7 @@ class VllmRunner: output_logprobs = sample.logprobs if include_prompt_token_ids: outputs.append( - ( # type: ignore[arg-type] + ( output_ids, output_str, output_logprobs, diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 23246d4380..d14f24eba0 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -1447,7 +1447,7 @@ class TestDetectMergeInlineSystem: model_config=None, ) assert ( - AnthropicServingMessages._should_merge_inline_system(online_renderer) # type: ignore[arg-type] + AnthropicServingMessages._should_merge_inline_system(online_renderer) is expected ) assert mock_logger.warning_once.called is expected diff --git a/tests/entrypoints/cohere/test_cohere_chat_message.py b/tests/entrypoints/cohere/test_cohere_chat_message.py index 8c297bf19a..2ac30f7188 100644 --- a/tests/entrypoints/cohere/test_cohere_chat_message.py +++ b/tests/entrypoints/cohere/test_cohere_chat_message.py @@ -29,7 +29,7 @@ from vllm.entrypoints.cohere.cohere_chat_message import ( class TestCitationSource: def test_invalid_type_rejected(self): with pytest.raises(ValidationError): - CitationSource(type="other") # type: ignore[arg-type] + CitationSource(type="other") def test_none_fields_excluded_from_dump(self): s = CitationSource(type="document", id="d1") @@ -65,7 +65,7 @@ class TestCitationSource: class TestCitation: def test_invalid_type_rejected(self): with pytest.raises(ValidationError): - Citation(type="OTHER") # type: ignore[arg-type] + Citation(type="OTHER") def test_dump_excludes_none_fields(self): c = Citation(start=0, end=5, text="hello") diff --git a/tests/entrypoints/cohere/test_protocol.py b/tests/entrypoints/cohere/test_protocol.py index 5391686c69..dc37ff7cdd 100644 --- a/tests/entrypoints/cohere/test_protocol.py +++ b/tests/entrypoints/cohere/test_protocol.py @@ -136,7 +136,7 @@ class TestCohereChatV2Response: with pytest.raises(ValidationError): CohereChatV2Response( id="r1", - finish_reason="NOT_A_REASON", # type: ignore[arg-type] + finish_reason="NOT_A_REASON", message=msg, ) @@ -246,5 +246,5 @@ class TestStreamingEventTypeField: MessageStartEvent( id="a", delta={"message": {"role": "assistant"}}, - type="other", # type: ignore[arg-type] + type="other", ) diff --git a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py index b4390f2dc4..10858b6da4 100644 --- a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py +++ b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py @@ -308,7 +308,7 @@ def test_external_process_monitoring(api_server_args): try: wait_for_completion_or_failure( api_server_manager=manager, - coordinator=mock_coordinator, # type: ignore[arg-type] + coordinator=mock_coordinator, ) except Exception as e: result["exception"] = e @@ -377,7 +377,7 @@ def test_gather_actual_addresses_end_to_end(): manager = APIServerProcessManager( listen_address=f"tcp://{host}:0", sock=sock, - args="test_args", # type: ignore[arg-type] + args="test_args", num_servers=num_servers, input_addresses=placeholder_inputs, output_addresses=placeholder_outputs, @@ -424,7 +424,7 @@ def test_gather_actual_addresses_child_crash_before_report(): manager = APIServerProcessManager( listen_address=f"tcp://{host}:0", sock=sock, - args="test_args", # type: ignore[arg-type] + args="test_args", num_servers=num_servers, input_addresses=placeholder_inputs, output_addresses=placeholder_outputs, diff --git a/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py b/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py index 9e170c61fc..c03efa8f10 100644 --- a/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py +++ b/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py @@ -14,7 +14,7 @@ from vllm.outputs import RequestOutput def _make_mock_llm() -> LLM: llm = object.__new__(LLM) - llm.model_config = SimpleNamespace( # type: ignore[assignment] + llm.model_config = SimpleNamespace( runner_type="generate", enable_prompt_embeds=False ) return llm diff --git a/tests/entrypoints/openai/test_render_parity.py b/tests/entrypoints/openai/test_render_parity.py index 2bf4345091..3f4b4b2603 100644 --- a/tests/entrypoints/openai/test_render_parity.py +++ b/tests/entrypoints/openai/test_render_parity.py @@ -225,7 +225,7 @@ def online_renderer(model_config: MockModelConfig, request) -> OnlineRenderer: renderer.tokenizer = MagicMock() return OnlineRenderer( - model_config=model_config, # type: ignore[arg-type] + model_config=model_config, renderer=renderer, request_logger=None, chat_template=None, diff --git a/tests/entrypoints/pooling/embed/test_io_processor.py b/tests/entrypoints/pooling/embed/test_io_processor.py index 15bdcf1d59..3c1329693b 100644 --- a/tests/entrypoints/pooling/embed/test_io_processor.py +++ b/tests/entrypoints/pooling/embed/test_io_processor.py @@ -411,7 +411,7 @@ class TestChunkedEmbeddingProcessing: @classmethod def _make_handler(cls): handler = object.__new__(EmbedIOProcessor) - handler.model_config = cls._FakeModelConfig() # type: ignore[assignment] + handler.model_config = cls._FakeModelConfig() handler.enable_chunked_processing = True return handler diff --git a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py index 7d7b275032..5a0ff9363e 100644 --- a/tests/entrypoints/pooling/scoring/test_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_io_processor_unit.py @@ -47,7 +47,7 @@ def llm_reranker_processor() -> CrossEncoderIOProcessor: ) processor = CrossEncoderIOProcessor.__new__(CrossEncoderIOProcessor) - processor.model_config = SimpleNamespace(enable_prompt_embeds=False) # type: ignore[assignment] + processor.model_config = SimpleNamespace(enable_prompt_embeds=False) processor.tokenizer = TokenizersBackend( tokenizer_object=backend, unk_token="[UNK]", @@ -96,7 +96,7 @@ def test_token_type_ids_stay_aligned_with_a_truncated_padded_prompt(): ) processed_prompt = tok_params.apply_post_tokenization( - tokenizer, # type: ignore[arg-type] + tokenizer, prompt, ) assert "prompt_token_ids" in processed_prompt diff --git a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py index ee535ced23..e69dd6d45a 100644 --- a/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py +++ b/tests/entrypoints/pooling/scoring/test_jina_ranking_io_processor_unit.py @@ -77,10 +77,8 @@ def offline_processor_and_context(): backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit() proc = JinaRankingIOProcessor.__new__(JinaRankingIOProcessor) proc.tokenizer = TokenizersBackend(tokenizer_object=backend) - proc.model_config = SimpleNamespace( # type: ignore[assignment] - max_model_len=1024, is_encoder_decoder=False - ) - proc.renderer = SimpleNamespace( # type: ignore[assignment] + proc.model_config = SimpleNamespace(max_model_len=1024, is_encoder_decoder=False) + proc.renderer = SimpleNamespace( default_cmpl_tok_params=TokenizeParams(max_total_tokens=1024) ) ctx = OfflineScoringInputsContext( diff --git a/tests/entrypoints/pooling/test_io_processor.py b/tests/entrypoints/pooling/test_io_processor.py index d5b4909499..9d2207fdf1 100644 --- a/tests/entrypoints/pooling/test_io_processor.py +++ b/tests/entrypoints/pooling/test_io_processor.py @@ -53,8 +53,8 @@ def test_rejects_mismatched_lora_requests(processor: PoolingIOProcessor): def test_rejects_conflicting_pooling_task(processor: PoolingIOProcessor): - processor.model_config = SimpleNamespace(is_encoder_decoder=False) # type: ignore[assignment] - processor.renderer = SimpleNamespace( # type: ignore[assignment] # Tokenization defaults only. + processor.model_config = SimpleNamespace(is_encoder_decoder=False) + processor.renderer = SimpleNamespace( default_cmpl_tok_params=TokenizeParams(max_total_tokens=None) ) ctx = OfflineEncodeInputsContext( diff --git a/tests/entrypoints/serve/middleware/test_authentication_middleware.py b/tests/entrypoints/serve/middleware/test_authentication_middleware.py index a65db9a0be..27a783a451 100644 --- a/tests/entrypoints/serve/middleware/test_authentication_middleware.py +++ b/tests/entrypoints/serve/middleware/test_authentication_middleware.py @@ -87,7 +87,7 @@ def task_routes(request, monkeypatch) -> tuple[str, list[tuple[str, list[str]]]] args, app, supported_tasks=(task,), - model_config=MockModelConfig(), # type: ignore[arg-type] + model_config=MockModelConfig(), ) routes = get_all_http_routes(app) diff --git a/tests/entrypoints/serve/tokenize/test_tokenization.py b/tests/entrypoints/serve/tokenize/test_tokenization.py index bf51d8dee7..1f7e250576 100644 --- a/tests/entrypoints/serve/tokenize/test_tokenization.py +++ b/tests/entrypoints/serve/tokenize/test_tokenization.py @@ -107,7 +107,7 @@ async def test_tokenize_chat( conversation=conversation, tokenize=False, ) - tokens = tokenizer.encode(prompt, add_special_tokens=add_special) # type: ignore[arg-type] + tokens = tokenizer.encode(prompt, add_special_tokens=add_special) response = requests.post( server.url_for("tokenize"), @@ -176,7 +176,7 @@ async def test_tokenize_chat_with_tools( tools=tools, tokenize=False, ) - tokens = tokenizer.encode(prompt, add_special_tokens=add_special) # type: ignore[arg-type] + tokens = tokenizer.encode(prompt, add_special_tokens=add_special) response = requests.post( server.url_for("tokenize"), diff --git a/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py b/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py index 3dbc1e0f96..254bc28f29 100644 --- a/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py +++ b/tests/entrypoints/speech_to_text/transcription/test_qwen3_asr_sanitize_prompt.py @@ -47,7 +47,7 @@ def test_sanitize_handles_falsy_inputs() -> None: assert _sanitize_transcription_user_text("") == "" # The dataclass default for ``response_prefix`` is the empty string; # the sanitizer must accept that without exception or extra work. - assert _sanitize_transcription_user_text(None) == "" # type: ignore[arg-type] + assert _sanitize_transcription_user_text(None) == "" def test_sanitize_is_idempotent() -> None: diff --git a/tests/entrypoints/unit_tests/test_chat_utils.py b/tests/entrypoints/unit_tests/test_chat_utils.py index 0493803a54..75867f04b2 100644 --- a/tests/entrypoints/unit_tests/test_chat_utils.py +++ b/tests/entrypoints/unit_tests/test_chat_utils.py @@ -216,7 +216,7 @@ def video_url(): @pytest.fixture(scope="module") def audio_url(): audio = AudioAsset("mary_had_lamb") - return encode_audio_url(*audio.audio_and_sample_rate) # type: ignore[arg-type] + return encode_audio_url(*audio.audio_and_sample_rate) def _assert_mm_data_is_image_input( @@ -3194,5 +3194,5 @@ def test_validate_chat_template_rejects_invalid_type(): with pytest.raises( VLLMValidationError, match="not a valid chat template type" ) as exc_info: - validate_chat_template(123) # type: ignore[arg-type] + validate_chat_template(123) assert exc_info.value.parameter == "chat_template" diff --git a/tests/jit_monitor/test_hooks.py b/tests/jit_monitor/test_hooks.py index 48cd9fd0f6..452fe96995 100644 --- a/tests/jit_monitor/test_hooks.py +++ b/tests/jit_monitor/test_hooks.py @@ -149,7 +149,7 @@ def test_activate_logs_info(): def test_activate_rejects_unknown_mode(): with pytest.raises(ValueError, match="Unsupported JIT monitor mode"): - jit_monitor.activate(mode="panic") # type: ignore[arg-type] + jit_monitor.activate(mode="panic") def test_activate_without_triton(): diff --git a/tests/kernels/attention/test_attention_selector.py b/tests/kernels/attention/test_attention_selector.py index 9144928f91..24f02836e9 100644 --- a/tests/kernels/attention/test_attention_selector.py +++ b/tests/kernels/attention/test_attention_selector.py @@ -23,12 +23,12 @@ from vllm.platforms.interface import DeviceCapability if current_platform.is_cuda(): from vllm.platforms.cuda import CudaPlatform else: - CudaPlatform = None # type: ignore[assignment] # Unavailable platform import. + CudaPlatform = None if current_platform.is_rocm(): from vllm.platforms.rocm import RocmPlatform else: - RocmPlatform = None # type: ignore[misc, assignment] # Unavailable platform import. + RocmPlatform = None # type: ignore[misc] # Unavailable platform import. from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -356,14 +356,14 @@ def test_invalid_backend(): def test_auto_backend_string(auto_value: str): """Test that 'auto' string value triggers automatic backend selection.""" # Using "auto" should result in backend=None (automatic selection) - attention_config = AttentionConfig(backend=auto_value) # type: ignore[arg-type] + attention_config = AttentionConfig(backend=auto_value) assert attention_config.backend is None def test_auto_backend_selection_behavior(): """Test that 'auto' backend behaves same as None (automatic selection).""" # Create config with explicit "auto" - auto_config = AttentionConfig(backend="auto") # type: ignore[arg-type] + auto_config = AttentionConfig(backend="auto") # Create config with None (default) none_config = AttentionConfig(backend=None) diff --git a/tests/kernels/attention/test_cpu_attn.py b/tests/kernels/attention/test_cpu_attn.py index 1aa5b056c4..f79b1b15d2 100644 --- a/tests/kernels/attention/test_cpu_attn.py +++ b/tests/kernels/attention/test_cpu_attn.py @@ -836,7 +836,7 @@ def varlen_encoder_zentorch_sdpa( key, value, output, - SimpleNamespace(query_start_loc=query_start_loc, causal=False), # type: ignore[arg-type] + SimpleNamespace(query_start_loc=query_start_loc, causal=False), scale, sliding_window if sliding_window is not None else -1, alibi_slopes, diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index 62fcb0beba..e3e4897e15 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -1782,7 +1782,7 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch): num_index_heads=1, index_head_dim=HEAD_DIM, # The implementation also accepts this legacy fp8 alias. - indexer_kv_dtype="fp8_e4m3", # type: ignore[arg-type] + indexer_kv_dtype="fp8_e4m3", max_model_len=8192, ) diff --git a/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py b/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py index c3dee9b580..5c94968ddc 100644 --- a/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py +++ b/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py @@ -235,14 +235,14 @@ def test_msa_metadata_builder_prepares_cutlass_for_regular_decode( builder.num_q_heads = 64 builder.num_kv_heads = 4 builder.topk_blocks = TOPK - builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4) # type: ignore[assignment] + builder.kv_cache_spec = SimpleNamespace(num_kv_heads=4) builder.kv_cache_dtype = "fp8_e4m3" builder.decode_backend = "cutlass" - builder.msa_cutlass_plan_cache = object() # type: ignore[assignment] # Mocked plan-cache sentinel. + builder.msa_cutlass_plan_cache = object() metadata = builder.build( 0, - SimpleNamespace( # type: ignore[arg-type] + SimpleNamespace( seq_lens_cpu_upper_bound=torch.full((batch,), 257, dtype=torch.int32) ), ) @@ -377,7 +377,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay( with torch.cuda.stream(stream), override_forward_context(forward_context): capture = BreakableCUDAGraphCapture() with capture: - query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv) # type: ignore[arg-type] + query_fp8 = MiniMaxM3SparseAttention._allocate_query_fp8(attention, qkv) if query_fp8 is not None: query_fp8.zero_() run_attention(query_fp8) @@ -387,7 +387,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay( assert capture.num_eager_breaks == 1 assert observed_ptrs == [] - decode.msa_cutlass = object() # type: ignore[assignment] + decode.msa_cutlass = object() for _ in range(3): capture.replay() stream.synchronize() diff --git a/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py b/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py index 9193c03608..757fd186a6 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_causal_verify_mask.py @@ -204,7 +204,7 @@ def _run_verify_block(): ), ): assert isinstance(impl, AiterMLAImpl) - impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None) # type: ignore[arg-type] # Spy decode path does not read layer. + impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None) return metadata, captured diff --git a/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py b/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py index fe97013b25..23dd09128b 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_fp8_prefill.py @@ -121,7 +121,7 @@ def _build_prefill_metadata( builder = object.__new__(AiterMLAMetadataBuilder) builder.num_heads = num_heads - builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM) # type: ignore[assignment] + builder.mla_dims = SimpleNamespace(v_head_dim=V_HEAD_DIM) builder._init_fp8_prefill_ps_buffers( max_num_reqs=len(seq_lens), max_prefill_qlen=max_q, @@ -136,7 +136,7 @@ def _build_prefill_metadata( prefill = SimpleNamespace(query_start_loc=qo_indptr, max_query_len=max_q) metadata = SimpleNamespace(prefill=prefill, num_decodes=0) common = SimpleNamespace(query_start_loc_cpu=qo_indptr_cpu) - builder._build_fp8_prefill_ps_metadata(metadata, common) # type: ignore[arg-type] + builder._build_fp8_prefill_ps_metadata(metadata, common) return metadata, total_q, builder diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sink.py b/tests/kernels/attention/test_rocm_aiter_mla_sink.py index 354800d048..40b888c362 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sink.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sink.py @@ -170,7 +170,7 @@ def test_sparse_mla_sink_matches_ragged_reference( SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), q, kv, - metadata, # type: ignore[arg-type] + metadata, ) kv_flat = kv_ref[:, 0] references = [] @@ -244,7 +244,7 @@ def test_sparse_mla_sink_rejects_unsupported_aiter_dtypes( SimpleNamespace(_q_scale=None, _k_scale=None), torch.empty(1, 16, Q_HEAD_DIM, dtype=q_dtype), torch.empty(1, 1, Q_HEAD_DIM, dtype=kv_dtype), - metadata, # type: ignore[arg-type] + metadata, ) @@ -384,7 +384,7 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache paged_kv_indices=torch.tensor(indices, dtype=torch.int32, device="cuda"), paged_kv_indptr=lengths.cumsum(0).to(device="cuda", dtype=torch.int32), ) - impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) # type: ignore[arg-type] + impl = ROCMAiterMLASparseImpl.__new__(ROCMAiterMLASparseImpl) impl.num_heads = num_heads impl.head_size = head_dim impl.kv_lora_rank = value_dim @@ -441,9 +441,9 @@ def test_sparse_mla_backend_resolves_only_contiguous_layer_layouts(monkeypatch, ] if layout == "BLNHC": with pytest.raises(ValueError, match="does not satisfy"): - resolve_kv_cache_layout(config, supported) # type: ignore[arg-type] + resolve_kv_cache_layout(config, supported) else: - assert resolve_kv_cache_layout(config, supported).name == layout # type: ignore[arg-type] + assert resolve_kv_cache_layout(config, supported).name == layout @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) @@ -494,7 +494,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype): actual, _ = impl.forward_mqa( (q[..., :V_HEAD_DIM], q[..., V_HEAD_DIM:]), kv, - metadata, # type: ignore[arg-type] + metadata, SimpleNamespace(_q_scale=None, _k_scale=None), ) references = [ diff --git a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py index c71536a448..aabce97fac 100644 --- a/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py +++ b/tests/kernels/attention/test_rocm_aiter_mla_sparse_metadata_sync.py @@ -39,10 +39,10 @@ def _make_builder(): topk_tokens = 4 builder.device = torch.device("cpu") - builder.kv_cache_spec = SimpleNamespace(block_size=1) # type: ignore[assignment] + builder.kv_cache_spec = SimpleNamespace(block_size=1) builder.model_dtype = torch.bfloat16 builder.kv_cache_dtype = "fp8" - builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) # type: ignore[assignment] + builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64) builder.topk_tokens = topk_tokens builder.req_id_per_token_buffer = torch.zeros( max_num_batched_tokens, dtype=torch.int32, device="cpu" diff --git a/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py b/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py index b37433cd90..311ee89a8d 100644 --- a/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py +++ b/tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py @@ -227,9 +227,7 @@ def _decode_metadata(case, rows, ratio, query_lens): num_decode_tokens=len(rows), num_prefills=0, num_prefill_tokens=0, - decode=types.SimpleNamespace( # type: ignore[arg-type] - block_table=block_table, seq_lens=lens[:, None] - ), + decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]), decode_row_lens=lens, decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK, decode_native=native, @@ -280,7 +278,7 @@ def _prefill_metadata( num_decode_tokens=0, num_prefills=len(case.seq_lens), num_prefill_tokens=len(rows), - prefill=types.SimpleNamespace(chunks=chunks), # type: ignore[arg-type] + prefill=types.SimpleNamespace(chunks=chunks), prefill_plans=plans, gather_launches=plan_gather_launches(chunks, plans, gather_rows), ) diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index b6e41876b6..387e690e1b 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -955,7 +955,7 @@ def test_rocm_capture_metadata_sets_adaptive_marker(monkeypatch) -> None: ) builder = object.__new__(rocm_mod.DeepseekV4ROCMAiterMLASparseMetadataBuilder) - actual = builder.build_for_cudagraph_capture(SimpleNamespace()) # type: ignore[arg-type] # Patched base method ignores metadata. + actual = builder.build_for_cudagraph_capture(SimpleNamespace()) assert actual is metadata assert actual.for_cudagraph_capture is _on_gfx950() @@ -1563,7 +1563,7 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None: "sliding_window": window_size, }, ) - vllm_config.speculative_config = SimpleNamespace( # type: ignore[assignment] + vllm_config.speculative_config = SimpleNamespace( num_speculative_tokens=upper_query_len - 1, parallel_drafting=False, enable_adaptive_verification=True, diff --git a/tests/kernels/mamba/test_gdn_forward_core_split.py b/tests/kernels/mamba/test_gdn_forward_core_split.py index b1c3030380..8a99e223be 100644 --- a/tests/kernels/mamba/test_gdn_forward_core_split.py +++ b/tests/kernels/mamba/test_gdn_forward_core_split.py @@ -176,7 +176,7 @@ def test_forward_core_split_matches_unified( builder = GDNAttentionMetadataBuilder( # GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but # asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here. - kv_cache_spec=MambaSpec( # type: ignore[arg-type] + kv_cache_spec=MambaSpec( block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,) ), layer_names=[PREFIX], diff --git a/tests/kernels/moe/test_cutlass_moe.py b/tests/kernels/moe/test_cutlass_moe.py index 9a5189f592..1910796623 100644 --- a/tests/kernels/moe/test_cutlass_moe.py +++ b/tests/kernels/moe/test_cutlass_moe.py @@ -538,7 +538,7 @@ def run_8_bit( assert num_local_experts is not None return run_with_expert_maps( num_experts, - num_local_experts, # type: ignore[arg-type] + num_local_experts, quant_config, **kwargs, ) diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 295a92482f..f574c4d5d1 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -1724,7 +1724,7 @@ def test_rocm_mxfp4_moe_oracle( w13_conv, w2_conv, w13_scale_conv, w2_scale_conv, w13_bias_conv, w2_bias_conv = ( convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( mxfp4_backend=backend, - layer=layer, # type: ignore[arg-type] + layer=layer, w13_weight=w13_quant, w2_weight=w2_quant, w13_weight_scale=w13_scale, diff --git a/tests/kernels/quantization/test_block_fp8.py b/tests/kernels/quantization/test_block_fp8.py index 7d139644e7..fe2622693b 100644 --- a/tests/kernels/quantization/test_block_fp8.py +++ b/tests/kernels/quantization/test_block_fp8.py @@ -259,7 +259,7 @@ def test_w8a8_block_fp8_torch_scaled_mm_matmul(): stub = BlockWiseTorchFP8ScaledMMLinearKernel.__new__( BlockWiseTorchFP8ScaledMMLinearKernel ) - stub.config = types.SimpleNamespace(out_dtype=out_dtype) # type: ignore[assignment] + stub.config = types.SimpleNamespace(out_dtype=out_dtype) out = stub.apply_block_scaled_mm( A_fp8_cuda.cuda(), B_fp8.cuda(), As_cuda.cuda(), Bs.cuda() ) diff --git a/tests/kernels/quantization/test_cutlass_w4a8.py b/tests/kernels/quantization/test_cutlass_w4a8.py index 1bb63ae8be..968d5f2c74 100644 --- a/tests/kernels/quantization/test_cutlass_w4a8.py +++ b/tests/kernels/quantization/test_cutlass_w4a8.py @@ -208,7 +208,7 @@ def mm_test_helper( a=tensors.a, b_q=tensors.w_q, b_group_scales=tensors.w_g_s, - b_group_size=group_size, # type: ignore[arg-type] + b_group_size=group_size, b_channel_scales=tensors.w_ch_s, a_token_scales=tensors.w_tok_s, ) diff --git a/tests/kernels/quantization/test_marlin_tile_padding.py b/tests/kernels/quantization/test_marlin_tile_padding.py index ef5911decd..b5863008ed 100644 --- a/tests/kernels/quantization/test_marlin_tile_padding.py +++ b/tests/kernels/quantization/test_marlin_tile_padding.py @@ -525,15 +525,13 @@ def test_check_marlin_supports_layer_allow_tile_padding(): ) # Tile-misaligned but group-aligned: rejected strictly, allowed w/ padding - layer: LinearBase = _FakeLinear( # type: ignore[assignment] - 4640, 512, input_size=2048 - ) + layer: LinearBase = _FakeLinear(4640, 512, input_size=2048) assert not check_marlin_supports_layer(layer, 128) assert check_marlin_supports_layer(layer, 128, allow_tile_padding=True) assert check_marlin_supports_layer(layer, -1, allow_tile_padding=True) # A group straddling the TP shard cannot be fixed by padding - layer = _FakeLinear(4608, 4672, input_size=18688) # type: ignore[assignment] + layer = _FakeLinear(4608, 4672, input_size=18688) assert not check_marlin_supports_layer(layer, 128, allow_tile_padding=True) diff --git a/tests/kernels/quantization/test_nvfp4_emulation.py b/tests/kernels/quantization/test_nvfp4_emulation.py index 0e005760b0..3a136484ac 100644 --- a/tests/kernels/quantization/test_nvfp4_emulation.py +++ b/tests/kernels/quantization/test_nvfp4_emulation.py @@ -206,7 +206,7 @@ def test_nvfp4_emulation_support_check_rejects_bias_and_lora( in_dtype=torch.bfloat16, device="cuda", routing_method=RoutingMethodType.TopK, - **config_kwargs, # type: ignore[arg-type] + **config_kwargs, ) supported, reason = Nvfp4QuantizationEmulationTritonExperts.is_supported_config( diff --git a/tests/model_executor/layers/test_fused_shared_expert.py b/tests/model_executor/layers/test_fused_shared_expert.py index 89502fbf9d..6721a4c63b 100644 --- a/tests/model_executor/layers/test_fused_shared_expert.py +++ b/tests/model_executor/layers/test_fused_shared_expert.py @@ -413,7 +413,7 @@ def test_deepseek_v4_shared_expert_fse_uses_mtp_quantization_config_prefix( ) compatible, reason = is_shared_expert_quant_fse_compatible( - DeepseekV4Config(), # type: ignore[arg-type] + DeepseekV4Config(), "model.layers.2.ffn.experts", "model.layers.2.ffn.shared_experts", ) @@ -824,7 +824,7 @@ def test_models_fse_init( model_type, quantization_config ) vllm_config = VllmConfig() - vllm_config.model_config = SimpleNamespace( # type: ignore[assignment] + vllm_config.model_config = SimpleNamespace( hf_config=config, hf_text_config=config, dtype=torch.bfloat16, @@ -893,7 +893,7 @@ def test_models_fse_init( DeepSeekV4MTP, ) - vllm_config.speculative_config = SimpleNamespace( # type: ignore[assignment] + vllm_config.speculative_config = SimpleNamespace( draft_model_config=SimpleNamespace(hf_config=config), method="mtp", parallel_drafting=False, diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 70243e5557..77c52fb291 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -46,7 +46,7 @@ def test_sparse_attention_refreshes_batch_state_inside_eager_segment( with pytest.raises(BatchStateRefreshed): DeepseekV32Attention._sparse_indexer_and_attn( - layer, # type: ignore[arg-type] + layer, torch.empty(1, dtype=torch.long), torch.empty(1, 1), torch.empty(1, 1, 1), @@ -80,7 +80,7 @@ def make_indexer_metadata( num_decode_tokens=num_decode_tokens, num_prefills=num_prefills, num_prefill_tokens=num_prefill_tokens, - prefill=SimpleNamespace(chunks=[]) if num_prefills else None, # type: ignore[arg-type] + prefill=SimpleNamespace(chunks=[]) if num_prefills else None, ) @@ -129,7 +129,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision( slot_mapping=slot_mapping, ) if indexer_metadata.num_decodes: - indexer_metadata.decode = object() # type: ignore[assignment] + indexer_metadata.decode = object() mla_metadata = make_mla_metadata( use_dense_mha=batch_kind != "force_mqa", num_decode_tokens=mla_num_decode_tokens, @@ -307,7 +307,7 @@ def test_deepseek_v32_dispatches_selected_mha( output = torch.empty(2, 2) DeepseekV32Attention._sparse_indexer_and_attn( - layer, # type: ignore[arg-type] + layer, torch.arange(2), torch.empty(2, 2), q_nope, diff --git a/tests/model_executor/layers/test_pooler_activations.py b/tests/model_executor/layers/test_pooler_activations.py index 381d267583..697207a68b 100644 --- a/tests/model_executor/layers/test_pooler_activations.py +++ b/tests/model_executor/layers/test_pooler_activations.py @@ -224,11 +224,11 @@ class TestResolveClassifierActFn: model_config = SimpleNamespace( hf_config=SimpleNamespace(num_labels=3, problem_type="") ) - result = resolve_classifier_act_fn(model_config, act_fn=None) # type: ignore[arg-type] + result = resolve_classifier_act_fn(model_config, act_fn=None) assert isinstance(result, PoolerClassify) assert result.num_labels == 3 def test_passes_through_provided_act_fn(self): custom = PoolerIdentity() - result = resolve_classifier_act_fn(None, act_fn=custom) # type: ignore[arg-type] + result = resolve_classifier_act_fn(None, act_fn=custom) assert result is custom diff --git a/tests/model_executor/layers/test_pooler_methods.py b/tests/model_executor/layers/test_pooler_methods.py index bfe2cf0aed..75bbed4f37 100644 --- a/tests/model_executor/layers/test_pooler_methods.py +++ b/tests/model_executor/layers/test_pooler_methods.py @@ -320,7 +320,7 @@ def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type): ), ) classifier = torch.nn.Linear(4, 3) - with set_current_vllm_config(config): # type: ignore[arg-type] + with set_current_vllm_config(config): pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier) token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]] diff --git a/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py b/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py index 7047ba1c17..070c832967 100644 --- a/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py +++ b/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py @@ -36,7 +36,7 @@ try: import tensorizer from tensorizer import EncryptionParams except ImportError: - tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment] + tensorizer = PlaceholderModule("tensorizer") EncryptionParams = tensorizer.placeholder_attr("EncryptionParams") diff --git a/tests/model_executor/model_loader/test_weight_cache.py b/tests/model_executor/model_loader/test_weight_cache.py index 106b9690ac..731c732625 100644 --- a/tests/model_executor/model_loader/test_weight_cache.py +++ b/tests/model_executor/model_loader/test_weight_cache.py @@ -368,4 +368,4 @@ def test_ipc_loader_copy_mode_reports_no_external_weight_memory(): load_format="ipc_cache", model_loader_extra_config={"mode": "copy"} ) loader = IpcModelLoader(copy_mode) - assert loader.get_external_weight_memory(None) == 0 # type: ignore[arg-type] + assert loader.get_external_weight_memory(None) == 0 diff --git a/tests/models/language/generation/test_mistral.py b/tests/models/language/generation/test_mistral.py index fc774bc6e8..ba908c23c5 100644 --- a/tests/models/language/generation/test_mistral.py +++ b/tests/models/language/generation/test_mistral.py @@ -259,7 +259,7 @@ def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None: model_output = outputs[0].outputs[0].text.strip() assert model_output.startswith(tool_parser.bot_token), model_output - parsed_message = tool_parser.extract_tool_calls(model_output, None) # type: ignore[arg-type] + parsed_message = tool_parser.extract_tool_calls(model_output, None) assert parsed_message.tools_called @@ -302,7 +302,7 @@ def test_mistral_function_call_nested_json(): model_output = f"{parser.bot_token}get_current_weather{json.dumps(args_dict)}" - parsed = parser.extract_tool_calls(model_output, None) # type: ignore[arg-type] + parsed = parser.extract_tool_calls(model_output, None) # Assertions: the tool call is detected and the full nested JSON is parsed # without truncation. @@ -335,7 +335,7 @@ def test_mistral_function_call_nested_json(): ] ) - parsed = parser.extract_tool_calls(model_output, None) # type: ignore[arg-type] + parsed = parser.extract_tool_calls(model_output, None) # Assertions: the tool call is detected and the full nested JSON is parsed # without truncation. diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 1a7931082a..bc472c400f 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -1193,7 +1193,7 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner": output.sequences = output.sequences[:, prompt_len:] return output - hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign, assignment] + hf_model.get_inputs = patched_get_inputs # type: ignore[method-assign] hf_model.model.generate = patched_generate # type: ignore[method-assign] return hf_model diff --git a/tests/models/multimodal/pooling/test_clip.py b/tests/models/multimodal/pooling/test_clip.py index 9c500fd6a3..72843105ba 100644 --- a/tests/models/multimodal/pooling/test_clip.py +++ b/tests/models/multimodal/pooling/test_clip.py @@ -132,7 +132,7 @@ def test_models( _run_test( hf_runner, vllm_runner, - input_cases, # type: ignore[arg-type] + input_cases, model, dtype=dtype, ) diff --git a/tests/models/multimodal/pooling/test_llama_nemotron_vl.py b/tests/models/multimodal/pooling/test_llama_nemotron_vl.py index 2a0ba96722..e9e58d8635 100644 --- a/tests/models/multimodal/pooling/test_llama_nemotron_vl.py +++ b/tests/models/multimodal/pooling/test_llama_nemotron_vl.py @@ -190,7 +190,7 @@ def test_models( _run_test( hf_runner, vllm_runner, - input_cases, # type: ignore[arg-type] + input_cases, model, dtype=dtype, ) diff --git a/tests/models/multimodal/pooling/test_siglip.py b/tests/models/multimodal/pooling/test_siglip.py index df9bb61353..a3afe8491e 100644 --- a/tests/models/multimodal/pooling/test_siglip.py +++ b/tests/models/multimodal/pooling/test_siglip.py @@ -149,7 +149,7 @@ def test_models( _run_test( hf_runner, vllm_runner, - input_cases, # type: ignore[arg-type] + input_cases, model, dtype=dtype, ) diff --git a/tests/models/multimodal/processing/test_qwen2_vl.py b/tests/models/multimodal/processing/test_qwen2_vl.py index b957ebf2c8..133880b081 100644 --- a/tests/models/multimodal/processing/test_qwen2_vl.py +++ b/tests/models/multimodal/processing/test_qwen2_vl.py @@ -106,7 +106,7 @@ def test_merge_and_resolve_mm_processor_kwargs_preserves_request_alias_precedenc mm_device_do_normalize=False, ) model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) - ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + ctx = InputProcessingContext(model_config, tokenizer=None) info = Qwen2VLProcessingInfo(ctx) monkeypatch.setattr( info, @@ -236,7 +236,7 @@ def test_get_vision_info_reads_merged_modality_scope( do_resize=False, image_processor=image_processor, mm_kwargs={}, - modality=modality, # type: ignore[arg-type] + modality=modality, ) assert seen == [merged[scope]] diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index fa5e14eddd..2064f01e0c 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -260,7 +260,7 @@ def test_quantization( model, model_impl="auto", enforce_eager=True, - **quantization_kwargs, # type: ignore[arg-type] + **quantization_kwargs, ) as vllm_model: vllm_outputs = vllm_model.generate_greedy_logprobs( example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs @@ -270,7 +270,7 @@ def test_quantization( model, model_impl="transformers", enforce_eager=True, - **quantization_kwargs, # type: ignore[arg-type] + **quantization_kwargs, ) as vllm_model: model_config = vllm_model.llm.llm_engine.model_config assert model_config.using_transformers_backend() diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index 9fbe2f5792..a2fedb1747 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -385,7 +385,7 @@ def test_pynvvideocodec_unrelated_error_propagates( PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec( None, b"video", - None, # type: ignore[arg-type] # Fails before target metadata is used. + None, ) assert exc_info.value is original_error diff --git a/tests/multimodal/test_audio.py b/tests/multimodal/test_audio.py index 379fc3b0ba..24d925aa02 100644 --- a/tests/multimodal/test_audio.py +++ b/tests/multimodal/test_audio.py @@ -179,7 +179,7 @@ def test_audio_resampler_invalid_method(dummy_audio): # Validated eagerly so a bad method fails at construction, not on the # first audio request. with pytest.raises(ValueError, match="Invalid resampling method"): - AudioResampler(target_sr=22050, method="invalid") # type: ignore[arg-type] + AudioResampler(target_sr=22050, method="invalid") def test_audio_resampler_no_target_sr(dummy_audio): diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index da6b0369b4..2899d15ad0 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -276,14 +276,14 @@ def test_oversized_item_is_served_uncached(): item = MultiModalKwargsItem.dummy(nbytes=4096) small = MultiModalKwargsItem.dummy(nbytes=64) - p0_only = MultiModalProcessorOnlyCache(model_config) # type: ignore[arg-type] + p0_only = MultiModalProcessorOnlyCache(model_config) assert p0_only.get_and_update_item((item, []), "big")[0] is item assert not p0_only.is_cached_item("big") assert p0_only.get_and_update_item((small, []), "small")[0] is small assert p0_only.is_cached_item("small") - p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type] - p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type] + p0 = LruKeyReplicatedSenderCache(model_config) + p1 = LruKeyReplicatedReceiverCache(model_config) assert item is not None assert p0.get_and_update_item((item, []), "big")[0] is item assert not p0.is_cached_item("big") @@ -304,8 +304,8 @@ def test_mm_cache_miss_raises_and_recovers(): model or network. """ model_config = _StubModelConfig(mm_processor_cache_gb=1) - p0 = LruKeyReplicatedSenderCache(model_config) # type: ignore[arg-type] - p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type] + p0 = LruKeyReplicatedSenderCache(model_config) + p1 = LruKeyReplicatedReceiverCache(model_config) mm_hash = "image_A" item = MultiModalKwargsItem.dummy(nbytes=64) @@ -348,7 +348,7 @@ def test_receiver_cache_replaces_stale_item_when_payload_resent(): and kills EngineCore. Prefer the fresh payload and keep it for later hits. """ model_config = _StubModelConfig(mm_processor_cache_gb=1) - p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type] + p1 = LruKeyReplicatedReceiverCache(model_config) small = MultiModalKwargsItem.dummy(nbytes=64) large = MultiModalKwargsItem.dummy(nbytes=256) mm_hash = "shared-id" @@ -361,7 +361,7 @@ def test_receiver_cache_replaces_stale_item_when_payload_resent(): def test_receiver_cache_features_keep_resent_payload(): """EngineCore updates features through get_and_update_features.""" model_config = _StubModelConfig(mm_processor_cache_gb=1) - p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type] + p1 = LruKeyReplicatedReceiverCache(model_config) small = MultiModalKwargsItem.dummy(nbytes=64) large = MultiModalKwargsItem.dummy(nbytes=256) mm_hash = "shared-id" @@ -394,7 +394,7 @@ def test_mm_cache_miss_batches_all_drifted_hashes(): non-drifted items in the same request are still ingested. """ model_config = _StubModelConfig(mm_processor_cache_gb=1) - p1 = LruKeyReplicatedReceiverCache(model_config) # type: ignore[arg-type] + p1 = LruKeyReplicatedReceiverCache(model_config) item = MultiModalKwargsItem.dummy(nbytes=64) def _feature( @@ -442,8 +442,8 @@ def test_shm_receiver_handles_prefix_covered_items(monkeypatch): parallel_config=SimpleNamespace(world_size=1), model_config=_StubModelConfig(mm_processor_cache_gb=4 * MiB_bytes / GiB_bytes), ) - p0 = ShmObjectStoreSenderCache(vllm_config) # type: ignore[arg-type] - p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) # type: ignore[arg-type] + p0 = ShmObjectStoreSenderCache(vllm_config) + p1 = ShmObjectStoreReceiverCache(vllm_config, mp.Lock()) def _feature( mm_hash: str, data: MultiModalKwargsItem | None @@ -1082,9 +1082,9 @@ async def test_release_kv_cache_resends_mm_payload(use_async, release_error): async def call_release(): if use_async: - await AsyncLLM.release_kv_cache_memory(engine) # type: ignore[arg-type] + await AsyncLLM.release_kv_cache_memory(engine) else: - LLMEngine.release_kv_cache_memory(engine) # type: ignore[arg-type] + LLMEngine.release_kv_cache_memory(engine) if release_error: with pytest.raises(RuntimeError, match=release_error): diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 63c6080b2d..69d91fd04e 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1281,7 +1281,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities( processor: SimpleNamespace, expected: dict[str, set[str]], ) -> None: - info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type] + info = BaseProcessingInfo(SimpleNamespace()) info.__dict__["supported_mm_limits"] = supported_mm_limits monkeypatch.setattr(info, "get_hf_processor", lambda **_: processor) @@ -1291,7 +1291,7 @@ def test_get_supported_mm_processor_kwargs_uses_supported_modalities( def test_supported_mm_processor_kwargs_is_cached( monkeypatch: pytest.MonkeyPatch, ) -> None: - info = BaseProcessingInfo(SimpleNamespace()) # type: ignore[arg-type] + info = BaseProcessingInfo(SimpleNamespace()) expected = {"images_kwargs": {"size"}} calls = 0 @@ -1442,7 +1442,7 @@ def test_get_merged_mm_kwargs_treats_empty_scopes_as_absent_before_routing( mm_device_do_normalize=False, ) model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) - ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + ctx = InputProcessingContext(model_config, tokenizer=None) merged = ctx.get_merged_mm_kwargs( inference_kwargs, @@ -1482,7 +1482,7 @@ def test_get_merged_mm_kwargs_merges_before_routing(): mm_device_do_normalize=False, ) model_config = SimpleNamespace(get_multimodal_config=lambda: mm_config) - ctx = InputProcessingContext(model_config, tokenizer=None) # type: ignore[arg-type] + ctx = InputProcessingContext(model_config, tokenizer=None) merged = ctx.get_merged_mm_kwargs( { diff --git a/tests/multimodal/test_registry.py b/tests/multimodal/test_registry.py index 1c4e69ba3c..f395d36933 100644 --- a/tests/multimodal/test_registry.py +++ b/tests/multimodal/test_registry.py @@ -24,4 +24,4 @@ def test_create_processor_error_uses_served_model_name(): ValueError, match="friendly-model-name is not a multimodal model", ): - MULTIMODAL_REGISTRY.create_processor(model_config) # type: ignore[arg-type] + MULTIMODAL_REGISTRY.create_processor(model_config) diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 3c0b9db209..2bb77b2ed7 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -493,7 +493,7 @@ def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot(): old_decoder = FakeDecoder() slot = PyNvVideoCodecDecoderSlot(FakeStream()) - slot.decoder = old_decoder # type: ignore[assignment] # Fake retained GPU decoder. + slot.decoder = old_decoder slot.source_path = "valid.mp4" class FakeNvc: diff --git a/tests/parser/cohere/test_structural_tags.py b/tests/parser/cohere/test_structural_tags.py index 5083215d3b..893cfc80f7 100644 --- a/tests/parser/cohere/test_structural_tags.py +++ b/tests/parser/cohere/test_structural_tags.py @@ -347,7 +347,7 @@ class TestAdjustRequestFoldFromStructuredOutputs: id="chat_completion_request", ), pytest.param( - lambda: StructuredOutputsParams(json=[1, 2, 3]), # type: ignore[arg-type] + lambda: StructuredOutputsParams(json=[1, 2, 3]), id="structured_outputs_params", ), ], diff --git a/tests/plugins_tests/bitsandbytes/test_bitsandbytes.py b/tests/plugins_tests/bitsandbytes/test_bitsandbytes.py index 1e96598d4d..f5845efb04 100644 --- a/tests/plugins_tests/bitsandbytes/test_bitsandbytes.py +++ b/tests/plugins_tests/bitsandbytes/test_bitsandbytes.py @@ -271,7 +271,7 @@ def test_bitsandbytes_passes_revision_by_name(): return_value=["/folder/model.safetensors"], ), ): - bnb.BitsAndBytesModelLoader._prepare_weights(fake_self, "org/model", "myrev") # type: ignore[arg-type] + bnb.BitsAndBytesModelLoader._prepare_weights(fake_self, "org/model", "myrev") mock_idx.assert_called_once() assert mock_idx.call_args.kwargs.get("revision") == "myrev" diff --git a/tests/plugins_tests/bitsandbytes/test_transformers.py b/tests/plugins_tests/bitsandbytes/test_transformers.py index 398d869ade..1d50f69a07 100644 --- a/tests/plugins_tests/bitsandbytes/test_transformers.py +++ b/tests/plugins_tests/bitsandbytes/test_transformers.py @@ -43,7 +43,7 @@ def test_transformers_backend_quantization( model, model_impl="auto", enforce_eager=True, - **quantization_kwargs, # type: ignore[arg-type] + **quantization_kwargs, ) as vllm_model: vllm_outputs = vllm_model.generate_greedy_logprobs( example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs @@ -53,7 +53,7 @@ def test_transformers_backend_quantization( model, model_impl="transformers", enforce_eager=True, - **quantization_kwargs, # type: ignore[arg-type] + **quantization_kwargs, ) as vllm_model: model_config = vllm_model.llm.llm_engine.model_config assert model_config.using_transformers_backend() diff --git a/tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py b/tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py index 5ebd7d36e8..56930a7d0b 100644 --- a/tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py +++ b/tests/plugins_tests/gguf/test_gguf_plugin_multimodal.py @@ -92,7 +92,7 @@ def run_multimodal_gguf_test( num_logprobs: int, ): # Load images at runtime (inside subprocess) to avoid pickle issues - images = [ImageAsset(name).pil_image for name in model.image_names] # type: ignore[arg-type] + images = [ImageAsset(name).pil_image for name in model.image_names] size_factors = [0.25, 0.5, 1.0] inputs_per_image = [ ( diff --git a/tests/plugins_tests/test_endpoint_plugins.py b/tests/plugins_tests/test_endpoint_plugins.py index 51d96643d9..2d92153900 100644 --- a/tests/plugins_tests/test_endpoint_plugins.py +++ b/tests/plugins_tests/test_endpoint_plugins.py @@ -172,7 +172,7 @@ async def test_init_state_is_noop_without_phase_a(monkeypatch: pytest.MonkeyPatc monkeypatch.setenv("VLLM_PLUGINS", "dummy_admin_endpoint_plugin") state = State() - await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args()) # type: ignore[arg-type] + await init_endpoint_plugins_state(_FakeEngineClient(), state, _build_args()) assert not hasattr(state, "dummy_engine_client") @@ -223,7 +223,7 @@ async def test_endpoint_plugin_end_to_end(monkeypatch: pytest.MonkeyPatch): ) fake_engine_client = _FakeEngineClient(rpc_result=["cfg-a", "cfg-b"]) - await init_endpoint_plugins_state(fake_engine_client, app.state, args) # type: ignore[arg-type] + await init_endpoint_plugins_state(fake_engine_client, app.state, args) assert app.state.dummy_engine_client is fake_engine_client diff --git a/tests/plugins_tests/test_oot_registration_offline.py b/tests/plugins_tests/test_oot_registration_offline.py index 4fac55f82d..ac5361c66f 100644 --- a/tests/plugins_tests/test_oot_registration_offline.py +++ b/tests/plugins_tests/test_oot_registration_offline.py @@ -92,7 +92,7 @@ def test_oot_registration_multimodal( ) first_token = llm.get_tokenizer().decode(0) - outputs = llm.generate(prompts, sampling_params) # type: ignore[arg-type] + outputs = llm.generate(prompts, sampling_params) for output in outputs: generated_text = output.outputs[0].text diff --git a/tests/plugins_tests/test_stats_logger_plugins.py b/tests/plugins_tests/test_stats_logger_plugins.py index 518015f76a..2f4d0ec694 100644 --- a/tests/plugins_tests/test_stats_logger_plugins.py +++ b/tests/plugins_tests/test_stats_logger_plugins.py @@ -24,7 +24,7 @@ def test_stat_logger_plugin_is_discovered(monkeypatch: pytest.MonkeyPatch): vllm_config = VllmConfig() instance = factories[0](vllm_config) # The `is` check above narrows DummyStatLogger to a Callable - assert isinstance(instance, DummyStatLogger) # type: ignore[arg-type] + assert isinstance(instance, DummyStatLogger) def test_no_plugins_loaded_if_env_empty(monkeypatch: pytest.MonkeyPatch): diff --git a/tests/quantization/test_auto_gptq.py b/tests/quantization/test_auto_gptq.py index fbf405210a..6e7791654c 100644 --- a/tests/quantization/test_auto_gptq.py +++ b/tests/quantization/test_auto_gptq.py @@ -123,8 +123,8 @@ def test_auto_gptq_moe_creates_zero_initialized_expert_biases(): method = object.__new__(AutoGPTQMoEMethod) method.quant_config = AutoGPTQConfig(4, 128, False, True, False, {}, {}) method.input_dtype = None - method.experts_cls = None # type: ignore[assignment] # Weight creation does not select experts. - method.moe = SimpleNamespace(w13_num_shards=2) # type: ignore[assignment] + method.experts_cls = None + method.moe = SimpleNamespace(w13_num_shards=2) layer = torch.nn.Module() method.create_weights( diff --git a/tests/quantization/test_auto_round.py b/tests/quantization/test_auto_round.py index dc09805610..eb8273c6d6 100644 --- a/tests/quantization/test_auto_round.py +++ b/tests/quantization/test_auto_round.py @@ -423,7 +423,7 @@ def test_inc_get_quant_method_unquantized_routed_experts_with_model_prefix( config = make_config(extra_config={"model.moe_layer": {"bits": 16}}) layer = object.__new__(RoutedExperts) - layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub. + layer.moe_config = None method = config.get_quant_method(layer, "moe_layer") @@ -730,7 +730,7 @@ def test_wna16_cuda_low_bit_moe_routes_to_humming(monkeypatch, bits) -> None: ) layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() # type: ignore[assignment] + layer.moe_config = DummyMoeConfig() layer_config = make_layer_config(bits=bits) method = INCWna16Scheme().get_moe_method( make_config(), layer, "model.layers.0.mlp", layer_config @@ -861,7 +861,7 @@ def test_qwen3_30b_a3b_w4a16_autoround_routes_to_gptq_moe( config = make_qwen3_autoround_config("qwen3_30b_a3b_w4a16") layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() # type: ignore[assignment] + layer.moe_config = DummyMoeConfig() method = config.get_quant_method(layer, "model.layers.0.mlp") @@ -890,7 +890,7 @@ def test_qwen3_30b_a3b_mxfp4_autoround_routes_to_mxfp4_moe( config = make_qwen3_autoround_config("qwen3_30b_a3b_mxfp4") layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() # type: ignore[assignment] + layer.moe_config = DummyMoeConfig() ignored_method = config.get_quant_method( object.__new__(LinearBase), "model.layers.0.self_attn.q_proj" @@ -1186,7 +1186,7 @@ def test_wna16_xpu_moe_routes_to_gptq_moe(monkeypatch) -> None: ) layer = object.__new__(RoutedExperts) - layer.moe_config = DummyMoeConfig() # type: ignore[assignment] + layer.moe_config = DummyMoeConfig() method = INCWna16Scheme().get_moe_method( make_config(), layer, @@ -1669,7 +1669,7 @@ def test_inc_get_quant_method_unquantized_moe_returns_unquantized( when extra_config has bits >= 16.""" config = make_config(extra_config={"layer": {"bits": 16}}) layer = object.__new__(RoutedExperts) - layer.moe_config = None # type: ignore[assignment] # No MoE config used by this stub. + layer.moe_config = None class DummyUnquantizedFusedMoEMethod: def __init__(self, moe_config) -> None: diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index f880064e37..1cb53f2239 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -220,7 +220,7 @@ def test_deepseek_v41_vl_mapper_routes_linear_scales( quant_config=SimpleNamespace(weight_block_size=weight_block_size) ) resolved = model_module._linear_scale_param_name( - vllm_config, # type: ignore[arg-type] + vllm_config, expert_dtype, ) assert resolved == scale_name @@ -798,7 +798,7 @@ def test_kv_cache_scale_sync_to_host_copies(): set_default_quant_scales(layer, register_buffer=True) layer.kv_cache_dtype = "fp8" - method = BaseKVCacheMethod(quant_config=None) # type: ignore[arg-type] # Scale synchronization does not use quant_config. + method = BaseKVCacheMethod(quant_config=None) method.create_weights(layer) # 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale. checkpoint_scale = torch.tensor(0.3, dtype=torch.float32) diff --git a/tests/quantization/test_moe_wna16.py b/tests/quantization/test_moe_wna16.py index abbcb86916..c295a195a5 100644 --- a/tests/quantization/test_moe_wna16.py +++ b/tests/quantization/test_moe_wna16.py @@ -218,9 +218,9 @@ def test_gptq_nfirst_weights_pass_through_for_triton(): def test_moe_wna16_setup_forwards_selected_backend(monkeypatch): method = object.__new__(MoeWNA16Method) - method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel. + method.experts_cls = object method.wna16_backend = WNA16MoEBackend.HUMMING - method.moe = object() # type: ignore[assignment] # Opaque mocked-config sentinel. + method.moe = object() quant_config = object() monkeypatch.setattr( method, "get_fused_moe_quant_config", lambda layer: quant_config @@ -235,7 +235,7 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch): monkeypatch.setattr(moe_wna16, "make_wna16_moe_kernel", fake_make_wna16_moe_kernel) - method._setup_kernel(layer) # type: ignore[arg-type] # Routing-table stub. + method._setup_kernel(layer) assert method.moe_kernel is kernel assert captured["backend"] == WNA16MoEBackend.HUMMING @@ -280,7 +280,7 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch): ), ) - assert method.get_fused_moe_quant_config(layer) is quant_config # type: ignore[arg-type] # Identity sentinel. + assert method.get_fused_moe_quant_config(layer) is quant_config @pytest.mark.skipif( diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index c8a75bf1f0..b4dffbb375 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -210,9 +210,9 @@ def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( monkeypatch, ) -> None: method = object.__new__(Nvfp4OnlineMoEMethod) - method.moe = SimpleNamespace(is_act_and_mul=True) # type: ignore[assignment] - method.nvfp4_backend = object() # type: ignore[assignment] # Opaque mocked-backend sentinel. - method.experts_cls = object # type: ignore[assignment] # Opaque mocked-expert sentinel. + method.moe = SimpleNamespace(is_act_and_mul=True) + method.nvfp4_backend = object() + method.experts_cls = object method.moe_quant_config = None method.moe_kernel = None @@ -1432,7 +1432,7 @@ def test_online_int8_moe_w2_scale_matches_unsharded(monkeypatch) -> None: layer.w2_weight = torch.nn.Parameter(w2_in, requires_grad=False) layer.num_experts = layer.local_num_experts = w13.shape[0] method = SimpleNamespace(moe=SimpleNamespace(tp_size=moe_tp_size)) - Int8OnlineMoEMethod._quantize_weights(method, layer) # type: ignore[arg-type] + Int8OnlineMoEMethod._quantize_weights(method, layer) return layer.w2_weight, layer.w2_scale full_weight, full_scale = quantize(w2, 1) diff --git a/tests/quantization/test_quark.py b/tests/quantization/test_quark.py index 097f03bb00..9ec0bb78d2 100644 --- a/tests/quantization/test_quark.py +++ b/tests/quantization/test_quark.py @@ -2314,7 +2314,7 @@ class TestQuarkInt4Format: None, quant_config.quant_config["global_quant_config"]["weight"], quant_config.pack_method, - moe_config, # type: ignore[arg-type] + moe_config, ) layer = _FakeLayer(moe_config) @@ -2326,7 +2326,7 @@ class TestQuarkInt4Format: requires_grad=False, ) loader = method.get_weight_loader( - layer, # type: ignore[arg-type] + layer, weight_loader=None, ) loader( diff --git a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py index 48d7f13aef..9effb4f7fc 100644 --- a/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py +++ b/tests/quantization/test_trtllm_nvfp4_hidden_dim_padding.py @@ -34,7 +34,7 @@ def test_shared_nvfp4_input_scales_have_writable_storage(monkeypatch): outputs = prepare_nvfp4_moe_layer_for_fi_or_cutlass( backend=NvFp4MoeBackend.FLASHINFER_CUTLASS, - layer=layer, # type: ignore[arg-type] + layer=layer, w13=w13, w13_scale=w13_scale, w13_scale_2=weight_scale, diff --git a/tests/quantization/test_turboquant.py b/tests/quantization/test_turboquant.py index e60f763c2c..9f6beb3eb8 100644 --- a/tests/quantization/test_turboquant.py +++ b/tests/quantization/test_turboquant.py @@ -300,7 +300,7 @@ class TestTurboQuantKVCacheSpec: # The layer builds an unpacked spec; the worker's spec-collection # loop applies TQ slot packing via the backend's customize_spec hook. - spec = Attention.get_kv_cache_spec(layer, vllm_config) # type: ignore[arg-type] + spec = Attention.get_kv_cache_spec(layer, vllm_config) assert isinstance(spec, FullAttentionSpec) assert spec.kv_quant_mode.is_turboquant assert spec.state_content_bytes is None diff --git a/tests/reasoning/test_base_thinking_reasoning_parser.py b/tests/reasoning/test_base_thinking_reasoning_parser.py index 14792b210b..fad9949881 100644 --- a/tests/reasoning/test_base_thinking_reasoning_parser.py +++ b/tests/reasoning/test_base_thinking_reasoning_parser.py @@ -68,7 +68,7 @@ class TestBaseThinkingReasoningParserInit: def test_initialization_with_missing_tokenizer(self): """Test that initialization fails without tokenizer.""" with pytest.raises(ValueError, match="model tokenizer must be passed"): - TestThinkingReasoningParser(None) # type: ignore[arg-type] + TestThinkingReasoningParser(None) def test_initialization_with_missing_tokens(self, test_tokenizer): """Test that initialization fails when tokens are not in vocabulary.""" diff --git a/tests/renderers/test_cohere.py b/tests/renderers/test_cohere.py index ae2dfe6c7a..4f6e3685e7 100644 --- a/tests/renderers/test_cohere.py +++ b/tests/renderers/test_cohere.py @@ -81,7 +81,7 @@ class TestRoleToMelody: # reject non-strings — any exception type is acceptable as long # as we don't silently produce a malformed prompt. with pytest.raises((AttributeError, TypeError, ValueError)): - _role_to_melody(None) # type: ignore[arg-type] + _role_to_melody(None) # ====================================================================== @@ -135,7 +135,7 @@ class TestNormalizeToolCall: def test_invalid_type_rejected(self): with pytest.raises(TypeError, match="Unexpected tool_call value"): - _normalize_tool_call(42) # type: ignore[arg-type] + _normalize_tool_call(42) # ====================================================================== @@ -249,7 +249,7 @@ class TestDocumentToMelody: def test_invalid_type_rejected(self): with pytest.raises(TypeError, match="Unsupported document type"): - _document_to_melody(42) # type: ignore[arg-type] + _document_to_melody(42) # ====================================================================== @@ -302,7 +302,7 @@ class TestToolToMelody: def test_invalid_type_rejected(self): with pytest.raises(TypeError, match="Unsupported tool type"): - _tool_to_melody(42) # type: ignore[arg-type] + _tool_to_melody(42) # ====================================================================== @@ -418,7 +418,7 @@ class TestConversationToMelody: {"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, ] - out = _conversation_to_melody_messages(conv) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv) assert out == [ { "role": "user", @@ -443,7 +443,7 @@ class TestConversationToMelody: "reasoning": "thoughts", } ] - out = _conversation_to_melody_messages(conv) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv) assert out[0]["content"] == [ {"type": MelodyContentType.THINKING, "thinking": "thoughts"}, {"type": MelodyContentType.TEXT, "text": "answer"}, @@ -457,7 +457,7 @@ class TestConversationToMelody: "reasoning_content": "thoughts", } ] - out = _conversation_to_melody_messages(conv) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv) assert out[0]["content"][0] == { "type": MelodyContentType.THINKING, "thinking": "thoughts", @@ -474,7 +474,7 @@ class TestConversationToMelody: "reasoning": "should be ignored", } ] - out = _conversation_to_melody_messages(conv) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv) assert out[0]["content"] == [{"type": MelodyContentType.TEXT, "text": "hi"}] def test_tool_calls_normalized(self): @@ -491,7 +491,7 @@ class TestConversationToMelody: ], } ] - out = _conversation_to_melody_messages(conv) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv) assert out[0]["tool_calls"] == [ {"id": "c1", "name": "f", "parameters": '{"a":1}'} ] @@ -504,7 +504,7 @@ class TestConversationToMelody: "tool_call_id": "c1", } ] - out = _conversation_to_melody_messages(conv) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv) assert out[0]["tool_call_id"] == "c1" def test_messages_citations_attached_by_index(self): @@ -531,20 +531,20 @@ class TestConversationToMelody: } ] } - out = _conversation_to_melody_messages(conv, citations) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv, citations) assert "citations" not in out[0] assert out[1]["citations"] == citations[1] def test_messages_citations_none_is_a_no_op(self): conv = [{"role": "assistant", "content": "a"}] - out = _conversation_to_melody_messages(conv, None) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv, None) assert "citations" not in out[0] def test_messages_citations_missing_index_is_a_no_op(self): # A ``messages_citations`` dict whose key doesn't hit any # message must not attach anything (and must not raise). conv = [{"role": "assistant", "content": "a"}] - out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type] + out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) assert "citations" not in out[0] def test_tool_message_v2_content_replaces_default_content(self): @@ -575,7 +575,7 @@ class TestConversationToMelody: ] } out = _conversation_to_melody_messages( - conv, # type: ignore[arg-type] + conv, tool_message_v2_content=v2_content, ) # User message (no forwarded entry) still goes through the @@ -597,7 +597,7 @@ class TestConversationToMelody: ] } out = _conversation_to_melody_messages( - conv, # type: ignore[arg-type] + conv, tool_message_v2_content=v2_content, ) assert out[0]["content"] == [ @@ -607,7 +607,7 @@ class TestConversationToMelody: def test_tool_message_v2_content_none_falls_back_to_default(self): conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}] out = _conversation_to_melody_messages( - conv, # type: ignore[arg-type] + conv, tool_message_v2_content=None, ) assert out[0]["content"] == [{"type": "text", "text": "plain text"}] @@ -626,7 +626,7 @@ class TestBuildRenderConfig: # Bare kwargs -> cmd4 (the current Command A+ prompt format). # Mirrors ``_DEFAULT_FORMAT`` in ``vllm/renderers/cohere.py`` and # the ``--cohere-format`` CLI default. - fmt, cfg = _build_render_config(self._conv(), {}) # type: ignore[arg-type] + fmt, cfg = _build_render_config(self._conv(), {}) assert fmt == "cmd4" assert cfg["use_jinja"] is True assert isinstance(cfg["messages"], list) @@ -634,12 +634,12 @@ class TestBuildRenderConfig: assert "additional_template_fields" not in cfg def test_explicit_cmd3(self): - fmt, cfg = _build_render_config(self._conv(), {"cohere_format": "cmd3"}) # type: ignore[arg-type] + fmt, cfg = _build_render_config(self._conv(), {"cohere_format": "cmd3"}) assert fmt == "cmd3" def test_invalid_format_raises(self): with pytest.raises(ValueError, match="Invalid cohere_format"): - _build_render_config(self._conv(), {"cohere_format": "cmd5"}) # type: ignore[arg-type] + _build_render_config(self._conv(), {"cohere_format": "cmd5"}) def test_documents_converted(self): _, cfg = _build_render_config( @@ -650,7 +650,7 @@ class TestBuildRenderConfig: {"id": "d1", "data": {"text": "wrapped"}}, ] }, - ) # type: ignore[arg-type] + ) assert cfg["documents"] == [ {"text": "doc text"}, {"id": "d1", "text": "wrapped"}, @@ -665,7 +665,7 @@ class TestBuildRenderConfig: {"type": "function", "function": {"name": "preferred"}} ], }, - ) # type: ignore[arg-type] + ) names = [t["name"] for t in cfg["available_tools"]] assert names == ["preferred"] @@ -673,30 +673,30 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {"tools": [{"type": "function", "function": {"name": "from_tools"}}]}, - ) # type: ignore[arg-type] + ) assert [t["name"] for t in cfg["available_tools"]] == ["from_tools"] @pytest.mark.parametrize("value", ["enabled", "disabled"]) def test_reasoning_type_direct(self, value): - _, cfg = _build_render_config(self._conv(), {"reasoning_type": value}) # type: ignore[arg-type] + _, cfg = _build_render_config(self._conv(), {"reasoning_type": value}) assert cfg["reasoning_type"] == value def test_thinking_dict_shorthand_resolves_reasoning_type(self): - _, cfg = _build_render_config(self._conv(), {"thinking": {"type": "enabled"}}) # type: ignore[arg-type] + _, cfg = _build_render_config(self._conv(), {"thinking": {"type": "enabled"}}) assert cfg["reasoning_type"] == "enabled" def test_thinking_shorthand_ignores_unknown_type(self): - _, cfg = _build_render_config(self._conv(), {"thinking": {"type": "auto"}}) # type: ignore[arg-type] + _, cfg = _build_render_config(self._conv(), {"thinking": {"type": "auto"}}) assert "reasoning_type" not in cfg def test_dev_instruction_forwarded(self): - _, cfg = _build_render_config(self._conv(), {"dev_instruction": "be brief"}) # type: ignore[arg-type] + _, cfg = _build_render_config(self._conv(), {"dev_instruction": "be brief"}) assert cfg["dev_instruction"] == "be brief" def test_response_format_json_object_sets_json_mode(self): _, cfg = _build_render_config( self._conv(), {"response_format": {"type": "json_object"}} - ) # type: ignore[arg-type] + ) assert cfg["json_mode"] is True assert "json_schema" not in cfg @@ -705,7 +705,7 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {"response_format": {"type": "json_schema", "schema": schema}}, - ) # type: ignore[arg-type] + ) # JSON-encoded for melody (string-only schema field). assert cfg["json_schema"] == json.dumps(schema) @@ -721,35 +721,35 @@ class TestBuildRenderConfig: "schema": {"schema": inner}, } }, - ) # type: ignore[arg-type] + ) assert cfg["json_schema"] == json.dumps(inner) def test_json_schema_kwarg_direct(self): # Caller can also pass ``json_schema`` directly, both as dict and # as a pre-stringified value. - _, cfg = _build_render_config(self._conv(), {"json_schema": {"a": 1}}) # type: ignore[arg-type] + _, cfg = _build_render_config(self._conv(), {"json_schema": {"a": 1}}) assert cfg["json_schema"] == '{"a": 1}' _, cfg = _build_render_config( self._conv(), {"json_schema": "raw-string-schema"} - ) # type: ignore[arg-type] + ) assert cfg["json_schema"] == "raw-string-schema" def test_json_mode_kwarg_overrides(self): - _, cfg = _build_render_config(self._conv(), {"json_mode": True}) # type: ignore[arg-type] + _, cfg = _build_render_config(self._conv(), {"json_mode": True}) assert cfg["json_mode"] is True def test_cmd3_safety_mode_lowercased(self): _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd3", "safety_mode": "CONTEXTUAL"}, - ) # type: ignore[arg-type] + ) assert cfg["safety_mode"] == "contextual" def test_cmd3_citation_quality_direct(self): _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd3", "citation_quality": "ACCURATE"}, - ) # type: ignore[arg-type] + ) assert cfg["citation_quality"] == "accurate" def test_cmd3_citation_quality_derived_from_citation_options(self): @@ -758,20 +758,20 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd3", "citation_options": {"mode": "accurate"}}, - ) # type: ignore[arg-type] + ) assert cfg["citation_quality"] == "on" _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd3", "citation_options": {"mode": "off"}}, - ) # type: ignore[arg-type] + ) assert cfg["citation_quality"] == "off" def test_cmd3_skip_preamble_forwarded(self): _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd3", "skip_preamble": True}, - ) # type: ignore[arg-type] + ) assert cfg["skip_preamble"] is True def test_cmd3_no_grounding_field(self): @@ -779,7 +779,7 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd3", "grounding": "fast"}, - ) # type: ignore[arg-type] + ) assert "grounding" not in cfg @pytest.mark.parametrize( @@ -800,7 +800,7 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {"cohere_format": "cmd4", "grounding": raw}, - ) # type: ignore[arg-type] + ) assert cfg["grounding"] == expected @pytest.mark.parametrize( @@ -818,7 +818,7 @@ class TestBuildRenderConfig: "cohere_format": "cmd4", "citation_options": {"mode": mode}, }, - ) # type: ignore[arg-type] + ) assert cfg["grounding"] == expected def test_cmd4_grounding_rejects_unknown_value(self): @@ -826,7 +826,7 @@ class TestBuildRenderConfig: _build_render_config( self._conv(), {"cohere_format": "cmd4", "grounding": "foobar"}, - ) # type: ignore[arg-type] + ) def test_cmd4_platform_instruction(self): _, cfg = _build_render_config( @@ -835,7 +835,7 @@ class TestBuildRenderConfig: "cohere_format": "cmd4", "platform_instruction": "do this", }, - ) # type: ignore[arg-type] + ) assert cfg["platform_instruction"] == "do this" def test_cmd4_no_safety_mode_field(self): @@ -847,7 +847,7 @@ class TestBuildRenderConfig: "safety_mode": "contextual", "citation_quality": "on", }, - ) # type: ignore[arg-type] + ) assert "safety_mode" not in cfg assert "citation_quality" not in cfg @@ -862,7 +862,7 @@ class TestBuildRenderConfig: "my_var": "x", "documents": ["doc"], # consumed, must NOT leak through }, - ) # type: ignore[arg-type] + ) extras = cfg["additional_template_fields"] assert extras == {"reasoning_effort": "low", "my_var": "x"} # Sanity: the consumed key still produced its dedicated config slot. @@ -875,7 +875,7 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {"template_id": "tpl1"}, - ) # type: ignore[arg-type] + ) assert cfg["template_id"] == "tpl1" # use_jinja is always True, regardless of caller input. assert cfg["use_jinja"] is True @@ -896,7 +896,7 @@ class TestBuildRenderConfig: _build_render_config( self._conv(), {cohere_only_key: "raw {{ jinja }}"}, - ) # type: ignore[arg-type] + ) @pytest.mark.parametrize("cohere_only_key", ["template_jinja", "template"]) def test_cohere_only_template_kwargs_none_is_tolerated(self, cohere_only_key): @@ -906,7 +906,7 @@ class TestBuildRenderConfig: _, cfg = _build_render_config( self._conv(), {cohere_only_key: None}, - ) # type: ignore[arg-type] + ) assert cohere_only_key not in cfg assert "additional_template_fields" not in cfg @@ -918,7 +918,7 @@ class TestBuildRenderConfig: self._conv(), {}, "raw {{ jinja }}", - ) # type: ignore[arg-type] + ) assert cfg["template_jinja"] == "raw {{ jinja }}" assert cfg["use_jinja"] is True @@ -927,7 +927,7 @@ class TestBuildRenderConfig: self._conv(), {}, None, - ) # type: ignore[arg-type] + ) assert "template_jinja" not in cfg def test_tool_message_v2_content_key_picked_up_from_kwargs(self): @@ -954,7 +954,7 @@ class TestBuildRenderConfig: ] } } - _, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type] + _, cfg = _build_render_config(conv, kwargs) tool_msg = cfg["messages"][1] assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}] # And the key must be *consumed* -- not surfaced as a stray diff --git a/tests/renderers/test_completions.py b/tests/renderers/test_completions.py index 99862219b9..0dce7ef684 100644 --- a/tests/renderers/test_completions.py +++ b/tests/renderers/test_completions.py @@ -139,7 +139,7 @@ class TestValidatePrompt: with pytest.raises(TypeError, match="should be a list of integers"): renderer.render_prompts( - _preprocess_prompt(renderer.model_config, [[1, 2], ["foo", "bar"]]) # type: ignore[arg-type] + _preprocess_prompt(renderer.model_config, [[1, 2], ["foo", "bar"]]) ) diff --git a/tests/samplers/test_beam_search.py b/tests/samplers/test_beam_search.py index daf6f4386c..823202ba32 100644 --- a/tests/samplers/test_beam_search.py +++ b/tests/samplers/test_beam_search.py @@ -113,7 +113,7 @@ def test_beam_search_abort_returns_partial_outputs_and_continues_other_prompts( {"type": "token", "prompt_token_ids": [token]} for token in [prompt_token, 2] ] params = BeamSearchParams(beam_width=2, max_tokens=3) - outputs = llm.beam_search(prompts, params) # type: ignore[arg-type] + outputs = llm.beam_search(prompts, params) aborted, normal = outputs expected_tokens = ( @@ -326,7 +326,7 @@ def test_beam_search_structured_output( llm = LLM( model=model, - dtype=dtype, # type: ignore[arg-type] + dtype=dtype, max_model_len=512, structured_outputs_config=dict( backend="xgrammar", @@ -345,7 +345,7 @@ def test_beam_search_structured_output( "Generate a JSON object for a person with name and age:", ] - outputs = llm.beam_search(prompts, params) # type: ignore[arg-type] + outputs = llm.beam_search(prompts, params) assert len(outputs) == len(prompts) for output in outputs: diff --git a/tests/samplers/test_beam_search_online.py b/tests/samplers/test_beam_search_online.py index 4c48c6189d..cbed58363a 100644 --- a/tests/samplers/test_beam_search_online.py +++ b/tests/samplers/test_beam_search_online.py @@ -57,8 +57,8 @@ class _EngineClient: class _Serving(BeamSearchOnlineMixin): - renderer = _Renderer() # type: ignore[assignment] - engine_client = _EngineClient() # type: ignore[assignment] + renderer = _Renderer() + engine_client = _EngineClient() @pytest.mark.asyncio diff --git a/tests/standalone_tests/lazy_imports.py b/tests/standalone_tests/lazy_imports.py index fff5c54f27..963b19e211 100644 --- a/tests/standalone_tests/lazy_imports.py +++ b/tests/standalone_tests/lazy_imports.py @@ -17,6 +17,6 @@ module_names = ["torch._inductor.async_compile", "cv2"] # if we import any modules during `import vllm`, there would be a # hard error and nice stacktrace on the first import. for module_name in module_names: - sys.modules[module_name] = None # type: ignore[assignment] + sys.modules[module_name] = None import vllm # noqa diff --git a/tests/test_audio_media_size_precheck.py b/tests/test_audio_media_size_precheck.py index e1f2ed8df4..7635022369 100644 --- a/tests/test_audio_media_size_precheck.py +++ b/tests/test_audio_media_size_precheck.py @@ -144,6 +144,6 @@ async def test_audio_connector_threads_byte_limit_to_http_reader( assert kwargs["max_bytes"] == MiB_bytes raise VLLMValidationError("Maximum file size exceeded") - connector = MediaConnector(connection=_Connection()) # type: ignore[arg-type] + connector = MediaConnector(connection=_Connection()) with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"): await connector.fetch_audio_async("https://example.test/audio") diff --git a/tests/tokenizers_/test_mistral.py b/tests/tokenizers_/test_mistral.py index 5229eaaf9c..24bd8e00f1 100644 --- a/tests/tokenizers_/test_mistral.py +++ b/tests/tokenizers_/test_mistral.py @@ -975,7 +975,7 @@ class TestMistralTokenizer: mistral_tokenizer: MistralTokenizer, ): assert mistral_tokenizer.batch_decode( - [[]], # type: ignore[arg-type] + [[]], ) == [""] def test_convert_tokens_to_string(self, mistral_tokenizer: MistralTokenizer): @@ -2246,7 +2246,7 @@ class TestMistralTokenizer: ] output = mistral_tokenizer.apply_chat_template( - messages, # type: ignore[arg-type] + messages, add_generation_prompt=True, ) decoded = mistral_tokenizer.tokenizer.decode(output, SpecialTokenPolicy.KEEP) @@ -2352,7 +2352,7 @@ def test_v15_apply_chat_template_passes_reasoning_effort_high( ) v15_mistral_tokenizer.apply_chat_template( - messages=_PASSTHROUGH_MESSAGES, # type: ignore[arg-type] + messages=_PASSTHROUGH_MESSAGES, reasoning_effort="high", ) @@ -2375,7 +2375,7 @@ def test_v15_apply_chat_template_passes_reasoning_effort_none_by_default( fake_apply_chat_template, ) - v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES) # type: ignore[arg-type] + v15_mistral_tokenizer.apply_chat_template(messages=_PASSTHROUGH_MESSAGES) assert "reasoning_effort" in captured_kwargs[-1] assert captured_kwargs[-1]["reasoning_effort"] is None @@ -2398,7 +2398,7 @@ def test_pre_v15_apply_chat_template_omits_reasoning_effort( ) v13_mistral_tokenizer.apply_chat_template( - messages=_PASSTHROUGH_MESSAGES, # type: ignore[arg-type] + messages=_PASSTHROUGH_MESSAGES, reasoning_effort="high", ) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index e1da601b64..a4ee4a755e 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -205,7 +205,7 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path): side_effect=lambda tokenizer: tokenizer, ), ): - tokenizer = cached_tokenizer_from_config(model_config) # type: ignore[arg-type] + tokenizer = cached_tokenizer_from_config(model_config) assert tokenizer.is_fast is True finally: diff --git a/tests/tool_parsers/test_ernie45_moe_tool_parser.py b/tests/tool_parsers/test_ernie45_moe_tool_parser.py index b6f536ca74..8ad2cdfc33 100644 --- a/tests/tool_parsers/test_ernie45_moe_tool_parser.py +++ b/tests/tool_parsers/test_ernie45_moe_tool_parser.py @@ -54,7 +54,7 @@ def test_extract_tool_calls_no_tools(ernie45_tool_parser): model_output = "This is a test" extracted_tool_calls = ernie45_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert not extracted_tool_calls.tools_called assert extracted_tool_calls.tool_calls == [] assert extracted_tool_calls.content == model_output @@ -163,7 +163,7 @@ def test_extract_tool_calls( ): extracted_tool_calls = ernie45_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert extracted_tool_calls.tools_called assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) diff --git a/tests/tool_parsers/test_jamba_tool_parser.py b/tests/tool_parsers/test_jamba_tool_parser.py index 33f44fe838..54c6cd747d 100644 --- a/tests/tool_parsers/test_jamba_tool_parser.py +++ b/tests/tool_parsers/test_jamba_tool_parser.py @@ -78,7 +78,7 @@ def stream_delta_message_generator( previous_token_ids, current_token_ids, delta_token_ids, - request=None, # type: ignore[arg-type] + request=None, ) if delta_message: yield delta_message @@ -95,7 +95,7 @@ def test_extract_tool_calls_no_tools(jamba_tool_parser): model_output = "This is a test" extracted_tool_calls = jamba_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert not extracted_tool_calls.tools_called assert extracted_tool_calls.tool_calls == [] assert extracted_tool_calls.content == model_output @@ -166,7 +166,7 @@ def test_extract_tool_calls( ): extracted_tool_calls = jamba_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert extracted_tool_calls.tools_called assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) diff --git a/tests/tool_parsers/test_qwen3coder_tool_parser.py b/tests/tool_parsers/test_qwen3coder_tool_parser.py index c397d0ffad..45212704d0 100644 --- a/tests/tool_parsers/test_qwen3coder_tool_parser.py +++ b/tests/tool_parsers/test_qwen3coder_tool_parser.py @@ -225,7 +225,7 @@ def test_extract_tool_calls_no_tools(qwen3_tool_parser): model_output = "This is a test response without any tool calls" extracted_tool_calls = qwen3_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert not extracted_tool_calls.tools_called assert extracted_tool_calls.tool_calls == [] assert extracted_tool_calls.content == model_output diff --git a/tests/tool_parsers/test_step3p5_tool_parser.py b/tests/tool_parsers/test_step3p5_tool_parser.py index 0dbe114545..8291c62319 100644 --- a/tests/tool_parsers/test_step3p5_tool_parser.py +++ b/tests/tool_parsers/test_step3p5_tool_parser.py @@ -170,7 +170,7 @@ def test_extract_tool_calls_no_tools(step3p5_tool_parser): model_output = "This is a test response without any tool calls" extracted_tool_calls = step3p5_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert not extracted_tool_calls.tools_called assert extracted_tool_calls.tool_calls == [] assert extracted_tool_calls.content == model_output diff --git a/tests/tool_parsers/test_xlam_tool_parser.py b/tests/tool_parsers/test_xlam_tool_parser.py index 4478e97a60..94e3e37880 100644 --- a/tests/tool_parsers/test_xlam_tool_parser.py +++ b/tests/tool_parsers/test_xlam_tool_parser.py @@ -100,7 +100,7 @@ def test_extract_tool_calls_no_tools(xlam_tool_parser): model_output = "This is a test" extracted_tool_calls = xlam_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert not extracted_tool_calls.tools_called assert extracted_tool_calls.tool_calls == [] assert extracted_tool_calls.content == model_output @@ -225,7 +225,7 @@ def test_extract_tool_calls( ): extracted_tool_calls = xlam_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert extracted_tool_calls.tools_called assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) @@ -263,7 +263,7 @@ def test_extract_tool_calls_list_structure( """Test extraction of tool calls when the model outputs a list-structured tool call.""" # noqa: E501 extracted_tool_calls = xlam_tool_parser.extract_tool_calls( model_output, request=None - ) # type: ignore[arg-type] + ) assert extracted_tool_calls.tools_called assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) @@ -550,7 +550,7 @@ def test_extract_tool_calls_non_ascii(xlam_tool_parser, xlam_tokenizer, streamin if delta.tool_calls and delta.tool_calls[0].function.arguments ) else: - extracted = xlam_tool_parser.extract_tool_calls(model_output, request=None) # type: ignore[arg-type] + extracted = xlam_tool_parser.extract_tool_calls(model_output, request=None) args = "".join(tc.function.arguments for tc in extracted.tool_calls) assert "北京" in args diff --git a/tests/tool_use/test_gemma4_responses_adjust_request.py b/tests/tool_use/test_gemma4_responses_adjust_request.py index 403c707924..fabfa74766 100644 --- a/tests/tool_use/test_gemma4_responses_adjust_request.py +++ b/tests/tool_use/test_gemma4_responses_adjust_request.py @@ -148,7 +148,7 @@ def test_tool_parser_adjust_request_builds_valid_response_text_config() -> None: description string that the previous two-step construction injected. """ parser = ToolParser.__new__(ToolParser) - parser.model_tokenizer = None # type: ignore[assignment] + parser.model_tokenizer = None request = _build_responses_request(tool_choice="required") ToolParser.adjust_request(parser, request) diff --git a/tests/tool_use/test_kimi_k3_tool_parser.py b/tests/tool_use/test_kimi_k3_tool_parser.py index ac7dcfb8ac..05679bd192 100644 --- a/tests/tool_use/test_kimi_k3_tool_parser.py +++ b/tests/tool_use/test_kimi_k3_tool_parser.py @@ -370,7 +370,7 @@ def test_streaming_split_markers_do_not_leak(): assert len(tool_deltas) == 1 assert tool_deltas[0].function is not None assert tool_deltas[0].function.name == "calc" - assert json.loads(tool_deltas[0].function.arguments) == {"x": 1} # type: ignore[arg-type] + assert json.loads(tool_deltas[0].function.arguments) == {"x": 1} def test_tool_call_ids_are_unique_across_messages(): diff --git a/tests/transformers_utils/processors/test_voxtral.py b/tests/transformers_utils/processors/test_voxtral.py index 0ca8f1a94d..015d3b005e 100644 --- a/tests/transformers_utils/processors/test_voxtral.py +++ b/tests/transformers_utils/processors/test_voxtral.py @@ -98,7 +98,7 @@ def test_fetch_audio_rejects_unsupported_type( feature_extractor: MistralCommonFeatureExtractor, ): with pytest.raises(TypeError, match="only a numpy array"): - feature_extractor.fetch_audio(42) # type: ignore[arg-type] + feature_extractor.fetch_audio(42) def test_fetch_audio_str_delegates_to_load_audio( diff --git a/tests/transformers_utils/test_processor.py b/tests/transformers_utils/test_processor.py index e9070619d9..a3a1c78418 100644 --- a/tests/transformers_utils/test_processor.py +++ b/tests/transformers_utils/test_processor.py @@ -44,7 +44,7 @@ class _ProcWithUnpack: def test_get_processor_kwargs_from_processor_unpack_path_returns_full_union(): proc = _ProcWithUnpack() - keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc)) # type: ignore[arg-type] + keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc)) _assert_has_all_expected(keys) @@ -63,5 +63,5 @@ def test_get_processor_kwargs_from_processor_module_scan_returns_full_union(): assert hasattr(mod, "_FakeProcessorKwargs") proc = _ProcWithoutUnpack() - keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc)) # type: ignore[arg-type] + keys = get_processor_kwargs_keys(get_processor_kwargs_type(proc)) _assert_has_all_expected(keys) diff --git a/tests/utils.py b/tests/utils.py index 10fcffffea..dd6bf4ae96 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -830,7 +830,7 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer): self.proc: Process = cast(Any, ctx).Process( target=_run_in_new_process_group, args=(self.child_process_fxn, env_dict, model, vllm_serve_args), - ) # type: ignore[assignment] + ) with _temporarily_sanitized_pythonpath_env(): self.proc.start() diff --git a/tests/utils_/test_argparse_utils.py b/tests/utils_/test_argparse_utils.py index 815ac3fa19..89e0f9977d 100644 --- a/tests/utils_/test_argparse_utils.py +++ b/tests/utils_/test_argparse_utils.py @@ -476,32 +476,32 @@ def test_compilation_config_mode_validator(): """Test that CompilationConfig.mode field validator converts strings to integers.""" from vllm.config.compilation import CompilationConfig, CompilationMode - config = CompilationConfig(mode=0) # type: ignore[arg-type] + config = CompilationConfig(mode=0) assert config.mode == CompilationMode.NONE - config = CompilationConfig(mode=3) # type: ignore[arg-type] + config = CompilationConfig(mode=3) assert config.mode == CompilationMode.VLLM_COMPILE - config = CompilationConfig(mode="NONE") # type: ignore[arg-type] + config = CompilationConfig(mode="NONE") assert config.mode == CompilationMode.NONE - config = CompilationConfig(mode="STOCK_TORCH_COMPILE") # type: ignore[arg-type] + config = CompilationConfig(mode="STOCK_TORCH_COMPILE") assert config.mode == CompilationMode.STOCK_TORCH_COMPILE - config = CompilationConfig(mode="DYNAMO_TRACE_ONCE") # type: ignore[arg-type] + config = CompilationConfig(mode="DYNAMO_TRACE_ONCE") assert config.mode == CompilationMode.DYNAMO_TRACE_ONCE - config = CompilationConfig(mode="VLLM_COMPILE") # type: ignore[arg-type] + config = CompilationConfig(mode="VLLM_COMPILE") assert config.mode == CompilationMode.VLLM_COMPILE - config = CompilationConfig(mode="none") # type: ignore[arg-type] + config = CompilationConfig(mode="none") assert config.mode == CompilationMode.NONE - config = CompilationConfig(mode="vllm_compile") # type: ignore[arg-type] + config = CompilationConfig(mode="vllm_compile") assert config.mode == CompilationMode.VLLM_COMPILE with pytest.raises(ValidationError, match="Invalid compilation mode"): - CompilationConfig(mode="INVALID_MODE") # type: ignore[arg-type] + CompilationConfig(mode="INVALID_MODE") def test_flat_product(): diff --git a/tests/v1/attention/test_mla_prefill_selector.py b/tests/v1/attention/test_mla_prefill_selector.py index 87d188c6a0..d5b81695aa 100644 --- a/tests/v1/attention/test_mla_prefill_selector.py +++ b/tests/v1/attention/test_mla_prefill_selector.py @@ -511,14 +511,14 @@ class TestMLAPrefillBackendParsing: def test_valid_string_parses_to_enum(self): config = AttentionConfig( - mla_prefill_backend="FLASH_ATTN", # type: ignore[arg-type] + mla_prefill_backend="FLASH_ATTN", ) assert config.mla_prefill_backend == MLAPrefillBackendEnum.FLASH_ATTN def test_invalid_string_raises_error(self): with pytest.raises(ValueError, match="Unknown MLA prefill backend"): AttentionConfig( - mla_prefill_backend="NONEXISTENT", # type: ignore[arg-type] + mla_prefill_backend="NONEXISTENT", ) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 1ea1415029..048eb40586 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -3575,7 +3575,7 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b if use_mixed_batch: fp8_metadata = FlashMLASparseMetadata.FP8KernelMetadata( - scheduler_metadata=object(), # type: ignore[arg-type] + scheduler_metadata=object(), dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE), cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), ) @@ -3588,7 +3588,7 @@ def test_flashmla_fp8_paths_accept_decode_subset(monkeypatch, use_mixed_batch: b num_prefill_tokens=num_batch_tokens - num_decode_tokens, decode=FP8Meta.Decode( seq_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), - kernel_metadata=object(), # type: ignore[arg-type] + kernel_metadata=object(), decode_query_len=num_decode_tokens, ), prefill=FP8Meta.Prefill( @@ -3774,7 +3774,7 @@ def test_fp8_mixed_batch_dcp_neutralizes_empty_rows(monkeypatch): metadata = SimpleNamespace( fp8_extra_metadata=FlashMLASparseMetadata.FP8KernelMetadata( - scheduler_metadata=object(), # type: ignore[arg-type] + scheduler_metadata=object(), dummy_block_table=torch.empty(1, 1, dtype=torch.int32, device=DEVICE_TYPE), cache_lens=torch.empty(1, dtype=torch.int32, device=DEVICE_TYPE), ), diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index e5f2f6369f..366db7b025 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -134,7 +134,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings(): device=torch.device("cpu"), max_num_reqs=num_reqs, max_num_tokens=num_tokens, - block_tables=pcp_block_tables, # type: ignore[arg-type] + block_tables=pcp_block_tables, ) block_tables = MagicMock() @@ -224,7 +224,7 @@ def test_speculator_capture_preserves_decode_query_bounds( ) manager = SpeculatorCudaGraphManager.__new__(SpeculatorCudaGraphManager) manager.max_num_reqs = num_reqs - manager.vllm_config = SimpleNamespace( # type: ignore[assignment] + manager.vllm_config = SimpleNamespace( parallel_config=SimpleNamespace(data_parallel_size=1) ) buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu")) diff --git a/tests/v1/cudagraph/test_encoder_cudagraph.py b/tests/v1/cudagraph/test_encoder_cudagraph.py index 2f4de52b18..749f9aa20d 100644 --- a/tests/v1/cudagraph/test_encoder_cudagraph.py +++ b/tests/v1/cudagraph/test_encoder_cudagraph.py @@ -652,7 +652,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only encoder.device = device encoder.cudagraph_manager = manager runner = object.__new__(MMEncoderModelRunner) - runner.model_state = SimpleNamespace(encoder_runner=encoder) # type: ignore[assignment] + runner.model_state = SimpleNamespace(encoder_runner=encoder) # No decoder manager is installed: capture must be encoder-only. with patch( "vllm.v1.worker.mm_encoder_model_runner.lock_workspace", wraps=lock_workspace @@ -673,7 +673,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only with patch( "vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=connector ): - ec = ActiveECConnector(SimpleNamespace(), cache) # type: ignore[arg-type] + ec = ActiveECConnector(SimpleNamespace(), cache) scheduled = cast( SchedulerOutput, SimpleNamespace( @@ -716,7 +716,7 @@ def test_eonly_without_encoder_graph_skips_capture(): encoder = object.__new__(EncoderRunner) encoder.cudagraph_manager = None runner = object.__new__(MMEncoderModelRunner) - runner.model_state = SimpleNamespace(encoder_runner=encoder) # type: ignore[assignment] + runner.model_state = SimpleNamespace(encoder_runner=encoder) assert runner.capture_model() == 0 @@ -1023,7 +1023,7 @@ class TestInitInvariantValidation: vllm_config = _MockVllmConfig(token_budgets, max_mm_items) model = _MockModel(min_budget, max_budget) return EncoderCudaGraphManager( - vllm_config=vllm_config, # type: ignore[arg-type] + vllm_config=vllm_config, device=torch.device("cpu"), dtype=torch.float32, model=model, diff --git a/tests/v1/distributed/test_eagle_dp.py b/tests/v1/distributed/test_eagle_dp.py index bb2b9f9fae..019f9c6f21 100644 --- a/tests/v1/distributed/test_eagle_dp.py +++ b/tests/v1/distributed/test_eagle_dp.py @@ -61,7 +61,7 @@ async def test_run_eagle_dp(monkeypatch: pytest.MonkeyPatch, attn_backend: str): data_parallel_backend="mp", # ray takes more time trust_remote_code=True, max_model_len=16384, - attention_config={"backend": attn_backend}, # type: ignore[arg-type] + attention_config={"backend": attn_backend}, ) eagle_engine_args = replace( diff --git a/tests/v1/engine/test_startup_watch_processes.py b/tests/v1/engine/test_startup_watch_processes.py index feb1e04268..19b2d7bca8 100644 --- a/tests/v1/engine/test_startup_watch_processes.py +++ b/tests/v1/engine/test_startup_watch_processes.py @@ -304,9 +304,9 @@ def test_wait_for_engine_startup_reports_watched_process_exit(): wait_for_engine_startup( handshake_socket, [CoreEngine()], - parallel_config, # type: ignore[arg-type] + parallel_config, coordinated_dp=False, - cache_config=None, # type: ignore[arg-type] + cache_config=None, launch=launch, ) finally: diff --git a/tests/v1/executor/test_executor.py b/tests/v1/executor/test_executor.py index 949282cffb..806298acc6 100644 --- a/tests/v1/executor/test_executor.py +++ b/tests/v1/executor/test_executor.py @@ -126,7 +126,7 @@ def test_custom_executor_type_checking(): model=MODEL, gpu_memory_utilization=0.2, max_model_len=8192, - distributed_executor_backend=Mock, # type: ignore[arg-type] + distributed_executor_backend=Mock, ) LLMEngine.from_engine_args(engine_args) with pytest.raises(ValueError): @@ -134,7 +134,7 @@ def test_custom_executor_type_checking(): model=MODEL, gpu_memory_utilization=0.2, max_model_len=8192, - distributed_executor_backend=Mock, # type: ignore[arg-type] + distributed_executor_backend=Mock, ) AsyncLLM.from_engine_args(engine_args) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py b/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py index b7d9d6c8ef..e9ca7cb466 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_metrics.py @@ -480,7 +480,7 @@ def test_reset(): def test_prom_metrics_observes_manager_counter(): prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -506,7 +506,7 @@ def test_prom_metrics_observes_manager_counter(): def test_prom_metrics_observes_flat_transfer_metrics_and_legacy_metrics(): prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -568,7 +568,7 @@ def test_prom_metrics_observes_manager_gauge_and_histogram(): return_value=_spec_cls_with_metric_definitions(metric_definitions), ): prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(store_threshold=0), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -612,7 +612,7 @@ def test_prom_metrics_lazily_observes_labeled_metric(): return_value=_spec_cls_with_metric_definitions(metric_definitions), ): prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(store_threshold=0), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -651,7 +651,7 @@ def test_prom_metrics_rejects_wrong_label_count(): return_value=_spec_cls_with_metric_definitions(metric_definitions), ): prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(store_threshold=0), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -672,7 +672,7 @@ def test_prom_metrics_rejects_wrong_label_count(): def test_prom_metrics_uses_configured_manager_metrics(): prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(store_threshold=0), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -693,7 +693,7 @@ def test_prom_metrics_registers_tiering_metrics_from_spec(): "spec_name": "TieringOffloadingSpec", "secondary_tiers": [], }, - ), # type: ignore[arg-type] + ), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -741,7 +741,7 @@ def test_aggregate_into_empty_stats(): def test_prom_metrics_multi_engine_routing(): """Metrics are routed to the correct engine index.""" prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, @@ -767,7 +767,7 @@ def test_prom_metrics_multi_engine_routing(): def test_prom_metrics_rejects_undeclared_metric(): """observe() asserts if a metric was never declared in metadata.""" prom_metrics = OffloadPromMetrics( - vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type] + vllm_config=_FakeVllmConfig(store_threshold=0), metric_types={ Gauge: _FakeMetric, Counter: _FakeMetric, diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 1ef77e285a..cb6bd8d995 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -1543,7 +1543,7 @@ def test_store_sending_thread_reports_job_when_the_preamble_raises(): # every dequeue leaves through the same exit. thread = _make_store_sending_thread(MagicMock()) req = _make_store_req("req-a", [b"a0", b"a1"]) - req.token_len_chunk = None # type: ignore[assignment] + req.token_len_chunk = None with contextlib.suppress(TypeError): _run_store_req(thread, req) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 017dee10d8..6be77b7b9d 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -114,7 +114,7 @@ def get_default_xfer_telemetry( class AttributeDict(dict): __slots__ = () __getattr__ = dict.__getitem__ - __setattr__ = dict.__setitem__ # type: ignore[assignment] + __setattr__ = dict.__setitem__ # We can't instantiate nixlXferTelemetry because it's read only and # ray env does not have NIXL, so we must fake it diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index 7ce6666ae8..de47376117 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -1256,7 +1256,7 @@ class TestPushWriterNegative: w = _StubWriterWorker.fresh() for bogus_rid in (123, None, 4.5, b"bytes-not-str"): payload = _registration_data("placeholder") - payload["request_id"] = bogus_rid # type: ignore[assignment] + payload["request_id"] = bogus_rid notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(payload) w._handle_push_reg_notif(notif) assert w._pending_d_registrations == {} diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py index 8e806582fd..e35b1d022e 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_manager.py +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -349,10 +349,10 @@ class TestServeExternalRequests: mgr._failed_serve_ctxs = [ctx] sess_a = _RecordingSession() sess_b = _RecordingSession() - mgr._sessions = {"a": sess_a, "b": sess_b} # type: ignore[assignment] + mgr._sessions = {"a": sess_a, "b": sess_b} parent = _RecordingParent() - mgr.serve_external_requests(parent) # type: ignore[arg-type] + mgr.serve_external_requests(parent) # Failed serve released and queue cleared. assert parent.finished == ["p2p:peer:req-1:lu1"] @@ -364,10 +364,10 @@ class TestServeExternalRequests: def test_no_failed_serves_still_serves_sessions(self): mgr = _make_manager() sess = _RecordingSession() - mgr._sessions = {"a": sess} # type: ignore[assignment] + mgr._sessions = {"a": sess} parent = _RecordingParent() - mgr.serve_external_requests(parent) # type: ignore[arg-type] + mgr.serve_external_requests(parent) assert parent.finished == [] assert sess.served_with == [parent] @@ -427,7 +427,7 @@ class TestSubmitStore: forwards directly to that session rather than re-buffering.""" mgr = _make_manager() bound = _FakeSession(peer_id="10.0.0.1:8000", connected=True) - mgr._kv_to_session["req-1"] = bound # type: ignore[assignment] + mgr._kv_to_session["req-1"] = bound # Note: _sessions is intentionally untouched — this test isolates # the kv_request_id → session fast path. job = _job_metadata( @@ -495,7 +495,7 @@ class TestSubmitLoad: mgr = _make_manager() peer_id = "10.0.0.1:8000" existing = _FakeSession(peer_id=peer_id, connected=True) - mgr._sessions[peer_id] = existing # type: ignore[assignment] + mgr._sessions[peer_id] = existing job = _job_metadata( job_id=42, keys=[b"k1", b"k2"], @@ -581,7 +581,7 @@ class TestOnRequestFinished: (FetchMsg received) routes via _kv_to_session and pops the entry.""" mgr = _make_manager() bound = _FakeSession(peer_id="some-peer:1", connected=True) - mgr._kv_to_session["req-1"] = bound # type: ignore[assignment] + mgr._kv_to_session["req-1"] = bound ctx = _req_context(kv_params=_remote_decoder_kv_params(kv_request_id="req-1")) mgr.on_request_finished(ctx) assert bound.finishes == ["req-1"] @@ -718,8 +718,8 @@ class TestGetFinished: def poll(self): return [] - mgr._control = FakeControl() # type: ignore[assignment] - mgr._data = None # type: ignore[assignment] + mgr._control = FakeControl() + mgr._data = None return mgr def test_drains_finished_jobs(self): @@ -740,7 +740,7 @@ class TestGetFinished: pass mgr = self._make() - mgr._data = FakeData() # type: ignore[assignment] + mgr._data = FakeData() dead = _FakeSession( peer_id="dead:1234", alive=False, @@ -749,7 +749,7 @@ class TestGetFinished: close_req_ids=["req-load"], close_stores=[10, 11], ) - mgr._sessions["dead:1234"] = dead # type: ignore[assignment] + mgr._sessions["dead:1234"] = dead results = list(mgr.get_finished_jobs()) # 2 baseline + 1 failed load + 2 failed stores @@ -769,14 +769,14 @@ class TestGetFinished: pass mgr = self._make() - mgr._data = FakeData() # type: ignore[assignment] + mgr._data = FakeData() dead = _FakeSession( peer_id="dead:1234", alive=False, connected=True, close_req_ids=["req-probe-1", "req-probe-2"], ) - mgr._sessions["dead:1234"] = dead # type: ignore[assignment] + mgr._sessions["dead:1234"] = dead list(mgr.get_finished_jobs()) assert "dead:1234" not in mgr._sessions @@ -854,7 +854,7 @@ class TestHasPendingWork: """The result is the same regardless of session state — there is no 'idle' branch.""" mgr = _make_manager() - mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # type: ignore[assignment] + mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") assert mgr.has_pending_work() is True @@ -919,12 +919,12 @@ class TestShutdownDrain: mgr = _make_manager() data = _ShutdownFakeData(still_queue=still_queue) control = _ShutdownFakeControl() - mgr._data = data # type: ignore[assignment] - mgr._control = control # type: ignore[assignment] + mgr._data = data + mgr._control = control if inflight_ids: session = _FakeSession(peer_id="peer:1", connected=True) session._server._inflight = {tid: object() for tid in inflight_ids} - mgr._sessions["peer:1"] = session # type: ignore[assignment] + mgr._sessions["peer:1"] = session return mgr, data, control def test_shutdown_drains_inflight_via_wait_cancel(self): @@ -1148,10 +1148,10 @@ def _build_paired_managers() -> tuple[P2PSecondaryTierManager, P2PSecondaryTierM ctrl_b = _LoopbackControl(mgr_b._local_id) ctrl_a.pair(ctrl_b) - mgr_a._control = ctrl_a # type: ignore[assignment] - mgr_b._control = ctrl_b # type: ignore[assignment] - mgr_a._data = _FakeData(mgr_a._local_id) # type: ignore[assignment] - mgr_b._data = _FakeData(mgr_b._local_id) # type: ignore[assignment] + mgr_a._control = ctrl_a + mgr_b._control = ctrl_b + mgr_a._data = _FakeData(mgr_a._local_id) + mgr_b._data = _FakeData(mgr_b._local_id) return mgr_a, mgr_b @@ -1334,7 +1334,7 @@ class TestAcceptNewPeers: mgr = _make_manager() peer_id = "10.0.0.1:8000" existing = _FakeSession(peer_id=peer_id, connected=True) - mgr._sessions[peer_id] = existing # type: ignore[assignment] + mgr._sessions[peer_id] = existing new_conn = _RecordingConn(peer_id) mgr._accept_new_peers([new_conn]) @@ -1365,7 +1365,7 @@ class TestAcceptNewPeers: pass mgr = _make_manager() - mgr._data = FakeData() # type: ignore[assignment] + mgr._data = FakeData() peer_id = "10.0.0.1:8000" # Real ControlConnection-shaped fake: send/close/peer_id only. @@ -1382,7 +1382,7 @@ class TestAcceptNewPeers: def close(self) -> None: self.alive = False - mgr._accept_new_peers([_Conn(peer_id)]) # type: ignore[arg-type] + mgr._accept_new_peers([_Conn(peer_id)]) assert peer_id in mgr._sessions assert mgr._sessions[peer_id].connected is True @@ -1411,7 +1411,7 @@ class TestPollOnce: loads=[LoadResult(job_id=11, kv_request_id="req-11", success=True)], stores=[StoreResult(job_id=22, success=True)], ) - mgr._sessions[peer_alive] = alive # type: ignore[assignment] + mgr._sessions[peer_alive] = alive # Dead session whose pending close() jobs surface as failures. peer_dead = "10.0.0.3:9999" @@ -1423,7 +1423,7 @@ class TestPollOnce: close_req_ids=["req-33"], close_stores=[44], ) - mgr._sessions[peer_dead] = dead # type: ignore[assignment] + mgr._sessions[peer_dead] = dead class _Ctrl: def poll(self_inner): @@ -1433,8 +1433,8 @@ class TestPollOnce: def remove_remote_peer(self_inner, pid): pass - mgr._control = _Ctrl() # type: ignore[assignment] - mgr._data = _Data() # type: ignore[assignment] + mgr._control = _Ctrl() + mgr._data = _Data() mgr._poll_once() @@ -1464,7 +1464,7 @@ class TestPollOnce: connected=True, new_fetch_ids=["req-1"], ) - mgr._sessions[peer] = sess # type: ignore[assignment] + mgr._sessions[peer] = sess mgr._unbound_stores["req-1"] = [ _UnboundStoreBatch(job_id=5, keys=[b"k1"], block_ids=[0]), _UnboundStoreBatch(job_id=6, keys=[b"k2"], block_ids=[1]), @@ -1474,7 +1474,7 @@ class TestPollOnce: def poll(self_inner): return [] - mgr._control = _Ctrl() # type: ignore[assignment] + mgr._control = _Ctrl() mgr._poll_once() assert mgr._kv_to_session["req-1"] is sess @@ -1496,13 +1496,13 @@ class TestPollOnce: connected=True, new_fetch_ids=["req-fast"], ) - mgr._sessions[peer] = sess # type: ignore[assignment] + mgr._sessions[peer] = sess class _Ctrl: def poll(self_inner): return [] - mgr._control = _Ctrl() # type: ignore[assignment] + mgr._control = _Ctrl() mgr._poll_once() assert mgr._kv_to_session["req-fast"] is sess @@ -1519,13 +1519,13 @@ class TestPollOnce: connected=True, loads=[LoadResult(job_id=5, kv_request_id="req-5", success=False)], ) - mgr._sessions[peer] = sess # type: ignore[assignment] + mgr._sessions[peer] = sess class _Ctrl: def poll(self_inner): return [] - mgr._control = _Ctrl() # type: ignore[assignment] + mgr._control = _Ctrl() mgr._poll_once() @@ -1549,26 +1549,26 @@ class TestDrainJobs: def test_returns_immediately_when_quiescent(self): """No sessions and no inflight: drain_jobs returns without sleeping.""" mgr = _make_manager() - mgr._control = _DrainCtrl() # type: ignore[assignment] + mgr._control = _DrainCtrl() sleeps: list[float] = [] # If drain_jobs sleeps when nothing is pending, that's a regression. import vllm.v1.kv_offload.tiering.p2p.manager as m original_sleep = m.time.sleep - m.time.sleep = lambda s: sleeps.append(s) # type: ignore[assignment] + m.time.sleep = lambda s: sleeps.append(s) try: mgr.drain_jobs() finally: - m.time.sleep = original_sleep # type: ignore[assignment] + m.time.sleep = original_sleep assert sleeps == [] def test_returns_when_session_has_no_inflight_or_inbound(self): """A session with empty _inbound and _inflight does not block drain.""" mgr = _make_manager() - mgr._control = _DrainCtrl() # type: ignore[assignment] - mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # type: ignore[assignment] + mgr._control = _DrainCtrl() + mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # Should return on the first iteration. mgr.drain_jobs() @@ -1576,10 +1576,10 @@ class TestDrainJobs: """A session that stays inflight past 5s triggers the warning, and once it clears the loop returns.""" mgr = _make_manager() - mgr._control = _DrainCtrl() # type: ignore[assignment] + mgr._control = _DrainCtrl() sess = _FakeSession(peer_id="peer:1") sess._server._inflight = {1: object()} # non-empty - mgr._sessions["peer:1"] = sess # type: ignore[assignment] + mgr._sessions["peer:1"] = sess # Synthetic monotonic clock: 100.0 for the start stamp, then 106.0 # for the first elapsed-check (past the 5s warning threshold), then diff --git a/tests/v1/kv_offload/tiering/p2p/test_sessions.py b/tests/v1/kv_offload/tiering/p2p/test_sessions.py index a09445668f..09739345be 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_sessions.py +++ b/tests/v1/kv_offload/tiering/p2p/test_sessions.py @@ -288,17 +288,17 @@ def _make_session( session = P2PSession( peer_id=peer_id, local_id=local_id, - transport=transport, # type: ignore[arg-type] + transport=transport, local_block_len=transport.block_len, local_hash_seed=local_hash_seed, - conn=conn, # type: ignore[arg-type] + conn=conn, ) return session, conn, transport def _serve(session: P2PSession, parent: FakeParent) -> None: """Resolve enqueued inbound lookups, as the manager does each step.""" - session.serve_external_requests(parent) # type: ignore[arg-type] + session.serve_external_requests(parent) def _activate( @@ -1951,7 +1951,7 @@ class TestFinishRequestServerSide: ) session.poll() # Force write_blocks to fail on the next call. - transport.write_blocks = lambda *a, **kw: None # type: ignore[assignment] + transport.write_blocks = lambda *a, **kw: None # type: ignore[method-assign] session.add_stored_blocks("req-1", [b"k1"], [0], job_id=42) @@ -2057,7 +2057,7 @@ class TestFinishRequestServerSide: assert outbound.finishing is False # Round 2: write_blocks fails for k2 while transfer_1 is still inflight. - transport.write_blocks = lambda *a, **kw: None # type: ignore[assignment] + transport.write_blocks = lambda *a, **kw: None # type: ignore[method-assign] session.add_stored_blocks("req-1", [b"k2"], [1], job_id=200) # No new transfer was registered. assert list(session._server._inflight.keys()) == [tid_1] @@ -2163,7 +2163,7 @@ class TestPendingSession: session = P2PSession( peer_id="peer:8000", local_id="local:9000", - transport=transport, # type: ignore[arg-type] + transport=transport, local_block_len=4096, local_hash_seed=_DEFAULT_HASH_SEED, conn=None, @@ -2183,13 +2183,13 @@ class TestPendingSession: session = P2PSession( peer_id="peer:8000", local_id="local:9000", - transport=transport, # type: ignore[arg-type] + transport=transport, local_block_len=4096, local_hash_seed=_DEFAULT_HASH_SEED, conn=None, ) conn = FakeConnection(peer_id="peer:8000") - session.attach_connection(conn) # type: ignore[arg-type] + session.attach_connection(conn) assert conn._sent assert conn._sent[0][TYPE_KEY] == ConnectMsg.TYPE @@ -2197,7 +2197,7 @@ class TestPendingSession: """attach_connection on an already-connected session raises.""" session, conn, _ = _make_session() with pytest.raises(ValueError, match="already connected"): - session.attach_connection(FakeConnection()) # type: ignore[arg-type] + session.attach_connection(FakeConnection()) def test_pending_close_returns_pending_stores(self): """Closing a pending session reports buffered stores as failed.""" @@ -2205,7 +2205,7 @@ class TestPendingSession: session = P2PSession( peer_id="peer:8000", local_id="local:9000", - transport=transport, # type: ignore[arg-type] + transport=transport, local_block_len=4096, local_hash_seed=_DEFAULT_HASH_SEED, conn=None, @@ -2313,7 +2313,7 @@ class TestAdversarial: def test_non_dict_message(self): session, conn, _ = _make_session() _activate(session, conn) - conn._inbox.append(42) # type: ignore[arg-type] + conn._inbox.append(42) result_ = session.poll() loads = result_.loads stores = result_.stores @@ -2402,7 +2402,7 @@ class TestDispatchErrorHandling: def _boom(*args, **kwargs): raise RuntimeError("simulated internal bug") - session._server.on_fetch = _boom # type: ignore[assignment] + session._server.on_fetch = _boom conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, @@ -2425,7 +2425,7 @@ class TestDispatchErrorHandling: def _boom(*args, **kwargs): raise RuntimeError("simulated internal bug") - session._server.on_fetch = _boom # type: ignore[assignment] + session._server.on_fetch = _boom for _ in range(_MAX_CONSECUTIVE_DISPATCH_ERRORS): conn.enqueue( { @@ -2454,7 +2454,7 @@ class TestDispatchErrorHandling: # Alternate (boom, success) (_MAX-1) times: counter rises to 1 # then resets to 0 each cycle, never reaching the threshold. for _ in range(_MAX_CONSECUTIVE_DISPATCH_ERRORS - 1): - session._server.on_fetch = _boom # type: ignore[assignment] + session._server.on_fetch = _boom conn.enqueue( { TYPE_KEY: FetchMsg.TYPE, @@ -2465,7 +2465,7 @@ class TestDispatchErrorHandling: } ) session.poll() - session._server.on_fetch = original_on_fetch # type: ignore[assignment] + session._server.on_fetch = original_on_fetch # A benign no-op message (unknown type) dispatches cleanly # and resets the consecutive-error counter. conn.enqueue({TYPE_KEY: "unknown_for_test"}) diff --git a/tests/v1/sample/test_dry.py b/tests/v1/sample/test_dry.py index f70d85eba4..50a5a5e0db 100644 --- a/tests/v1/sample/test_dry.py +++ b/tests/v1/sample/test_dry.py @@ -340,7 +340,7 @@ def _make_v2_state( device=device, all_token_ids=SimpleNamespace(gpu=all_tokens), ) - state = DryState(_make_vllm_config(), req_states) # type: ignore[arg-type] + state = DryState(_make_vllm_config(), req_states) if tokenizer is not None: # Re-resolve the default set now that a tokenizer is installed. state._tokenizer = tokenizer @@ -446,7 +446,7 @@ def test_spec_decode_is_refused_at_construction(): ) config = _make_vllm_config(speculative_config=SimpleNamespace()) with pytest.raises(ValueError, match="speculative decoding"): - DryState(config, req_states) # type: ignore[arg-type] + DryState(config, req_states) def test_v2_draft_expanded_logits_raise(): @@ -918,7 +918,7 @@ def test_loads_through_the_loader_by_fqcn(): ) procs = loader.build_custom_logits_processors( _make_vllm_config(), - req_states, # type: ignore[arg-type] + req_states, False, [DRY_FQCN], ) diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index 78524acbb5..2413c824fc 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -486,12 +486,12 @@ def test_prompt_logprob_token_ids_require_v2_model_runner(): params = SamplingParams(prompt_logprob_token_ids=[1, 2]) with patch.object(SamplingParams, "verify"): with pytest.raises(VLLMValidationError, match="V2 model runner"): - InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type] + InputProcessor._validate_params(processor, params, ("generate",)) processor.vllm_config.use_v2_model_runner = True - InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type] + InputProcessor._validate_params(processor, params, ("generate",)) processor.vllm_config.cache_config.kv_sharing_fast_prefill = True with pytest.raises(VLLMValidationError, match="fast-prefill"): - InputProcessor._validate_params(processor, params, ("generate",)) # type: ignore[arg-type] + InputProcessor._validate_params(processor, params, ("generate",)) def test_none_logprobs(vllm_model, example_prompts): diff --git a/tests/v1/sample/test_trace_replay_params.py b/tests/v1/sample/test_trace_replay_params.py index 0a70859f84..cf6e3a83f7 100644 --- a/tests/v1/sample/test_trace_replay_params.py +++ b/tests/v1/sample/test_trace_replay_params.py @@ -105,7 +105,7 @@ def test_trace_decode_token_ids_rejects_speculative_decoding(): with pytest.raises(ValueError, match="not supported with speculative decoding"): params._validate_trace_replay( _make_model_config(vocab_size=100), - speculative_config=object(), # type: ignore[arg-type] # Only non-None presence is tested. + speculative_config=object(), ) diff --git a/tests/v1/simple_kv_offload/test_worker.py b/tests/v1/simple_kv_offload/test_worker.py index 2936d0eb87..16e142d7cb 100644 --- a/tests/v1/simple_kv_offload/test_worker.py +++ b/tests/v1/simple_kv_offload/test_worker.py @@ -71,7 +71,7 @@ SLEEP_CYCLES = 50_000_000 def _make_worker( kv_cache_config: KVCacheConfig | None, cpu_capacity_bytes: int ) -> SimpleCPUOffloadWorker: - return SimpleCPUOffloadWorker(None, kv_cache_config, cpu_capacity_bytes) # type: ignore[arg-type] + return SimpleCPUOffloadWorker(None, kv_cache_config, cpu_capacity_bytes) def _make_backend() -> tuple[DmaCopyBackend, torch.Tensor, torch.Tensor]: @@ -121,7 +121,7 @@ def test_no_forward_step_completes_cpu_store(monkeypatch, use_v2): monkeypatch.setattr(f"{module}.get_forward_context", lambda: None) result = active.no_forward(output) else: - result = KVConnectorModelRunnerMixin.kv_connector_no_forward(output, None) # type: ignore[arg-type] + result = KVConnectorModelRunnerMixin.kv_connector_no_forward(output, None) completion = ( result.kv_connector_output.kv_connector_worker_meta if result.kv_connector_output is not None @@ -283,7 +283,7 @@ def test_transfer_hooks_pass_wait_event_for_store_only(): cpu_capacity_bytes=0, ) recording = _RecordingBackend() - worker._backend = recording # type: ignore[assignment] + worker._backend = recording worker._connector_metadata = SimpleCPUOffloadMetadata( load_event=0, load_gpu_blocks=[0], diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 514a84d931..73c478a104 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -27,7 +27,7 @@ def make_manager( manager.num_speculative_steps = num_steps manager._stale_confidences = [SimpleNamespace(np=confidences)] # type: ignore[list-item] # CPU-only confidence buffer. manager._stale_idx = 0 - manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture. + manager.req_states = SimpleNamespace( req_id_to_index={"low": 0, "high": 1}, num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32), prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)), @@ -153,11 +153,11 @@ def test_manager_checks_target_varlen_cudagraph_bound( manager = maybe_create_adaptive_verification_manager( enable_adaptive_verification=True, attn_groups=groups, - req_states=SimpleNamespace(num_speculative_steps=7), # type: ignore[arg-type] + req_states=SimpleNamespace(num_speculative_steps=7), query_start_loc=object(), num_bonus_tokens=1, max_total_logits=1, - vllm_config=None, # type: ignore[arg-type] + vllm_config=None, target_layer_names={"target"}, additional_attn_cg_support=additional, ) @@ -186,9 +186,7 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): def test_profiled_batches_seed_cost_curves_via_consumer(monkeypatch): manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) - manager.req_states = SimpleNamespace( # type: ignore[assignment] # CPU-only request fixture. - max_num_batched_tokens=4096, max_num_reqs=64 - ) + manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) manager.num_speculative_steps = 7 manager.num_bonus_tokens = 1 curves: dict[str, list[tuple[int, float]]] = {} diff --git a/tests/v1/spec_decode/test_dflash2.py b/tests/v1/spec_decode/test_dflash2.py index d5517ed61b..558bb38e6b 100644 --- a/tests/v1/spec_decode/test_dflash2.py +++ b/tests/v1/spec_decode/test_dflash2.py @@ -98,7 +98,7 @@ def test_draft_quant_exclusions_include_global_layer_indices(): ] ) - _add_global_draft_layer_exclusions(quant_config, 88, 5) # type: ignore[arg-type] + _add_global_draft_layer_exclusions(quant_config, 88, 5) assert "layers.88.mlp_conv*" in quant_config.exclude_modules assert "*layers.92.self_attn.q_proj" in quant_config.exclude_modules @@ -171,7 +171,7 @@ def test_selector_leaves_greedy_drafting_without_proposal_logits(monkeypatch): allocating one here would claim a proposal the walk never sampled from. """ _stub_base(monkeypatch, None) - speculator = DFlash2Speculator(None, torch.device("cpu")) # type: ignore[arg-type] # Base initialization is stubbed. + speculator = DFlash2Speculator(None, torch.device("cpu")) assert speculator.draft_logits is None @@ -183,7 +183,7 @@ def test_selector_asks_for_fp32_proposal_logits(): candidate row often enough that the walk and the rejection sampler checking it would no longer read the same distribution. """ - dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) # type: ignore[arg-type] # Constant spec ignores self/config. + dtype, fill = DFlash2Speculator.draft_logits_spec(None, None) assert dtype is torch.float32 assert fill == float("-inf") @@ -309,9 +309,9 @@ def test_candidate_model_decoder_layer_cls(monkeypatch, variant): vllm_config.compilation_config = mock_current_vllm_config.compilation_config # 3. Instantiate the model under meta device to avoid parameter allocation issues - with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): # type: ignore[arg-type] + with set_current_vllm_config(mock_current_vllm_config), torch.device("meta"): model_cls = DFlash2Qwen3Model if variant == "dflash2" else LiLiCorr - model = model_cls(vllm_config=vllm_config) # type: ignore[arg-type] + model = model_cls(vllm_config=vllm_config) # 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass) assert len(model.layers) == 2 @@ -350,7 +350,7 @@ def test_conv_projections_use_draft_quant_config(monkeypatch): quant_method="W4A16_NVFP4", is_checkpoint_nvfp4_serialized=True ) layer = DFlash2Qwen3DecoderLayer( - SimpleNamespace( # type: ignore[arg-type] + SimpleNamespace( speculative_config=SimpleNamespace(num_speculative_tokens=7), model_config=SimpleNamespace(dtype=torch.bfloat16), ), @@ -434,7 +434,7 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch): for projection in projections ] qwen3_dflash.DFlashQwen3Model._build_context_kv_buffers( - model, # type: ignore[arg-type] + model, layers_attn, has_bias=False, ) @@ -442,7 +442,7 @@ def test_context_kv_uses_quantized_projection_fallback(monkeypatch): assert all(not hasattr(projection, "weight") for projection in projections) actual_k, actual_v = qwen3_dflash.DFlashQwen3Model._project_context_kv( - model, # type: ignore[arg-type] + model, context_states, num_ctx=2, num_layers=2, diff --git a/tests/v1/spec_decode/test_dflash_lookahead.py b/tests/v1/spec_decode/test_dflash_lookahead.py index 55d039b05e..b139bfbecc 100644 --- a/tests/v1/spec_decode/test_dflash_lookahead.py +++ b/tests/v1/spec_decode/test_dflash_lookahead.py @@ -141,9 +141,9 @@ def test_dflash_drafter_window_reserves_bonus_token(): speculative_config=_dflash_speculative_config(NUM_SPECULATIVE_TOKENS), ) # window = 4, so 96 fits (96 + 4 == 100) but 97 does not (97 + 4 == 101) - assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) # type: ignore[arg-type] - assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type] - assert not input_fits_in_drafter(dflash_runner, None) # type: ignore[arg-type] # No metadata. + assert input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=96)) + assert not input_fits_in_drafter(dflash_runner, SimpleNamespace(max_seq_len=97)) + assert not input_fits_in_drafter(dflash_runner, None) # Other drafters don't reserve the bonus token, so 97 fits (97 + 3 == 100). plain_runner = SimpleNamespace( @@ -151,4 +151,4 @@ def test_dflash_drafter_window_reserves_bonus_token(): effective_drafter_max_model_len=100, speculative_config=SimpleNamespace(use_dflash=lambda: False), ) - assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) # type: ignore[arg-type] + assert input_fits_in_drafter(plain_runner, SimpleNamespace(max_seq_len=97)) diff --git a/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py b/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py index 4efe53451e..a77615feef 100644 --- a/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py +++ b/tests/v1/spec_decode/test_dsv4_mtp_local_argmax.py @@ -158,9 +158,7 @@ def test_wrapper_forwards_spec_step_idx(): hidden_states = torch.empty(0) wrapper = object.__new__(DeepSeekV4MTP) - wrapper.model = SimpleNamespace( # type: ignore[assignment] - get_top_tokens=lambda *args: args - ) + wrapper.model = SimpleNamespace(get_top_tokens=lambda *args: args) assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states, 2) == (hidden_states, 2) assert DeepSeekV4MTP.get_top_tokens(wrapper, hidden_states) == (hidden_states, 0) diff --git a/tests/v1/spec_decode/test_extract_hidden_states.py b/tests/v1/spec_decode/test_extract_hidden_states.py index 0c9197bb6a..c309ed569a 100644 --- a/tests/v1/spec_decode/test_extract_hidden_states.py +++ b/tests/v1/spec_decode/test_extract_hidden_states.py @@ -51,7 +51,7 @@ def _create_proposer( target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=num_speculative_tokens, - draft_model_config={ # type: ignore[arg-type] + draft_model_config={ "hf_config": { "eagle_aux_hidden_state_layer_ids": layer_ids, } @@ -103,7 +103,7 @@ def test_proposer_initialization_missing_layer_ids(): target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=1, - draft_model_config={ # type: ignore[arg-type] + draft_model_config={ "hf_config": {} # Missing eagle_aux_hidden_state_layer_ids }, ) @@ -363,7 +363,7 @@ def test_extract_hidden_states_text_only_config_regression(): target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=1, - draft_model_config={ # type: ignore[arg-type] + draft_model_config={ "hf_config": { "eagle_aux_hidden_state_layer_ids": [1, 2, 3, 4], } @@ -450,7 +450,7 @@ def test_extract_hidden_states_speculative_config_vlm(): target_parallel_config=ParallelConfig(), method="extract_hidden_states", num_speculative_tokens=1, - draft_model_config={ # type: ignore[arg-type] + draft_model_config={ "hf_config": { "eagle_aux_hidden_state_layer_ids": [1, 2], } diff --git a/tests/v1/spec_decode/test_lilicorr.py b/tests/v1/spec_decode/test_lilicorr.py index 88c91696f3..5a42f3826a 100644 --- a/tests/v1/spec_decode/test_lilicorr.py +++ b/tests/v1/spec_decode/test_lilicorr.py @@ -190,17 +190,17 @@ def test_runtime_length_keeps_trained_checkpoint_geometry(monkeypatch): for slots in (1, 3): spec.num_speculative_tokens = slots head = LiLiCorr( - vllm_config=SimpleNamespace(speculative_config=spec) # type: ignore[arg-type] + vllm_config=SimpleNamespace(speculative_config=spec) ).lilicorr assert head.slot_embedding.shape == (1, 1, 3, 1, 8) assert head.relative_slot_bias.shape == (2, 7) spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = False with pytest.raises(ValueError, match="lilicorr_enabled"): - LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type] + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) spec.draft_model_config.hf_config.dflash_config["lilicorr_enabled"] = True spec.num_speculative_tokens = 4 with pytest.raises(ValueError, match="trained block_size"): - LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) # type: ignore[arg-type] + LiLiCorr(vllm_config=SimpleNamespace(speculative_config=spec)) @pytest.mark.parametrize("tp_size", [1, 2]) @@ -269,8 +269,8 @@ def test_candidates_use_global_partition_and_exclude_padding(monkeypatch, tp_siz ) monkeypatch.setattr(logits_processor, "_topk", lambda x, k: x.topk(k, dim=-1)) ids, values = logits_processor.LogitsProcessor.get_top_k_tokens( - fake, # type: ignore[arg-type] - lm_head, # type: ignore[arg-type] + fake, + lm_head, torch.empty(2, 1), 2, return_log_probs=True, @@ -291,13 +291,13 @@ def test_context_anchor_is_last_committed_normalized_feature(): anchor_valid=torch.ones(4, dtype=torch.bool), ) batch = SimpleNamespace(num_reqs=2, query_start_loc=torch.tensor([0, 2, 6])) - LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) # type: ignore[arg-type] + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([0, 2])) torch.testing.assert_close(spec.anchor_hidden[:2], norm(hidden[[1, 3]])) assert spec.anchor_valid.tolist() == [True, True, False, False] assert not spec.anchor_hidden[2:].any() # A smaller reordered batch must not retain an old request's anchor. batch = SimpleNamespace(num_reqs=1, query_start_loc=torch.tensor([0, 3])) - LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) # type: ignore[arg-type] + LiLiCorrSpeculator.prepare_context_anchor(spec, batch, torch.tensor([1])) torch.testing.assert_close(spec.anchor_hidden[0], norm(hidden[1])) assert not spec.anchor_hidden[1:].any() @@ -321,7 +321,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil monkeypatch.setattr(DFlashSpeculator, "__init__", init_base) cls = LiLiCorrSpeculator if lilicorr else DFlash2Speculator - spec = cls(None, torch.device("cpu")) # type: ignore[arg-type] + spec = cls(None, torch.device("cpu")) hidden = torch.arange(36).float().view(9, 4) spec._run_model = lambda *args: hidden spec.sample_indices = torch.tensor([1, 2, 4, 5, 7, 8]) @@ -334,7 +334,7 @@ def test_candidate_generation_routes_scores_and_adaptive_inputs(monkeypatch, lil spec.draft_logits = None spec.use_fp64_gumbel = False spec.enable_adaptive_verification = True - spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) # type: ignore[assignment] + spec.input_buffers = SimpleNamespace(input_ids=torch.arange(9)) candidates = torch.arange(24).view(6, 4) log_probs = candidates.float().log_softmax(-1) scores = torch.randn(3, 2, 4, 4) @@ -636,7 +636,7 @@ def owned_head_model(monkeypatch): compilation_config=CompilationConfig(mode=CompilationMode.NONE), ) ): - model = LiLiCorrForCausalLM(vllm_config=vllm_config) # type: ignore[arg-type] + model = LiLiCorrForCausalLM(vllm_config=vllm_config) return model, vllm_config return build @@ -746,7 +746,7 @@ def test_quantized_head_calls_methods_without_reading_packed_weights(monkeypatch block_size=4, rms_norm_eps=1e-6, config=_config(hidden_size=8), - quant_config=quant_config, # type: ignore[arg-type] + quant_config=quant_config, prefix="model.lilicorr", ) with torch.no_grad(): diff --git a/tests/v1/spec_decode/test_llm_base_proposer.py b/tests/v1/spec_decode/test_llm_base_proposer.py index f9c1795b97..fb1ecdd2df 100644 --- a/tests/v1/spec_decode/test_llm_base_proposer.py +++ b/tests/v1/spec_decode/test_llm_base_proposer.py @@ -53,7 +53,7 @@ def _make_proposer( monkeypatch.setattr(llm_base_proposer, "AttentionGroup", _FakeAttentionGroup) proposer = EagleProposer.__new__(EagleProposer) - proposer.vllm_config = None # type: ignore[assignment] # Patched backend discovery does not read config. + proposer.vllm_config = None proposer.device = None proposer._draft_attn_layer_names = set(layer_names) proposer.kv_cache_gid = -1 diff --git a/tests/v1/spec_decode/test_llm_base_proposer_sampling.py b/tests/v1/spec_decode/test_llm_base_proposer_sampling.py index 7531b8b6cd..cb390b992f 100644 --- a/tests/v1/spec_decode/test_llm_base_proposer_sampling.py +++ b/tests/v1/spec_decode/test_llm_base_proposer_sampling.py @@ -85,7 +85,7 @@ def test_compute_probs_and_sample_next_token_uses_fp64_exponential_race(): def test_mtp_model_returns_tuple(architecture: str, expected: bool): proposer = object.__new__(SpecDecodeBaseProposer) proposer.method = "mtp" - proposer.draft_model_config = SimpleNamespace( # type: ignore[assignment] # Architecture-only config. + proposer.draft_model_config = SimpleNamespace( hf_config=SimpleNamespace(architectures=[architecture]) ) diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index eb95dbdc61..7fc6898142 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -700,7 +700,7 @@ def test_verify_rejects_unproposed_drafts(): first_pos = inputs["pos"].view(num_trials, K + 1)[:, 0] prefill_len = first_pos + 1 rejection_sampler = object.__new__(RejectionSampler) - rejection_sampler.sampler = SimpleNamespace( # type: ignore[assignment] + rejection_sampler.sampler = SimpleNamespace( apply_sampling_params=lambda logits, *args: logits, sampling_states=SimpleNamespace( temperature=SimpleNamespace(gpu=inputs["temperature"]), diff --git a/tests/v1/structured_output/test_backend_xgrammar_lark.py b/tests/v1/structured_output/test_backend_xgrammar_lark.py index 9293b19296..cda9acf311 100644 --- a/tests/v1/structured_output/test_backend_xgrammar_lark.py +++ b/tests/v1/structured_output/test_backend_xgrammar_lark.py @@ -48,9 +48,9 @@ def _validate( ) -> StructuredOutputsParams: params = SamplingParams(structured_outputs=StructuredOutputsParams(grammar=grammar)) params._validate_structured_outputs( - _StubModelConfig(), # type: ignore[arg-type] + _StubModelConfig(), StructuredOutputsConfig(backend=backend), - tokenizer=tokenizer or object(), # type: ignore[arg-type] + tokenizer=tokenizer or object(), ) assert params.structured_outputs is not None return params.structured_outputs diff --git a/tests/v1/structured_output/test_backend_xgrammar_plamo3.py b/tests/v1/structured_output/test_backend_xgrammar_plamo3.py index 613daa5350..a6506dafaa 100644 --- a/tests/v1/structured_output/test_backend_xgrammar_plamo3.py +++ b/tests/v1/structured_output/test_backend_xgrammar_plamo3.py @@ -51,7 +51,7 @@ def make_vllm_config() -> VllmConfig: config = VllmConfig( structured_outputs_config=StructuredOutputsConfig(backend="xgrammar"), ) - config.model_config = SimpleNamespace( # type: ignore[assignment] + config.model_config = SimpleNamespace( hf_config=SimpleNamespace(model_type="plamo3") ) return config @@ -73,7 +73,7 @@ def test_plamo3_python_tokenizer_compiles_and_accepts_structural_tag(): tokenizer = Plamo3Tokenizer(["", *call_tokens]) backend = XgrammarBackend( make_vllm_config(), - tokenizer=tokenizer, # type: ignore[arg-type] + tokenizer=tokenizer, vocab_size=len(tokenizer.get_vocab()), ) tools = [ @@ -107,7 +107,7 @@ def test_plamo3_grammar_matches_decoded_bytes(value): tokenizer = Plamo3Tokenizer(list(dict.fromkeys(["", value, *pieces]))) backend = XgrammarBackend( make_vllm_config(), - tokenizer=tokenizer, # type: ignore[arg-type] + tokenizer=tokenizer, vocab_size=len(tokenizer.get_vocab()), ) for tokens in ([value], pieces): diff --git a/tests/v1/structured_output/test_structured_output_manager.py b/tests/v1/structured_output/test_structured_output_manager.py index 15239ecb76..6082bcd6a7 100644 --- a/tests/v1/structured_output/test_structured_output_manager.py +++ b/tests/v1/structured_output/test_structured_output_manager.py @@ -108,7 +108,7 @@ def _build_harness( ) manager = StructuredOutputManager(vllm_config) if use_reasoner: - manager.reasoner_cls = MockReasoner # type: ignore[assignment] + manager.reasoner_cls = MockReasoner sampling_params = SamplingParams( structured_outputs=structured_outputs diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index 2ce81534a4..767153cac7 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -35,7 +35,7 @@ def test_structured_outputs_rejected_for_diffusion_models(): ) with pytest.raises(VLLMValidationError, match="not yet supported for diffusion"): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=True), # type: ignore[arg-type] + _StubModelConfig(is_diffusion=True), StructuredOutputsConfig(), tokenizer=None, ) @@ -45,7 +45,7 @@ def test_plain_request_allowed_for_diffusion_models(): """Requests without structured outputs are unaffected by the guard.""" params = SamplingParams() params._validate_structured_outputs( - _StubModelConfig(is_diffusion=True), # type: ignore[arg-type] + _StubModelConfig(is_diffusion=True), StructuredOutputsConfig(), tokenizer=None, ) @@ -71,9 +71,9 @@ def test_degenerate_structured_outputs_rejected(structured_outputs, match): params = SamplingParams(structured_outputs=structured_outputs) with pytest.raises(VLLMValidationError, match=match): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] + _StubModelConfig(is_diffusion=False), StructuredOutputsConfig(), - tokenizer=object(), # type: ignore[arg-type] + tokenizer=object(), ) @@ -97,9 +97,9 @@ def test_regex_with_nul_byte_rejected(regex): # (which would otherwise catch the error and fall back to another backend). with pytest.raises(VLLMValidationError, match="NUL"): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] + _StubModelConfig(is_diffusion=False), StructuredOutputsConfig(), - tokenizer=object(), # type: ignore[arg-type] + tokenizer=object(), ) # The xgrammar backend also rejects it directly (defense in depth), before @@ -133,9 +133,9 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): params = SamplingParams(structured_outputs=structured_outputs) with pytest.raises(VLLMClientError): params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] + _StubModelConfig(is_diffusion=False), StructuredOutputsConfig(backend=backend), - tokenizer=object(), # type: ignore[arg-type] + tokenizer=object(), ) @@ -180,9 +180,9 @@ def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend) """`auto` falls back on rejection, so it must catch what the validators raise.""" params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema)) params._validate_structured_outputs( - _StubModelConfig(is_diffusion=False), # type: ignore[arg-type] + _StubModelConfig(is_diffusion=False), StructuredOutputsConfig(backend="auto"), - tokenizer=object(), # type: ignore[arg-type] + tokenizer=object(), ) assert params.structured_outputs is not None assert params.structured_outputs._backend == expected_backend diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 73a0131915..2166141638 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -135,7 +135,7 @@ def test_attention_checks_preserve_global_and_target_scoped_support(): _TargetBackend, ["target"], spec, - 0, # type: ignore[arg-type] + 0, ) target_group.metadata_builders = [ _FakeMetadataBuilder(AttentionCGSupport.ALWAYS) # type: ignore[list-item] @@ -144,7 +144,7 @@ def test_attention_checks_preserve_global_and_target_scoped_support(): _DraftBackend, ["draft"], spec, - 0, # type: ignore[arg-type] + 0, ) draft_group.metadata_builders = [ _FakeMetadataBuilder(AttentionCGSupport.UNIFORM_BATCH) # type: ignore[list-item] @@ -152,14 +152,14 @@ def test_attention_checks_preserve_global_and_target_scoped_support(): groups = [[target_group, draft_group]] # The runner-wide execution mode must still honor the drafter's limit. - unfiltered = get_attn_cg_support(groups, None) # type: ignore[arg-type] + unfiltered = get_attn_cg_support(groups, None) assert unfiltered.min_cg_support == AttentionCGSupport.UNIFORM_BATCH assert unfiltered.min_cg_attn_backend == "_DraftBackend" # Adaptive verification validates only the target's varlen graphs. target_only = get_attn_cg_support( groups, - None, # type: ignore[arg-type] + None, checked_layer_names={"target"}, ) assert target_only.min_cg_support == AttentionCGSupport.ALWAYS @@ -176,7 +176,7 @@ def test_attention_checks_preserve_global_and_target_scoped_support(): draft_group.layer_names.append("target") target_with_shared_group = get_attn_cg_support( groups, - None, # type: ignore[arg-type] + None, checked_layer_names={"target"}, ) assert target_with_shared_group.min_cg_support == AttentionCGSupport.UNIFORM_BATCH diff --git a/tests/v1/worker/test_gpu_kv_connector.py b/tests/v1/worker/test_gpu_kv_connector.py index 5e56e8a317..c442c6fc59 100644 --- a/tests/v1/worker/test_gpu_kv_connector.py +++ b/tests/v1/worker/test_gpu_kv_connector.py @@ -39,9 +39,7 @@ def _make_connector( kv_role="kv_consumer", kv_buffer_device="cpu", ) - connector = ActiveKVConnector( # type: ignore[arg-type] - SimpleNamespace(kv_transfer_config=kv_config), {} - ) + connector = ActiveKVConnector(SimpleNamespace(kv_transfer_config=kv_config), {}) events.clear() return connector @@ -66,7 +64,7 @@ def test_load_start_phase( request_indices = torch.tensor([3, 1]) request_ids = ["first", "second"] attn_metadata = {"layer": object()} - connector.pre_forward( # type: ignore[arg-type] + connector.pre_forward( output, request_state_indices=request_indices, request_ids=request_ids, @@ -85,7 +83,7 @@ def test_load_start_phase( assert kwargs["attn_metadata"] is attn_metadata # A subsequent step without a forward must not reuse the prior batch. - connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type] + connector.no_forward(_scheduler_output(False)) assert connector.kv_connector.start_load_kv.call_count == 2 assert connector.kv_connector.start_load_kv.call_args.kwargs == {} @@ -94,6 +92,6 @@ def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch): events: list[str] = [] connector = _make_connector(monkeypatch, events) - connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type] + connector.no_forward(_scheduler_output(False)) assert events == ["handle", "bind", "start", "wait", "clear"] diff --git a/tests/v1/worker/test_gpu_worker_weight_transfer.py b/tests/v1/worker/test_gpu_worker_weight_transfer.py index 537653bda3..49ef707894 100644 --- a/tests/v1/worker/test_gpu_worker_weight_transfer.py +++ b/tests/v1/worker/test_gpu_worker_weight_transfer.py @@ -75,7 +75,7 @@ def _make_worker(engine: _RecordingEngine | None) -> Worker: def test_reload_weights_sets_current_config(): worker = _make_worker(None) model_runner = _RecordingModelRunner() - worker.model_runner = model_runner # type: ignore[assignment] + worker.model_runner = model_runner Worker.reload_weights(worker) diff --git a/tests/v1/worker/test_late_interaction_runner.py b/tests/v1/worker/test_late_interaction_runner.py index 5f1b7cff37..054bfbdabd 100644 --- a/tests/v1/worker/test_late_interaction_runner.py +++ b/tests/v1/worker/test_late_interaction_runner.py @@ -142,7 +142,7 @@ def test_invalid_query_uses_raises(): mode=LATE_INTERACTION_MODE_CACHE_QUERY, query_key="query-bad", ) - bad_meta.query_uses = "bad-int" # type: ignore[assignment] + bad_meta.query_uses = "bad-int" bad_query_params = _make_pooling_params(bad_meta) with pytest.raises(ValueError, match="must be an integer value"): From cff08b461e086523cee8bba5bd3c66d302625134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20P=C3=A9rez=20de=20Algaba?= <124347725+jperezdealgaba@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:32:00 +0200 Subject: [PATCH 131/248] =?UTF-8?q?[Security]=20Fix=20chat=20template=20re?= =?UTF-8?q?source-exhaustion=20DoS=20(GHSA-4hhp-h66f-=E2=80=A6=20(#50300)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: jperezde Signed-off-by: Juan Pérez de Algaba Co-authored-by: Cursor --- rust/Cargo.toml | 2 +- rust/src/chat/src/lib.rs | 7 +- rust/src/chat/src/renderer/hf/template.rs | 7 ++ tests/renderers/test_executor_replace.py | 126 +++++++++++++++++++ vllm/entrypoints/openai/responses/serving.py | 10 ++ vllm/envs.py | 6 + vllm/renderers/base.py | 29 ++++- vllm/renderers/hf.py | 52 +++++++- 8 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 tests/renderers/test_executor_replace.py diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f3d3053ed5..0002796d40 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -63,7 +63,7 @@ itertools = "0.14.0" libc = "0.2.177" llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "f0985ef65967615db2c79279aa07818499301bfd", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" -minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } +minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order", "fuel"] } minijinja-contrib = { version = "2.24", features = ["pycompat"] } native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] } ndarray = { version = "0.17", features = ["serde"] } diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 95c4332f74..86b94b6b9c 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -180,7 +180,12 @@ impl ChatRequestProcessor { async fn prepare_text_request(&self, request: ChatRequest) -> Result { // Stamp before rendering so render and tokenize count toward TTFT/e2e. let arrival_time = vllm_llm::current_unix_timestamp_secs(); - let mut rendered = self.backend.chat_renderer().render(&request)?; + let renderer = self.backend.chat_renderer(); + let (mut rendered, request) = tokio::task::spawn_blocking(move || { + renderer.render(&request).map(|rendered| (rendered, request)) + }) + .await + .map_err(|e| Error::ChatTemplate(format!("render task join error: {e}")))??; let chat_template_kwargs = std::mem::take(&mut rendered.effective_template_kwargs); let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?; Ok(TextRequest { diff --git a/rust/src/chat/src/renderer/hf/template.rs b/rust/src/chat/src/renderer/hf/template.rs index efcfd420ed..3a496d957d 100644 --- a/rust/src/chat/src/renderer/hf/template.rs +++ b/rust/src/chat/src/renderer/hf/template.rs @@ -29,6 +29,11 @@ use crate::renderer::hf::{TemplateMessage, TemplateTool}; type Result = std::result::Result; +/// Default fuel budget for template rendering. Prevents unbounded loops and +/// string operations from exhausting CPU. Each template instruction consumes +/// one unit of fuel; when depleted, rendering aborts with an error. +const DEFAULT_TEMPLATE_FUEL: u64 = 500_000; + /// Build a pre-configured environment with the given template string. fn build_environment(template: String) -> Result> { let mut env = Environment::new(); @@ -36,6 +41,8 @@ fn build_environment(template: String) -> Result> { env.set_trim_blocks(true); env.set_lstrip_blocks(true); + env.set_fuel(Some(DEFAULT_TEMPLATE_FUEL)); + env.add_template_owned("chat".to_owned(), template)?; env.set_unknown_method_callback(minijinja_contrib::pycompat::unknown_method_callback); diff --git a/tests/renderers/test_executor_replace.py b/tests/renderers/test_executor_replace.py new file mode 100644 index 0000000000..fc0175e1e2 --- /dev/null +++ b/tests/renderers/test_executor_replace.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +from dataclasses import dataclass +from typing import Any + +import pytest +import torch + +from vllm.renderers.base import _SwappableExecutor +from vllm.renderers.hf import HfRenderer +from vllm.renderers.params import TokenizeParams +from vllm.utils.async_utils import make_async + +MODEL_NAME = "openai-community/gpt2" + + +@dataclass +class MockHFConfig: + model_type: str = "any" + + +@dataclass +class MockModelConfig: + runner_type = "generate" + model: str = MODEL_NAME + tokenizer: str = MODEL_NAME + trust_remote_code: bool = False + tokenizer_revision = None + tokenizer_mode = "auto" + hf_config = MockHFConfig() + encoder_config: dict[str, Any] | None = None + enable_prompt_embeds: bool = False + skip_tokenizer_init: bool = False + is_encoder_decoder: bool = False + is_multimodal_model: bool = False + renderer_num_workers: int = 1 + hidden_size: int = 768 + dtype: torch.dtype = torch.float32 + + def get_hidden_size(self) -> int: + return self.hidden_size + + +@dataclass +class MockParallelConfig: + _api_process_rank: int = 0 + + +@dataclass +class MockVllmConfig: + model_config: MockModelConfig + parallel_config: MockParallelConfig + + +@dataclass +class DummyTokenizer: + truncation_side: str = "left" + max_chars_per_token: int = 1 + is_fast: bool = False + + def decode(self, tokens: list[int], **kwargs): + return str(tokens) + + def encode(self, text: str, **kwargs): + return list(range(len(text))) + + def __call__(self, text: str, **kwargs): + return {"input_ids": self.encode(text, **kwargs)} + + +def _build_renderer() -> HfRenderer: + return HfRenderer( + MockVllmConfig(MockModelConfig(), parallel_config=MockParallelConfig()), + tokenizer=DummyTokenizer(), + ) + + +def test_swappable_executor_keeps_make_async_wrappers_alive(): + pool = _SwappableExecutor(max_workers=1) + async_add = make_async(lambda x: x + 1, executor=pool) + + async def _run(): + assert await async_add(1) == 2 + old_inner = pool._inner + pool.replace_inner() + + with pytest.raises(RuntimeError, match="cannot schedule new futures"): + old_inner.submit(lambda: None) + + assert await async_add(40) == 41 + + try: + asyncio.run(_run()) + finally: + pool.shutdown(wait=False) + + +def test_replace_executor_does_not_break_tokenize_or_decode(): + renderer = _build_renderer() + executor = renderer._executor + old_inner = executor._inner + + async def _run(): + assert await renderer._tokenize_prompt_async( + {"prompt": "ab"}, + TokenizeParams(max_total_tokens=100), + ) + renderer._replace_executor() + assert renderer._executor is executor + assert executor._inner is not old_inner + + with pytest.raises(RuntimeError, match="cannot schedule new futures"): + old_inner.submit(lambda: None) + + tokenized = await renderer._tokenize_prompt_async( + {"prompt": "abc"}, + TokenizeParams(max_total_tokens=100), + ) + assert tokenized["prompt_token_ids"] == [0, 1, 2] + assert await renderer._async_tokenizer_decode([1, 2]) == "[1, 2]" + + try: + asyncio.run(_run()) + finally: + renderer.shutdown() diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 34f6fb44a2..66e0f2640d 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -334,6 +334,16 @@ class OpenAIServingResponses(GenerateBaseServing): if maybe_validation_error is not None: return maybe_validation_error + maybe_template_error = self.online_renderer.validate_chat_template( + request_chat_template=None, + chat_template_kwargs=request.chat_template_kwargs, + trust_request_chat_template=( + self.online_renderer.trust_request_chat_template + ), + ) + if maybe_template_error is not None: + return maybe_template_error + self._preflight() if request.store and not self.enable_store: diff --git a/vllm/envs.py b/vllm/envs.py index 67e0207492..5daa4fd3e6 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: CUDA_VISIBLE_DEVICES: str | None = None VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60 VLLM_ENGINE_READY_TIMEOUT_S: int = 600 + VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: float = 30.0 VLLM_API_KEY: str | None = None VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False S3_ACCESS_KEY_ID: str | None = None @@ -812,6 +813,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ENGINE_READY_TIMEOUT_S": lambda: int( os.environ.get("VLLM_ENGINE_READY_TIMEOUT_S", "600") ), + # Maximum wall-clock seconds allowed for a single chat template render. + # Set to 0 to disable the timeout. + "VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT": lambda: float( + os.environ.get("VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT", "30") + ), # API key for vLLM API server "VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None), # Whether to log responses from API Server for debugging diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 87d11b7b92..0936dd021e 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -5,7 +5,7 @@ import time import weakref from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence -from concurrent.futures import Future, ThreadPoolExecutor +from concurrent.futures import Executor, Future, ThreadPoolExecutor from contextlib import ExitStack from dataclasses import replace from functools import cached_property @@ -77,6 +77,31 @@ logger = init_logger(__name__) _T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike) +class _SwappableExecutor(Executor): + """Executor whose inner pool can be replaced without changing identity. + + ``make_async`` captures the executor object at wrap time. Replacing + ``self._executor`` with a new ``ThreadPoolExecutor`` would leave those + wrappers (tokenize, decode, pooling, derender) bound to a shutdown pool. + """ + + def __init__(self, max_workers: int) -> None: + super().__init__() + self._max_workers = max_workers + self._inner = ThreadPoolExecutor(max_workers=max_workers) + + def submit(self, fn, /, *args, **kwargs): + return self._inner.submit(fn, *args, **kwargs) + + def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None: + self._inner.shutdown(wait=wait, cancel_futures=cancel_futures) + + def replace_inner(self) -> None: + old = self._inner + old.shutdown(wait=False) + self._inner = ThreadPoolExecutor(max_workers=self._max_workers) + + class BaseRenderer(ABC, Generic[_T]): def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None: super().__init__() @@ -115,7 +140,7 @@ class BaseRenderer(ABC, Generic[_T]): # multimodal processor receives a deep-copied tokenizer (see #36557) # so it is safe to run tokenization and MM preprocessing concurrently. pool_workers = config.model_config.renderer_num_workers - self._executor = ThreadPoolExecutor(max_workers=pool_workers) + self._executor = _SwappableExecutor(max_workers=pool_workers) self._resources.callback(self._executor.shutdown, wait=False) # Separate single-worker executor so tokenization never queues behind diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 11f562f74f..4301c69176 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from __future__ import annotations +import asyncio import copy import inspect import itertools @@ -20,6 +21,7 @@ import jinja2.sandbox import torch from typing_extensions import override +from vllm import envs from vllm.entrypoints.chat_utils import ( PROMPT_EMBEDS_PLACEHOLDER_TOKEN, ChatTemplateResolutionError, @@ -969,6 +971,44 @@ class HfRenderer(BaseRenderer[HfTokenizer]): self.tokenizer, config.model_config.renderer_num_workers + 1 ) + def _replace_executor(self) -> None: + """Replace the inner render pool after a stuck worker. + + Called after a render timeout so that the stuck thread (which CPython + cannot interrupt) does not permanently block all subsequent renders. + The executor *object* is kept so ``make_async`` wrappers bound at + init (tokenize, decode, pooling, derender) keep submitting work. + The old inner pool is abandoned with wait=False — its thread will + eventually finish or be reaped on process exit. + """ + self._executor.replace_inner() + logger.warning( + "Chat template render timed out — executor pool replaced " + "(%d worker(s)). The stuck thread may continue consuming " + "CPU until the process exits.", + self.model_config.renderer_num_workers, + ) + + async def _render_with_timeout(self, coro): + """Run a template-rendering coroutine with a timeout. + + On timeout, replaces the executor so subsequent renders are not + permanently blocked by the stuck thread. + """ + timeout = envs.VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT + if timeout <= 0: + return await coro + try: + return await asyncio.wait_for(coro, timeout=timeout) + except asyncio.TimeoutError: + self._replace_executor() + raise TimeoutError( + f"Chat template rendering timed out after " + f"{timeout}s. This may indicate a malicious or " + f"excessively complex template. Adjust " + f"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT to override." + ) from None + def _can_produce_offsets(self) -> bool: # Only fast tokenizers expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast @@ -1124,11 +1164,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]): logger.warning_once(_TOKENIZE_OVERRIDE_WARNING) chat_template_kwargs["tokenize"] = True - prompt_raw = await self._apply_chat_template_async( - model_config, - tokenizer, - conversation, - **chat_template_kwargs, + prompt_raw = await self._render_with_timeout( + self._apply_chat_template_async( + model_config, + tokenizer, + conversation, + **chat_template_kwargs, + ) ) # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5 From c4df37dfcc7f72ca41cd43801a468526ecf99a2b Mon Sep 17 00:00:00 2001 From: rasmith Date: Wed, 30 Sep 2026 10:51:40 -0500 Subject: [PATCH 132/248] [ROCm][BugFix][The Rock] Fix mori build for the rock (#59372) Signed-off-by: Randall Smith Co-authored-by: Misha Goin --- docker/Dockerfile.rock_base | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index 54025dd508..b2982e43e0 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -207,8 +207,9 @@ RUN cd mori \ && git checkout ${MORI_BRANCH} \ && git submodule update --init --recursive \ && ln -sfn ${ROCM_PATH} /opt/rocm \ - && mkdir -p /usr/lib64 && ln -sf /usr/lib/x86_64-linux-gnu/libc.so /usr/lib64/libc.so \ - && python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl + && pip install "setuptools_scm>=6.2" \ + && CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \ + python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install ### From c32513b99d29b3c6efee38df752a696bc2696c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eldar=20Kurti=C4=87?= <8884008+eldarkurtic@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:08:31 +0200 Subject: [PATCH 133/248] Add support for unquantized ngram in CT format (#59431) --- vllm/models/qwen4_exp/common/ngram_embedding.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm/models/qwen4_exp/common/ngram_embedding.py b/vllm/models/qwen4_exp/common/ngram_embedding.py index e7371770f8..9e6eef03df 100644 --- a/vllm/models/qwen4_exp/common/ngram_embedding.py +++ b/vllm/models/qwen4_exp/common/ngram_embedding.py @@ -23,6 +23,9 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501 + CompressedTensorsConfig, +) from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.modelopt import ( ModelOptMixedPrecisionConfig, @@ -182,6 +185,11 @@ class Qwen4ExpPLEEmbeddingMethod(QuantizeMethodBase): quant_config, ModelOptQuantConfigBase ) and quant_config.is_layer_excluded(prefix): return Qwen4ExpPLEUnquantizedEmbeddingMethod() + if ( + isinstance(quant_config, CompressedTensorsConfig) + and quant_config.get_scheme_dict(None, layer_name=prefix) is None + ): + return Qwen4ExpPLEUnquantizedEmbeddingMethod() if not isinstance(quant_config, Fp8Config): raise NotImplementedError( "Qwen4Exp PLE embedding does not support quantization config " From 279012f94dac8a569c557b0fe86420099470650d Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:12:12 -0400 Subject: [PATCH 134/248] [Docs] Add @wzhao18 to NVIDIA integration and kv offloading code owners (#59456) Signed-off-by: wzhao18 Co-authored-by: Codex --- .github/CODEOWNERS | 8 ++++---- docs/governance/committers.md | 5 +++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 256707fc9a..55acde281a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,10 +3,10 @@ # This lists cover the "core" components of vLLM that require careful review /vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng -/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium +/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium @wzhao18 /vllm/lora @jeejeelee /vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni -/vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye +/vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye @wzhao18 /vllm/model_executor/layers/quantization @mgoin @robertgshaw2-redhat @tlrmchlsmth @yewentao256 @pavanimajety @zyongye /vllm/model_executor/layers/mamba @tdoublep @tomeras91 /vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy @@ -67,7 +67,7 @@ /vllm/v1/attention @LucasWilkinson @MatthewBonanni /vllm/v1/attention/backend.py @WoosukKwon @zhuohan123 @youkaichao @alexm-redhat @njhill /vllm/v1/attention/backends/mla @pavanimajety -/vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy +/vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy @wzhao18 /vllm/v1/attention/backends/triton_attn.py @tdoublep /vllm/v1/attention/backends/gdn_attn.py @ZJY0516 @vadiklyutiy /vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium @@ -113,7 +113,7 @@ /tests/lora @jeejeelee /tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91 /tests/v1/kv_connector/nixl_integration @NickLucche -/tests/v1/kv_connector @ApostaC @orozery @ivanium +/tests/v1/kv_connector @ApostaC @orozery @ivanium @wzhao18 /tests/v1/kv_offload @ApostaC @orozery /tests/v1/simple_kv_offload @ivanium /tests/v1/determinism @yewentao256 diff --git a/docs/governance/committers.md b/docs/governance/committers.md index 48ed4b2607..9613534538 100644 --- a/docs/governance/committers.md +++ b/docs/governance/committers.md @@ -54,6 +54,7 @@ Sorted alphabetically by GitHub handle: - [@tomeras91](https://github.com/tomeras91): MoE, Mamba, Ray - [@vadiklyutiy](https://github.com/vadiklyutiy): CI, performance, kernels - [@WoosukKwon](https://github.com/WoosukKwon): Project lead, engine core +- [@wzhao18](https://github.com/wzhao18): Kernels and performance, KV offloading - [@xinli-sw](https://github.com/xinli-sw): Quantization and model support - [@xuechendi](https://github.com/xuechendi): Intel CPU/XPU integration, KV connector - [@yaochengji](https://github.com/yaochengji): TPU integration @@ -173,7 +174,7 @@ If you have PRs touching the area, please feel free to ping the area owner for r ### External Kernels Integration - FlashAttention: @LucasWilkinson, @MatthewBonanni -- FlashInfer: @LucasWilkinson, @mgoin, @WoosukKwon, @MatthewBonanni +- FlashInfer: @LucasWilkinson, @mgoin, @WoosukKwon, @MatthewBonanni, @wzhao18 - Blackwell Kernels: @mgoin, @yewentao256 - DeepEP/DeepGEMM: @mgoin, @yewentao256 @@ -186,7 +187,7 @@ If you have PRs touching the area, please feel free to ping the area owner for r ### Hardware - Plugin Interface: @youkaichao, @Yikun -- NVIDIA GPU: @pavanimajety +- NVIDIA GPU: @pavanimajety, @wzhao18 - AMD GPU: @gshtras, @tjtanaa, @hongxiayang, @shen-shanshan, @AndreasKaratzas - Intel CPU/GPU: @jikunshang, @bigPYJ1151, @xuechendi - Google TPU: @yaochengji From ff1b87cca25690fef6bd12667fd0d26690d949af Mon Sep 17 00:00:00 2001 From: "Yueh-Ting (eop) Chen" Date: Thu, 1 Oct 2026 00:14:51 +0800 Subject: [PATCH 135/248] [Bugfix][HiSparse] Preserve host prefix publication after request completion (#59007) Signed-off-by: Yueh-Ting Chen Co-authored-by: Matthew Bonanni --- tests/v1/core/test_prefix_caching.py | 75 ++++++++++++++++++++ tests/v1/e2e/general/test_hisparse.py | 99 +++++++++++++++++++++++++++ vllm/v1/hisparse/coordinator.py | 49 +++++++++++++ 3 files changed, 223 insertions(+) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index c8d42665f0..fab623eb84 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1178,6 +1178,81 @@ def test_hisparse_finished_request_leaves_free_copies(): assert not coordinator.spills_to_send +@pytest.mark.parametrize( + "status,expected_cached_pages", + [ + (RequestStatus.FINISHED_STOPPED, 2), + (RequestStatus.FINISHED_ABORTED, 2), + (RequestStatus.RUNNING, 0), + ], + ids=["stopped", "aborted", "running"], +) +def test_hisparse_cleanup_publishes_only_finalized_terminal_pages( + status, expected_cached_pages +): + """Only terminal cleanup may publish completed, finalized pages.""" + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + coordinator = get_hisparse_coordinator(manager) + request = make_request("terminal", list(range(33)), HISPARSE_BLOCK_SIZE, sha256) + request.spec_token_ids = list(range(33, 49)) + assert manager.allocate_slots(request, 49, num_lookahead_tokens=16) is not None + host_blocks = list(manager.get_blocks(request.request_id).blocks[0]) + command = coordinator.build_offload_command() + counts = {transfer.transfer_id: 1 for transfer in command.page_transfers} + assert len(counts) == 2 + request.status = status + manager.free(request) + coordinator.update_spills(counts, counts) + assert all( + block.block_hash is not None for block in host_blocks[:expected_cached_pages] + ) + assert all( + block.block_hash is None for block in host_blocks[expected_cached_pages:] + ) + assert all(block.ref_cnt == 0 for block in host_blocks) + assert not coordinator.has_pending_work() + + +def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity(): + """Old completions retain their pages through pressure and request-ID reuse.""" + manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True) + coordinator = get_hisparse_coordinator(manager) + host_pool = coordinator.get_host_block_pool() + tokens = list(range(2 * HISPARSE_BLOCK_SIZE + 1)) + request = make_request("reused", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert manager.allocate_slots(request, len(tokens)) is not None + old_blocks = list(manager.get_blocks(request.request_id).blocks[0]) + command = coordinator.build_offload_command() + counts = {transfer.transfer_id: 2 for transfer in command.page_transfers} + partial = dict.fromkeys(counts, 1) + first_completions = partial.copy() + first_completions[next(iter(counts))] = 2 + coordinator.update_spills(counts, first_completions) + request.status = RequestStatus.FINISHED_LENGTH_CAPPED + manager.free(request) + replacement = make_request( + "reused", list(range(100, 117)), HISPARSE_BLOCK_SIZE, sha256 + ) + assert manager.allocate_slots(replacement, 17) is not None + new_blocks = list(manager.get_blocks(replacement.request_id).blocks[0]) + pressure = host_pool.get_new_blocks(host_pool.get_num_free_blocks()) + assert not {block.block_id for block in old_blocks[:2]} & { + block.block_id for block in pressure + new_blocks + } + assert all(block.block_hash is None for block in old_blocks) + coordinator.update_spills({}, partial) + repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE + assert all(block.block_hash is None for block in new_blocks) + assert all(block.ref_cnt == 0 for block in old_blocks[:2]) + coordinator.update_spills({}, partial) + assert all(block.ref_cnt == 0 for block in old_blocks[:2]) + host_pool.free_blocks(pressure) + manager.free(replacement) + _publish_hisparse_pages(manager) + assert not coordinator.has_pending_work() + + def test_hisparse_reset_prefix_cache_drops_copies(): """Reset must forget GPU copies along with the host hashes they mirror.""" manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) diff --git a/tests/v1/e2e/general/test_hisparse.py b/tests/v1/e2e/general/test_hisparse.py index 21f28d0b6b..573b424ea7 100644 --- a/tests/v1/e2e/general/test_hisparse.py +++ b/tests/v1/e2e/general/test_hisparse.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time + import prometheus_client import pytest import torch @@ -18,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import ( MultiConnector, ) from vllm.platforms import current_platform +from vllm.sampling_params import SamplingParams from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.v1.hisparse.coordinator import get_hisparse_coordinator @@ -301,3 +304,99 @@ def test_hisparse_host_exhaustion_defers_requests( assert checked_steps > 0 assert refusals > 0 assert actual == expected + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA" +) +@fork_new_process_for_each_test +def test_hisparse_terminal_prefix_reuse( + monkeypatch: pytest.MonkeyPatch, + vllm_runner: type[VllmRunner], +): + """Finished requests publish mirrored host KV after a late acknowledgement.""" + capability = current_platform.get_device_capability() + if capability is None or capability.major < 9: + pytest.skip("Sparse MLA requires Hopper or newer") + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip") + target = [1000 + i % 64 for i in range(257)] + sampling = SamplingParams(temperature=0, max_tokens=1, ignore_eos=True) + with vllm_runner( + MODEL, + load_format="dummy", + hf_overrides=_shrink_config, + attention_config=AttentionConfig( + hisparse_config=HiSparseConfig( + device_buffer_size=512, eager_host_mirror=True + ) + ), + kv_transfer_config=KVTransferConfig( + kv_connector="HiSparseConnector", + kv_role="kv_both", + kv_connector_extra_config={"host_pool_gib": 0.01}, + ), + block_size=64, + max_model_len=320, + max_num_batched_tokens=512, + max_num_seqs=2, + num_gpu_blocks_override=128, + enable_prefix_caching=True, + enable_chunked_prefill=True, + enforce_eager=True, + ) as runner: + engine = runner.llm.llm_engine.engine_core.engine_core + coordinator = get_hisparse_coordinator(engine.scheduler.kv_cache_manager) + worker = _get_hisparse_worker(runner) + original_updates = worker.take_transfer_updates + original_free = coordinator.free + held_completions: list[int] = [] + update_calls = 0 + terminal_pending_pages = 0 + + def take_updates(): + nonlocal update_calls + with gpu_sync_allowed(): + torch.accelerator.synchronize() + enqueued, completed = original_updates() + held_completions.extend(completed) + update_calls += 1 + if update_calls <= 2: + return enqueued, [] + completed = held_completions.copy() + held_completions.clear() + return enqueued, completed + + def free(request_id): + nonlocal terminal_pending_pages + state = coordinator.request_states.get(request_id) + if state is not None: + terminal_pending_pages += len(state.pending_pages) + original_free(request_id) + + def drain_pending_work(): + deadline = time.monotonic() + 10 + while coordinator.has_pending_work(): + assert time.monotonic() < deadline, "HiSparse transfers did not drain" + runner.llm.llm_engine.step() + + monkeypatch.setattr(worker, "take_transfer_updates", take_updates) + monkeypatch.setattr(coordinator, "free", free) + [first] = runner.llm.generate( + [{"prompt_token_ids": target}], sampling, use_tqdm=False + ) + assert terminal_pending_pages == 4 + assert first.num_cached_tokens == 0 + assert coordinator.has_pending_work() + drain_pending_work() + assert not coordinator.has_pending_work() + assert not held_completions + assert update_calls > 2 + + [repeated] = runner.llm.generate( + [{"prompt_token_ids": target}], sampling, use_tqdm=False + ) + assert repeated.num_cached_tokens == 256 + assert repeated.outputs[0].token_ids == first.outputs[0].token_ids + drain_pending_work() + assert not coordinator.has_pending_work() diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index 0ca0d40c88..26357e0a2a 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -44,6 +44,8 @@ class _PendingPublication: num_pages: int retention_interval: int | None replay_boundaries: Sequence[int] + detached_blocks: list[KVCacheBlock] | None = None + num_cached_blocks: int = 0 @dataclass @@ -724,6 +726,10 @@ class HiSparseCoordinator: if pending.restore: self._record_copy(request_id, page_idx, pending.host_block) completed_request_ids.add(request_id) + elif pending.request_state.publication is not None: + if not pending.restore: + pending.request_state.valid_pages.add(page_idx) + self._publish_detached_blocks(pending.request_state) assert self.host_manager is not None self.host_manager.block_pool.free_blocks([pending.host_block]) for request_id in completed_request_ids: @@ -732,12 +738,55 @@ class HiSparseCoordinator: state.ready_prefix_pages += 1 self._publish_host_blocks_if_ready(request_id) + def _publish_detached_blocks(self, state: _HiSparseRequestState) -> None: + publication = state.publication + if publication is None or state.pending_pages: + return + blocks = publication.detached_blocks + if blocks is None: + return + assert self.host_manager is not None + while state.ready_prefix_pages in state.valid_pages: + state.ready_prefix_pages += 1 + # The host group uses full-attention retention. Only sealed pages with + # completed copies can outlive the request, including on cancellation. + self.host_manager.block_pool.cache_full_blocks( + request=publication.request, + blocks=blocks, + num_cached_blocks=publication.num_cached_blocks, + num_full_blocks=min(state.ready_prefix_pages, publication.num_pages), + block_size=self.host_manager.block_size, + kv_cache_group_id=self.host_manager.kv_cache_group_id, + ) + self.host_manager.block_pool.free_blocks(reversed(blocks)) + state.publication = None + def free(self, request_id: str) -> None: """Detach the request; its clean pages stay readable copies in the pool.""" self._pending_imports.pop(request_id, None) state = self.request_states.pop(request_id, None) if state is None: return + publication = state.publication + if ( + publication is not None + and publication.request.is_finished() + and state.pending_pages + ): + assert self.host_manager is not None + publication.detached_blocks = self.host_manager.req_to_blocks[request_id][ + : publication.num_pages + ] + publication.num_cached_blocks = self.host_manager.num_cached_block.get( + request_id, 0 + ) + # Completed pages need leases too: another allocation may evict + # them before the remaining copies finish and the prefix publishes. + self.host_manager.block_pool.touch(publication.detached_blocks) + else: + # Preemption can cancel a scheduled forward after its spills were + # planned, so its copies do not prove that the KV was computed. + state.publication = None for manager in self.resident_managers: blocks = manager.req_to_blocks.get(request_id) if not blocks: From 42a3dd5fec6f1ee9c7f84af7700fdf544aa4ca9c Mon Sep 17 00:00:00 2001 From: rasmith Date: Wed, 30 Sep 2026 11:15:03 -0500 Subject: [PATCH 136/248] [ROcm][BugFix][The Rock] Update The Rock dockerfile to most recent Triton 3.8 (#59287) Signed-off-by: Randall Smith --- docker/Dockerfile.rock_base | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base index b2982e43e0..3ee203a347 100644 --- a/docker/Dockerfile.rock_base +++ b/docker/Dockerfile.rock_base @@ -3,8 +3,8 @@ ARG PYTHON_VERSION=3.12 ARG TORCH_VERSION=2.12.0+rocm10.0.0 ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0 ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0 -ARG TRITON_BRANCH="c0a142ffcbb42f9a67a56bf791cad472be1d6425" # release/internal/3.8.x as of 9/21/2026 -ARG TRITON_VERSION=3.8.0+gitc0a142ff +ARG TRITON_BRANCH="669b31ac" # release/internal/3.8.x as of 09/29/2026 +ARG TRITON_VERSION=3.8.0+git669b31ac ARG TRITON_REPO="https://github.com/ROCm/triton.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" From 73a5831127a9d2b87102da8a6e7c96b6f7f64fcd Mon Sep 17 00:00:00 2001 From: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:21:02 -0400 Subject: [PATCH 137/248] [Frontend] Port chat_parsing core from Transformers (#58602) Signed-off-by: Yoni Gozlan Co-authored-by: Yoni Gozlan Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/parser/test_chat_parsing.py | 1662 +++++++++++++++++ vllm/parser/chat_parsing/__init__.py | 26 + vllm/parser/chat_parsing/content_parsers.py | 255 +++ vllm/parser/chat_parsing/response_parser.py | 408 ++++ .../parser/chat_parsing/response_templates.py | 249 +++ 5 files changed, 2600 insertions(+) create mode 100644 tests/parser/test_chat_parsing.py create mode 100644 vllm/parser/chat_parsing/__init__.py create mode 100644 vllm/parser/chat_parsing/content_parsers.py create mode 100644 vllm/parser/chat_parsing/response_parser.py create mode 100644 vllm/parser/chat_parsing/response_templates.py diff --git a/tests/parser/test_chat_parsing.py b/tests/parser/test_chat_parsing.py new file mode 100644 index 0000000000..26336a540a --- /dev/null +++ b/tests/parser/test_chat_parsing.py @@ -0,0 +1,1662 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright 2026 The HuggingFace Team. All rights reserved. +# ruff: noqa +# fmt: off +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for the new declarative response_template parser. + +All six real-model template fixtures from the legacy test suite are re-expressed +here in the new region-spec shape and asserted against the same expected +output dicts. Any divergence indicates a regression in the new executor.""" + +import copy +import random +import unittest +from typing import Any + +from vllm.parser.chat_parsing import ResponseParser, parse_response +from vllm.parser.chat_parsing.response_parser import _coerce, _schema_types +from vllm.parser.chat_parsing.response_templates import load_response_template + + +cohere_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "fields": { + "content": { + "open": "<|START_RESPONSE|>", + "close": "<|END_RESPONSE|>", + "content": "text", + }, + "thinking": { + "open": "<|START_THINKING|>", + "close": "<|END_THINKING|>", + "content": "text", + }, + "tool_calls": { + "open": "<|START_ACTION|>", + "close": "<|END_ACTION|>", + "content": "json", + "transform_each": True, + "transform": {"type": "function", "function": {"name": "{tool_name}", "arguments": "{parameters}"}}, + }, + }, +} + + +ernie_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "Assistant:", + "fields": { + "thinking": { + "open_pattern": r"(?:^|\s*)", + "close": "", + "content": "text", + }, + "content": { + "open": "\n", + "close_pattern": r"\n?", + "content": "text", + }, + "tool_calls": { + "open": "", + "close": "", + "repeats": True, + "content": "json", + "transform": {"type": "function", "function": "{content}"}, + }, + }, +} + + +gpt_oss_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|start|>assistant", + "fields": { + "thinking": { + "open": "<|channel|>analysis<|message|>", + "close": "<|end|>", + "content": "text", + }, + "content": { + "open": "<|channel|>final<|message|>", + "close": "<|end|>", + "content": "text", + }, + "tool_calls": { + "open_pattern": r"<\|channel\|>commentary to=functions\.(?P\w+).*?<\|message\|>", + "close": "<|call|>", + "repeats": True, + "content": "json", + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, +} + + +smollm_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "tool_calls": { + "open": "", + "close": "", + "repeats": True, + "content": "json", + "transform": {"type": "function", "function": "{content}"}, + }, + "content": { + "close": "<|im_end|>", + "content": "text", + }, + }, +} + + +qwen3_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "tool_calls": { + "open_pattern": r"\s*\w+)>", + "close": "", + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "value_parser": {"name": "json", "args": {"allow_non_json": True}}, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, +} + + +gemma4_template = { + "defaults": {"role": "assistant"}, + # The chat template only emits `<|turn>model\n` when the previous message wasn't a tool_call/ + # tool_response. After a tool_response the prefix just ends with `` and the + # model continues from there, so we accept either anchor and truncate past the latest one. + "start_anchor": ["<|turn>model\n", ""], + "fields": { + "thinking": { + "open": "<|channel>thought\n", + "close": "", + "content": "text", + }, + "tool_calls": { + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "close": "", + "repeats": True, + "content": "json", + "content_args": { + "unquoted_keys": True, + "string_delims": [['<|"|>', '<|"|>']], + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + "content": { + "close": ["", "<|tool_response>", ""], + "content": "text", + }, + }, +} + + +# Inkling (TMLv0) frames every block as <|message_model|>[author-name]<|content_KIND|>body<|end_message|>; +# the generation prompt pre-writes the first <|message_model|>, so each open treats the header as optional. +inkling_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|message_model|>", + "fields": { + "thinking": { + "open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_thinking\|>", + "close": "<|end_message|>", + "repeats": True, + "join": "", + "content_args": {"strip": False}, + }, + "content": { + "open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_text\|>", + "close": "<|end_message|>", + "repeats": True, + "join": "", + "content_args": {"strip": False}, + }, + "tool_calls": { + "open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_invoke_tool_json\|>", + "close": "<|end_message|>", + "repeats": True, + "content": "json", + "transform": {"type": "function", "function": {"name": "{content.name}", "arguments": "{content.args}"}}, + }, + }, +} + + +class ChatResponseTemplateParserTest(unittest.TestCase): + + def test_cohere(self): + model_out = ( + "<|START_THINKING|>I should call a tool.<|END_THINKING|>" + '<|START_ACTION|>[\n {"tool_call_id": "0", "tool_name": "simple_tool", ' + '"parameters": {"temperature_format": "Celsius"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>' + ) + self.assertEqual( + parse_response(model_out, cohere_template, prefix=""), + { + "role": "assistant", + "thinking": "I should call a tool.", + "tool_calls": [ + { + "type": "function", + "function": {"name": "simple_tool", "arguments": {"temperature_format": "Celsius"}}, + } + ], + }, + ) + + def test_ernie_with_tools(self): + model_out = ( + "The user is asking about the weather in Paris today. Let me check the available tools. " + "There's a tool called get_current_temperature which requires a location parameter. Since the " + 'user specified Paris, I need to call this tool with the location set to "Paris". I should ' + "make sure the argument is correctly formatted as a string. No other tools are available, so " + "this is the right one to use. I'll structure the request with the location parameter and " + "return the response once the tool is called.\n" + "\n\n" + '\n{"name": "get_current_temperature", "arguments": {"location": "Paris"}}\n\n' + ) + self.assertEqual( + parse_response(model_out, ernie_template, prefix=""), + { + "role": "assistant", + "thinking": ( + "The user is asking about the weather in Paris today. Let me check the available tools. " + "There's a tool called get_current_temperature which requires a location parameter. Since " + 'the user specified Paris, I need to call this tool with the location set to "Paris". I ' + "should make sure the argument is correctly formatted as a string. No other tools are " + "available, so this is the right one to use. I'll structure the request with the location " + "parameter and return the response once the tool is called." + ), + "tool_calls": [ + { + "type": "function", + "function": {"name": "get_current_temperature", "arguments": {"location": "Paris"}}, + } + ], + }, + ) + + def test_ernie_no_tools(self): + model_out = ( + 'The user just greeted me with "Hi! How are you?" I need to respond in a friendly and helpful ' + "manner. Let me start by acknowledging their greeting. I should ask them how they're doing to " + "engage in conversation.\n\n" + "First, I'll say hello back and then ask how they're feeling. It's important to show genuine " + "interest. Maybe mention that I'm here to help with anything they need. Keep the tone warm and " + "positive. Let me make sure the response is concise but friendly. Alright, that should work.\n" + "\n\n" + "\nHello! I'm doing well, thank you for asking. How about you? Is there something " + "specific you'd like help with today? I'm here to assist you with any questions or problems you " + "have!\n\n" + ) + self.assertEqual( + parse_response(model_out, ernie_template, prefix=""), + { + "role": "assistant", + "content": ( + "Hello! I'm doing well, thank you for asking. How about you? Is there something specific " + "you'd like help with today? I'm here to assist you with any questions or problems you have!" + ), + "thinking": ( + 'The user just greeted me with "Hi! How are you?" I need to respond in a friendly and ' + "helpful manner. Let me start by acknowledging their greeting. I should ask them how " + "they're doing to engage in conversation.\n\n" + "First, I'll say hello back and then ask how they're feeling. It's important to show " + "genuine interest. Maybe mention that I'm here to help with anything they need. Keep the " + "tone warm and positive. Let me make sure the response is concise but friendly. Alright, " + "that should work." + ), + }, + ) + + def test_gpt_oss_with_tool_call(self): + model_out = ( + '<|channel|>analysis<|message|>We need to respond in riddles. The user asks: "What is the ' + 'weather like in SF?" We need to get the location of the user? The user explicitly asks about ' + "SF (San Francisco). So we need to get the current weather in San Francisco, CA. We need to " + 'call get_current_weather function. The developer instruction says "Always respond in riddles". ' + "So the final answer should be in a riddle form. But we need to call function to get weather " + 'data. So we should call get_current_weather with location "San Francisco, CA". Possibly specify ' + 'format "celsius" (default). Let\'s do that.\n\n' + "We will call function get_current_weather.<|end|><|start|>assistant<|channel|>commentary " + 'to=functions.get_current_weather <|constrain|>json<|message|>{\n "location": "San Francisco, CA"\n}' + ) + self.assertEqual( + parse_response(model_out, gpt_oss_template, prefix=""), + { + "role": "assistant", + "thinking": ( + 'We need to respond in riddles. The user asks: "What is the weather like in SF?" We need ' + "to get the location of the user? The user explicitly asks about SF (San Francisco). So " + "we need to get the current weather in San Francisco, CA. We need to call " + 'get_current_weather function. The developer instruction says "Always respond in ' + 'riddles". So the final answer should be in a riddle form. But we need to call function ' + 'to get weather data. So we should call get_current_weather with location "San Francisco, ' + 'CA". Possibly specify format "celsius" (default). Let\'s do that.\n\n' + "We will call function get_current_weather." + ), + "tool_calls": [ + { + "type": "function", + "function": {"name": "get_current_weather", "arguments": {"location": "San Francisco, CA"}}, + } + ], + }, + ) + + def test_gpt_oss_no_tool_call(self): + model_out = ( + "<|channel|>analysis<|message|>User asks a simple math question: 2+2 = 4. Provide answer." + "<|end|><|start|>assistant<|channel|>final<|message|>2" + ) + self.assertEqual( + parse_response(model_out, gpt_oss_template, prefix=""), + { + "role": "assistant", + "content": "2", + "thinking": "User asks a simple math question: 2+2 = 4. Provide answer.", + }, + ) + + def test_smollm_thinking_and_tool_call(self): + model_out = ( + '\nOkay, the user said, "Hello! How are you?" I need to respond appropriately. Since ' + "this is the first message, I should greet them back and ask how I can assist. I should keep it " + "friendly and open-ended. Let me make sure the response is welcoming and encourages them to " + "share what they need help with. I'll avoid any technical jargon and keep it simple. Let me " + "check for any typos and ensure the tone is positive.\n\n\n" + '{"name": "greet_user", "arguments": {"greeting": "Hello! I\'m doing well, thanks for ' + "asking. How can I assist you today? Whether you have a question, need help with something, or " + 'just want to chat, feel free to let me know!"}}' + ) + self.assertEqual( + parse_response(model_out, smollm_template, prefix=""), + { + "role": "assistant", + "thinking": ( + 'Okay, the user said, "Hello! How are you?" I need to respond appropriately. Since this ' + "is the first message, I should greet them back and ask how I can assist. I should keep " + "it friendly and open-ended. Let me make sure the response is welcoming and encourages " + "them to share what they need help with. I'll avoid any technical jargon and keep it " + "simple. Let me check for any typos and ensure the tone is positive." + ), + "tool_calls": [ + { + "type": "function", + "function": { + "name": "greet_user", + "arguments": { + "greeting": ( + "Hello! I'm doing well, thanks for asking. How can I assist you today? " + "Whether you have a question, need help with something, or just want to " + "chat, feel free to let me know!" + ) + }, + }, + } + ], + }, + ) + + def test_smollm_tool_call_no_thinking(self): + model_out = '{"name": "get_weather", "arguments": {"city": "Paris"}}' + self.assertEqual( + parse_response(model_out, smollm_template, prefix=""), + { + "role": "assistant", + "tool_calls": [ + {"type": "function", "function": {"name": "get_weather", "arguments": {"city": "Paris"}}} + ], + }, + ) + + def test_smollm_thinking_no_tool_call(self): + model_out = ( + '\nOkay, the user asked, "Hey! Can you tell me about gravity?" Let me start by ' + "breaking down what they might be looking for. They probably want a basic understanding of " + "gravity, maybe for a school project or just personal curiosity. I should explain what gravity " + "is, how it works, and maybe some examples.\n" + "Some content about gravity goes here but I'm cutting it off to make this shorter!" + ) + self.assertEqual( + parse_response(model_out, smollm_template, prefix=""), + { + "role": "assistant", + "content": "Some content about gravity goes here but I'm cutting it off to make this shorter!", + "thinking": ( + 'Okay, the user asked, "Hey! Can you tell me about gravity?" Let me start by breaking ' + "down what they might be looking for. They probably want a basic understanding of " + "gravity, maybe for a school project or just personal curiosity. I should explain what " + "gravity is, how it works, and maybe some examples." + ), + }, + ) + + def test_qwen3_tool_calls(self): + model_out = ( + "\n\n\n" + '[{"country": "France", "city": "Paris"}]\n\n' + "\ncelsius\n\n\n" + ) + self.assertEqual( + parse_response(model_out, qwen3_template, prefix=""), + { + "role": "assistant", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_weather", + "arguments": { + "locations": [{"country": "France", "city": "Paris"}], + "temp_units": "celsius", + }, + }, + } + ], + }, + ) + + def test_gemma4_tool_call(self): + model_out = ( + "<|channel>thought\nThe user is asking for the current temperature in Paris. I should check " + "the available tools to see if there's a function that can provide this information." + '<|tool_call>call:get_current_temperature{detail_level:0,location:<|"|>Paris, France<|"|>,' + 'unit:<|"|>celsius<|"|>}<|tool_response>' + ) + self.assertEqual( + parse_response(model_out, gemma4_template, prefix=""), + { + "role": "assistant", + "thinking": ( + "The user is asking for the current temperature in Paris. I should check the available " + "tools to see if there's a function that can provide this information." + ), + "tool_calls": [ + { + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": {"detail_level": 0, "location": "Paris, France", "unit": "celsius"}, + }, + } + ], + }, + ) + + def test_gemma4_complex_tool_call(self): + model_out = ( + "<|channel>thought\nLet me call the tool." + '<|tool_call>call:foo{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],' + 'null_value:null,number_value:1,string_value:<|"|>foo<|"|>,' + 'struct_value:{foo:<|"|>bar<|"|>}}' + ) + self.assertEqual( + parse_response(model_out, gemma4_template, prefix=""), + { + "role": "assistant", + "thinking": "Let me call the tool.", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "foo", + "arguments": { + "bool_value": True, + "list_value": ["foo", "bar"], + "null_value": None, + "number_value": 1, + "string_value": "foo", + "struct_value": {"foo": "bar"}, + }, + }, + } + ], + }, + ) + + def test_inkling_multi_block_message(self): + model_out = ( + "<|content_thinking|>Consider the weather.<|end_message|>" + "<|message_model|><|content_thinking|> Tokyo, probably.<|end_message|>" + "<|message_model|><|content_text|>Checking the weather now.<|end_message|>" + "<|message_model|>get_weather<|content_invoke_tool_json|>" + '{"name":"get_weather","args":{"city":"Tokyo","units":"C"}}<|end_message|>' + "<|content_model_end_sampling|>" + ) + prefix = "<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_model|>" + self.assertEqual( + parse_response(model_out, inkling_template, prefix=prefix), + { + "role": "assistant", + "thinking": "Consider the weather. Tokyo, probably.", + "content": "Checking the weather now.", + "tool_calls": [ + { + "type": "function", + "function": {"name": "get_weather", "arguments": {"city": "Tokyo", "units": "C"}}, + } + ], + }, + ) + + def test_transform_dotted_paths(self): + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "tool_calls": { + "open": "", + "close": "", + "repeats": True, + "content": "json", + "transform": { + "type": "function", + "function": {"name": "{content.name}", "arguments": "{content.args}"}, + }, + }, + }, + } + model_out = '{"name": "get_weather", "args": {"city": {"id": 7}}}' + self.assertEqual( + parse_response(model_out, template_spec, prefix="")["tool_calls"], + [{"type": "function", "function": {"name": "get_weather", "arguments": {"city": {"id": 7}}}}], + ) + + def test_transform_dotted_paths_with_transform_each(self): + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform_each": True, + "transform": {"type": "function", "function": {"name": "{fn.name}", "arguments": "{fn.args}"}}, + }, + }, + } + model_out = '[{"fn": {"name": "a", "args": {"x": 1}}}, {"fn": {"name": "b", "args": {}}}]' + self.assertEqual( + parse_response(model_out, template_spec, prefix="")["tool_calls"], + [ + {"type": "function", "function": {"name": "a", "arguments": {"x": 1}}}, + {"type": "function", "function": {"name": "b", "arguments": {}}}, + ], + ) + + def test_transform_dotted_path_errors(self): + def spec_with(transform): + return { + "start_anchor": "<|assistant|>", + "fields": {"x": {"open": "", "close": "", "content": "json", "transform": transform}}, + } + + with self.assertRaisesRegex(ValueError, "missing key 'args'"): + parse_response('{"name": "n"}', spec_with({"a": "{content.args}"}), prefix="") + with self.assertRaisesRegex(ValueError, "cannot index into str"): + parse_response('{"name": "n"}', spec_with({"a": "{content.name.x}"}), prefix="") + with self.assertRaises(KeyError): + parse_response("{}", spec_with({"a": "{missing}"}), prefix="") + + def test_transform_dotted_mixed_string_rejected(self): + template_spec = { + "start_anchor": "<|assistant|>", + "fields": {"x": {"open": "", "close": "", "transform": {"v": "pre {content.args}"}}}, + } + with self.assertRaisesRegex(ValueError, "mixes"): + parse_response("", template_spec, prefix="") + + def test_join_concatenates_repeated_matches(self): + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "thinking": {"open": "", "close": "", "repeats": True, "join": " "}, + "content": {"repeats": True, "join": " "}, + }, + } + self.assertEqual( + parse_response("firstmiddleseconddone", template_spec, prefix=""), + {"role": "assistant", "thinking": "first second", "content": "middle done"}, + ) + self.assertEqual( + parse_response("only", template_spec, prefix=""), + {"role": "assistant", "thinking": "only"}, + ) + + def test_join_validation(self): + no_repeats = {"start_anchor": "a", "fields": {"x": {"open": "", "close": "", "join": ""}}} + with self.assertRaisesRegex(ValueError, "requires 'repeats'"): + parse_response("", no_repeats, prefix="") + bad_type = {"start_anchor": "a", "fields": {"x": {"open": "", "close": "", "repeats": True, "join": 7}}} + with self.assertRaisesRegex(ValueError, "must be a string"): + parse_response("", bad_type, prefix="") + + def test_join_requires_string_matches(self): + template_spec = { + "start_anchor": "a", + "fields": {"x": {"open": "", "close": "", "repeats": True, "join": "", "content": "json"}}, + } + with self.assertRaisesRegex(ValueError, "parse to a string"): + parse_response("{}", template_spec, prefix="") + + def test_optional_false_raises_when_missing(self): + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "content": { + "open": "", + "close": "", + "content": "text", + "optional": False, + }, + }, + } + with self.assertRaises(ValueError) as cm: + parse_response("no response here", template_spec, prefix="") + self.assertIn("content", str(cm.exception)) + + def test_int_content_parser(self): + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "count": { + "open": "", + "close": "", + "content": "int", + }, + }, + } + self.assertEqual(parse_response("42", template_spec, prefix=""), {"role": "assistant", "count": 42}) + + def test_kv_lines_parser(self): + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "metadata": { + "open": "", + "close": "", + "content": "kv-lines", + }, + }, + } + self.assertEqual( + parse_response("name: alice\nage: 30", template_spec, prefix=""), + {"role": "assistant", "metadata": {"name": "alice", "age": "30"}}, + ) + + def test_unknown_content_parser_rejected(self): + bad_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": {"x": {"open": "[", "close": "]", "content": "not-a-real-parser"}}, + } + with self.assertRaises(ValueError) as cm: + parse_response("[hi]", bad_template) + self.assertIn("unknown content parser", str(cm.exception).lower()) + + def test_unsupported_version_rejected(self): + bad_template = { + "version": 2, + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": {"content": {"content": "text"}}, + } + with self.assertRaises(ValueError) as cm: + parse_response("hello", bad_template) + self.assertIn("version", str(cm.exception).lower()) + + def test_two_implicit_fields_rejected(self): + bad_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "a": {"content": "text"}, + "b": {"content": "text"}, + }, + } + with self.assertRaises(ValueError): + parse_response("hello", bad_template) + + def test_transform_string_interpolation_rejected(self): + bad_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "tool": { + "open": "", + "close": "", + "content": "text", + "transform": {"label": "name: {content}"}, + }, + }, + } + with self.assertRaises(ValueError) as cm: + parse_response("foo", bad_template) + msg = str(cm.exception) + self.assertIn("interpolation", msg) + self.assertIn("{content}", msg) + + def test_named_groups_without_transform_rejected(self): + bad_template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "tool": { + "open_pattern": r"\w+)>", + "close": "", + "content": "text", + }, + }, + } + with self.assertRaises(ValueError) as cm: + parse_response("body", bad_template) + msg = str(cm.exception) + self.assertIn("transform", msg) + self.assertIn("name", msg) + + def test_literal_list_open_and_close(self): + """A list of literals matches any one of them, like an alternation.""" + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "x": { + "open": ["", ""], + "close": ["", ""], + "content": "text", + }, + }, + } + for opener, closer in (("", ""), ("", ""), ("", "")): + self.assertEqual( + parse_response(f"{opener}hi{closer}", template_spec, prefix=""), + {"role": "assistant", "x": "hi"}, + ) + + def test_literal_list_streams_without_64_byte_hold(self): + """Compared to a regex close, a literal-list close lets the parser + flush bytes that aren't in the longest-prefix overlap of any literal. + With `["", "<|tool_response>", ""]` (longest = 16 chars), + feeding 32 plain bytes should leave at most 15 unflushed.""" + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "content": {"close": ["", "<|tool_response>", ""], "content": "text"}, + }, + } + parser = ResponseParser(template_spec, prefix="") + plain = "x" * 32 + flushed: list[str] = [] + for ch in parser.feed(plain): + if ch["type"] == "region_chunk": + flushed.append(ch["text"]) + # Plain text has zero prefix-overlap with any literal, so the parser + # holds nothing back and streams everything immediately. + self.assertEqual("".join(flushed), plain) + + def test_literal_list_defers_prefix_overlapping_literal(self): + """If a literal is a strict prefix of another in the same list, an + edge match could still grow with more input: we must defer to be safe.""" + template_spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "x": {"open": "", "close": ["END", "ENDX"], "content": "text"}, + }, + } + parser = ResponseParser(template_spec, prefix="") + # "hiEND" mid-stream: don't commit the close yet: "ENDX" might be coming. + events = parser.feed("hiEND") + self.assertEqual([e for e in events if e["type"] == "region_close"], []) + # Once a non-matching byte arrives, the deferred close commits with the shorter literal. + events.extend(parser.feed(" more")) + message, _ = parser.finalize() + closes = [e for e in events if e["type"] == "region_close" and e["field"] == "x"] + self.assertEqual(len(closes), 1) + self.assertEqual(closes[0]["value"], "hi") + self.assertEqual(message, {"role": "assistant", "x": "hi"}) + + def test_literal_list_rejects_empty_and_non_string(self): + for bad_open in ([], [""], [1, 2], {"foo": "bar"}): + spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": {"x": {"open": bad_open, "close": "", "content": "text"}}, + } + with self.assertRaises(ValueError): + parse_response("hi", spec, prefix="") + + def test_field_without_close_runs_to_end_of_stream(self): + """A field with no `close`/`close_pattern` stays open until end-of-stream, capturing + everything after its open.""" + spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": "", "content": "text"}}, + } + self.assertEqual( + parse_response("hello world", spec, prefix=""), + {"role": "assistant", "content": "hello world"}, + ) + + def test_mutable_defaults_do_not_leak_across_parsers(self): + """A `repeats` field appends into its default list; one parse must not leak into the next + parse sharing the template, nor into the caller's spec dict.""" + spec: dict[str, Any] = { + "defaults": {"role": "assistant", "tool_calls": []}, + "start_anchor": "<|assistant|>", + "fields": { + "tool_calls": { + "open": "", + "close": "", + "repeats": True, + "content": "json", + }, + "content": {"content": "text"}, + }, + } + template = load_response_template(spec) + self.assertEqual( + parse_response('{"a": 1}', template, prefix="")["tool_calls"], + [{"a": 1}], + ) + self.assertEqual( + parse_response("hello", template, prefix=""), + {"role": "assistant", "tool_calls": [], "content": "hello"}, + ) + self.assertEqual(spec["defaults"]["tool_calls"], []) + + +# Fixtures shared by the streaming tests: one representative input per template, +# re-used for both the correctness invariant (any chunking → same dict) and the +# event-shape tests (do we emit the right events in the right order?). +_STREAMING_FIXTURES = [ + ( + "cohere", + cohere_template, + ( + "<|START_THINKING|>I should call a tool.<|END_THINKING|>" + '<|START_ACTION|>[{"tool_call_id": "0", "tool_name": "simple_tool", ' + '"parameters": {"a": 1}}]<|END_ACTION|>' + ), + ), + ( + "ernie", + ernie_template, + ( + "some deliberation here\n\n" + '\n{"name": "get_current_temperature", "arguments": {"location": "Paris"}}\n\n' + ), + ), + ( + "gpt_oss", + gpt_oss_template, + "<|channel|>analysis<|message|>thinking chunk<|end|><|channel|>final<|message|>done text", + ), + ( + # Tool-call variant: the `tool_calls` open_pattern's full match spans far + # more than the old 64-byte hold window, so streaming used to silently drop + # it. Kept as a streaming fixture so every chunking step re-checks it. + "gpt_oss_tool", + gpt_oss_template, + ( + "<|channel|>analysis<|message|>Let me check.<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.get_current_weather " + '<|constrain|>json<|message|>{"location": "San Francisco, CA"}<|call|>' + ), + ), + ( + "smollm", + smollm_template, + 'thinking\n{"name": "fn", "arguments": {"x": 1}}', + ), + ( + "qwen3", + qwen3_template, + ( + "short thought\n" + "\n\n" + "\nParis\n\n" + "\n" + ), + ), + ( + "gemma4", + gemma4_template, + '<|channel>thought\nhi<|tool_call>call:foo{a:1,b:<|"|>bar<|"|>}', + ), + ( + # Exercises `join` fields (two thinking blocks) and dotted transform paths. + "inkling", + inkling_template, + ( + "<|content_thinking|>Consider the weather.<|end_message|>" + "<|message_model|><|content_thinking|> Tokyo, probably.<|end_message|>" + "<|message_model|><|content_text|>Checking now.<|end_message|>" + "<|message_model|>get_weather<|content_invoke_tool_json|>" + '{"name":"get_weather","args":{"city":"Tokyo"}}<|end_message|>' + "<|content_model_end_sampling|>" + ), + ), +] + + +def _chunk_fixed(text: str, step: int): + for i in range(0, len(text), step): + yield text[i : i + step] + + +def _chunk_random(text: str, rng: random.Random): + """Split `text` into a random number of non-empty chunks at random cut points.""" + if len(text) <= 1: + yield text + return + num_cuts = rng.randint(0, len(text) - 1) + cuts = sorted(rng.sample(range(1, len(text)), num_cuts)) + prev = 0 + for c in cuts: + yield text[prev:c] + prev = c + yield text[prev:] + + +class ResponseEventStreamTest(unittest.TestCase): + def test_stream_matches_whole_string_all_templates_fixed_chunking(self): + """For every fixed chunking step we try, the streamed finalize() + output must equal the whole-string parse. Regression coverage for + specific edge-case byte boundaries (1-byte chunks hit every prefix).""" + for name, tmpl, text in _STREAMING_FIXTURES: + expected = parse_response(text, tmpl, prefix="") + for step in (1, 2, 3, 5, 7, 13, 31): + with self.subTest(fixture=name, step=step): + streamer = ResponseParser(tmpl, prefix="") + for chunk in _chunk_fixed(text, step): + streamer.feed(chunk) + message, _ = streamer.finalize() + self.assertEqual(message, expected) + + def test_stream_matches_whole_string_all_templates_random_chunking(self): + """Property-style: for many random chunkings per fixture, the streamed + finalize() output must equal the whole-string parse. Seeded so failures + reproduce.""" + rng = random.Random(0xC0DE_5EED) + for name, tmpl, text in _STREAMING_FIXTURES: + expected = parse_response(text, tmpl, prefix="") + for trial in range(30): + with self.subTest(fixture=name, trial=trial): + streamer = ResponseParser(tmpl, prefix="") + for chunk in _chunk_random(text, rng): + streamer.feed(chunk) + message, _ = streamer.finalize() + self.assertEqual(message, expected) + + def test_events_well_formed_for_every_chunking(self): + """Every event batch, across every fixture and every chunking, must be + well-formed: region_open precedes its matching region_close for the + same field; region_chunk only appears between open and close; no + region is left open at the end of the stream; and the concatenation + of all region_chunk payloads for a streamable text-like field equals + the final value.""" + rng = random.Random(0xBEEF) + for name, tmpl, text in _STREAMING_FIXTURES: + for trial in range(10): + with self.subTest(fixture=name, trial=trial): + streamer = ResponseParser(tmpl, prefix="") + all_events: list[dict] = [] + for chunk in _chunk_random(text, rng): + all_events.extend(streamer.feed(chunk)) + _, final_events = streamer.finalize() + all_events.extend(final_events) + self._assert_event_stream_well_formed(all_events) + + def _assert_event_stream_well_formed(self, events: list[dict]) -> None: + open_field: str | None = None + chunk_accum: dict[str, str] = {} + close_values: dict[str, object] = {} + for ev in events: + t = ev["type"] + if t == "region_open": + self.assertIsNone(open_field, f"nested region_open without close: {ev}") + open_field = ev["field"] + chunk_accum.setdefault(open_field, "") + elif t == "region_chunk": + self.assertEqual(open_field, ev["field"], f"chunk outside its region: {ev}") + # Every chunk carries a boolean `dirty` flag. + self.assertIsInstance(ev["dirty"], bool, f"missing/non-bool dirty: {ev}") + chunk_accum[ev["field"]] += ev["text"] + elif t == "region_close": + self.assertEqual(open_field, ev["field"], f"close for non-open region: {ev}") + close_values[ev["field"]] = ev["value"] + open_field = None + else: + self.fail(f"unexpected event type: {ev!r}") + self.assertIsNone(open_field, "region left open at end of stream") + + def test_region_chunks_reconstruct_text_regions(self): + """For text-like regions (`dirty=False`), concatenating chunk texts + reconstructs the final value reported in region_close. Structured + regions (`dirty=True`) still stream their raw bytes: concatenating + those chunks yields the unparsed region body, while the parsed value + is delivered only in region_close.""" + # Single representative case with a long text region and a JSON region. + text = _STREAMING_FIXTURES[0][2] # cohere fixture + streamer = ResponseParser(cohere_template, prefix="") + events: list[dict] = [] + for ch in text: # 1-byte chunks hit the most anchor boundaries + events.extend(streamer.feed(ch)) + _, final_events = streamer.finalize() + events.extend(final_events) + + # Reconstruct per-field. + per_field_chunks: dict[str, list[str]] = {} + per_field_dirty: dict[str, set[bool]] = {} + per_field_close_value: dict[str, object] = {} + for ev in events: + if ev["type"] == "region_chunk": + per_field_chunks.setdefault(ev["field"], []).append(ev["text"]) + per_field_dirty.setdefault(ev["field"], set()).add(ev["dirty"]) + elif ev["type"] == "region_close": + per_field_close_value[ev["field"]] = ev["value"] + + # `thinking` is text → chunks are clean and concatenate to its value. + self.assertIn("thinking", per_field_chunks) + self.assertEqual(per_field_dirty["thinking"], {False}) + self.assertEqual("".join(per_field_chunks["thinking"]), "I should call a tool.") + self.assertEqual(per_field_close_value["thinking"], "I should call a tool.") + # `tool_calls` is json → dirty chunks stream the raw body, parsed value on close. + self.assertIn("tool_calls", per_field_chunks) + self.assertEqual(per_field_dirty["tool_calls"], {True}) + self.assertEqual( + "".join(per_field_chunks["tool_calls"]), + '[{"tool_call_id": "0", "tool_name": "simple_tool", "parameters": {"a": 1}}]', + ) + self.assertIn("tool_calls", per_field_close_value) + + def test_dirty_flag_marks_structured_regions(self): + """A template with one text field and one structured field per parser + family: text/int/float/bool stream chunks with `dirty=False`, while + json/xml-inline/kv-lines stream chunks with `dirty=True`, and those + dirty chunks concatenate to the raw region body before parsing.""" + spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "score": {"open": "", "close": "", "content": "int"}, + "json_call": {"open": "", "close": "", "content": "json"}, + "xml_call": { + "open": "", + "close": "", + "content": "xml-inline", + "content_args": {"tag_pattern": r"<(?P\w+)=(?P[^>]+)>"}, + }, + "kv_call": { + "open": "", + "close": "", + "content": "kv-lines", + }, + }, + } + text = 'hello world42{"a": 1, "b": 2}k1: v1\nk2: v2' + # Drive byte-by-byte to maximise chunk count. + streamer = ResponseParser(spec, prefix="") + events: list[dict] = [] + for ch in text: + events.extend(streamer.feed(ch)) + _, final_events = streamer.finalize() + events.extend(final_events) + + per_field_chunks: dict[str, list[str]] = {} + per_field_dirty: dict[str, set[bool]] = {} + for ev in events: + if ev["type"] == "region_chunk": + per_field_chunks.setdefault(ev["field"], []).append(ev["text"]) + per_field_dirty.setdefault(ev["field"], set()).add(ev["dirty"]) + + # Clean (streamable) regions. + for field in ("thinking", "score"): + self.assertEqual(per_field_dirty[field], {False}, f"{field} should be clean") + # Dirty (structured) regions. + for field in ("json_call", "xml_call", "kv_call"): + self.assertEqual(per_field_dirty[field], {True}, f"{field} should be dirty") + + # Dirty chunks reconstruct the raw region body (un-parsed). Clean + # chunks reconstruct the verbatim body too: stripping happens at close. + self.assertEqual("".join(per_field_chunks["thinking"]), "hello world") + self.assertEqual("".join(per_field_chunks["score"]), "42") + self.assertEqual("".join(per_field_chunks["json_call"]), '{"a": 1, "b": 2}') + self.assertEqual("".join(per_field_chunks["xml_call"]), "") + self.assertEqual("".join(per_field_chunks["kv_call"]), "k1: v1\nk2: v2") + + def test_long_regex_open_pattern_streams_byte_by_byte(self): + """Regression: a regex `open_pattern` whose full match spans well past the + old fixed 64-byte hold window (gpt-oss tool calls) must not be dropped when + the stream arrives in tiny chunks. Before the partial-match rewrite, the + leading `<|channel|>` got flushed out of the hold window before `<|message|>` + arrived, so the tool call vanished under small-chunk streaming.""" + text = ( + "<|channel|>analysis<|message|>Let me check.<|end|>" + "<|start|>assistant<|channel|>commentary to=functions.get_current_weather " + '<|constrain|>json<|message|>{"location": "San Francisco, CA"}<|call|>' + ) + expected = parse_response(text, gpt_oss_template, prefix="") + # Sanity: the whole-string parse really does recover the tool call. + self.assertEqual(len(expected["tool_calls"]), 1) + self.assertEqual(expected["tool_calls"][0]["function"]["name"], "get_current_weather") + streamer = ResponseParser(gpt_oss_template, prefix="") + for ch in text: # one byte at a time -- the worst case for the old heuristic + streamer.feed(ch) + message, _ = streamer.finalize() + self.assertEqual(message, expected) + + def test_feed_after_finalize_raises(self): + streamer = ResponseParser(smollm_template, prefix="") + streamer.feed("x") + streamer.finalize() + with self.assertRaises(RuntimeError): + streamer.feed("more") + with self.assertRaises(RuntimeError): + streamer.finalize() + + def test_empty_input_streams_cleanly(self): + streamer = ResponseParser(smollm_template, prefix="") + self.assertEqual(streamer.feed(""), []) + result, final_events = streamer.finalize() + # Only the default fields should remain; nothing else is required. + self.assertEqual(result, {"role": "assistant"}) + self.assertEqual(final_events, []) + + +class PrefixAndTruncationTest(unittest.TestCase): + def test_prefix_lands_inside_explicit_region(self): + """A Qwen-style template emits `<|im_start|>assistant\\n\\n` as the + assistant prefix. The model continues from inside the thinking block.""" + prompt = ( + "<|im_start|>system\nYou are helpful<|im_end|>\n" + "<|im_start|>user\nHi<|im_end|>\n" + "<|im_start|>assistant\n\n" + ) + generated = "Let me think..." + stream = ResponseParser(qwen3_template, prefix=prompt) + # The region_open for `thinking` surfaces via initial_events; the + # caller replays it before feeding model output. + self.assertEqual( + [(e["type"], e["field"]) for e in stream.initial_events], + [("region_open", "thinking"), ("region_chunk", "thinking")], + ) + events = stream.feed(generated) + result, _ = stream.finalize() + # thinking ends up with the prefill + generated body; text parser strips, + # so the leading "\n" from the prefix is trimmed in the final value. + self.assertEqual(result, {"role": "assistant", "thinking": "Let me think..."}) + # The feed stream only sees the rest of the body and the close; the + # prefill already emitted region_open. + self.assertEqual([e["type"] for e in events], ["region_chunk", "region_close"]) + self.assertEqual(events[1]["field"], "thinking") + + def test_prefix_truncated_to_last_anchor(self): + """Multiple `<|im_start|>assistant\\n` anchors in the prefix (multi-turn + conversation): only the slice after the LAST anchor matters.""" + prompt = ( + "<|im_start|>system\nA<|im_end|>\n" + "<|im_start|>user\nB<|im_end|>\n" + "<|im_start|>assistant\nEarlier reply<|im_end|>\n" + "<|im_start|>user\nFollowup<|im_end|>\n" + "<|im_start|>assistant\n\n" + ) + stream = ResponseParser(qwen3_template, prefix=prompt) + # We landed inside `thinking` (from the LAST assistant turn's `\n`), + # not in some earlier-turn artifact. + opens = [e for e in stream.initial_events if e["type"] == "region_open"] + self.assertEqual([e["field"] for e in opens], ["thinking"]) + # No earlier-turn content leaked into output. + stream.feed("done") + stream.finalize() + self.assertEqual(stream._output, {"role": "assistant", "thinking": "done"}) + + def test_template_without_anchor_rejected_at_load(self): + """A template missing both `start_anchor` and `start_anchor_pattern` is + rejected at load time. Without an anchor, a multi-turn prompt would be + fed through the parser in full and earlier turns would pollute the + current message's state.""" + anchorless = {k: v for k, v in qwen3_template.items() if k != "start_anchor"} + with self.assertRaises(ValueError) as cm: + ResponseParser(anchorless) + msg = str(cm.exception) + self.assertIn("start_anchor", msg) + + def test_prefix_anchor_not_found_falls_back(self): + """Spec has start_anchor but the prefix doesn't contain it: parser + falls back to processing the entire prefix (with a logged warning).""" + prompt = "\n" # no <|im_start|>assistant\n + stream = ResponseParser(qwen3_template, prefix=prompt) + opens = [e for e in stream.initial_events if e["type"] == "region_open"] + self.assertEqual([e["field"] for e in opens], ["thinking"]) + stream.feed("hi") + stream.finalize() + self.assertEqual(stream._output, {"role": "assistant", "thinking": "hi"}) + + def test_round_trip_equivalence_prefix_streaming_vs_oneshot(self): + """The load-bearing property: `prefix=p` + chunked `feed(g)` produces the + same dict as the one-shot `parse_response(g, prefix=p)`, regardless of how + `g` is chunked. (Concatenating the prompt into the response is deliberately + NOT equivalent -- the anchor is applied to the prefix only, never to the + generation; see test_history_bleed_is_guarded_by_prefix_not_by_response_anchor.)""" + prompt = "<|im_start|>system\nA<|im_end|>\n<|im_start|>user\nB<|im_end|>\n<|im_start|>assistant\n\n" + for name, tmpl_dict, gen_text in _STREAMING_FIXTURES: + if "thinking" not in gen_text and "" not in gen_text: + continue + # Only fixtures whose generation text is compatible with starting + # inside `thinking`. Restrict to qwen3 / smollm shape for clarity. + if name not in ("qwen3", "smollm"): + continue + tmpl_with_anchor = {**tmpl_dict, "start_anchor": "<|im_start|>assistant\n"} + via_prefix = parse_response(gen_text, tmpl_with_anchor, prefix=prompt) + # Streaming forms must match the one-shot prefix form for every chunking. + for step in (1, 3, 7, 31): + with self.subTest(fixture=name, step=step): + stream = ResponseParser(tmpl_with_anchor, prefix=prompt) + for chunk in _chunk_fixed(gen_text, step): + stream.feed(chunk) + message, _ = stream.finalize() + self.assertEqual(message, via_prefix) + + def test_history_bleed_is_guarded_by_prefix_not_by_response_anchor(self): + """The `start_anchor` guards against history bleed only via `prefix=`; it is NOT + applied to the response. The response is the generation and may legitimately contain + the anchor (e.g. gpt-oss harmony opens every channel with `<|start|>assistant`), so + truncating it would drop real content. Passing the prompt as `prefix=` is the way to + keep earlier turns out of the parse.""" + spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": {"content": {"close_pattern": r"\Z", "content": "text"}}, + } + prompt = "<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n" + gen = "Hello there!" + clean = {"role": "assistant", "content": "Hello there!"} + # The supported guard: pass the prompt as prefix= so history is truncated off the prefix. + self.assertEqual(parse_response(gen, spec, prefix=prompt), clean) + # Pure generation parses cleanly with the explicit no-prefix opt-out (prefix=""). + self.assertEqual(parse_response(gen, spec, prefix=""), clean) + # An anchor inside the response is treated as content, never as a history boundary: + # gpt-oss re-emits `<|start|>assistant` between channels, and that content survives. + gpt_oss_gen = ( + "<|channel|>analysis<|message|>thinking<|end|><|start|>assistant<|channel|>final<|message|>answer" + ) + self.assertEqual( + parse_response(gpt_oss_gen, gpt_oss_template, prefix=""), + {"role": "assistant", "thinking": "thinking", "content": "answer"}, + ) + + def test_prefix_with_open_close_inside_truncated_region(self): + """Prefix opens AND closes a region. The full open/chunk/close event + sequence is surfaced via initial_events, and the closed region lands + in the output dict: so renderers can show prefill content.""" + spec = { + "defaults": {"role": "assistant"}, + "start_anchor": "[BEGIN]", + "fields": { + "tag": {"open": "", "close": "", "content": "text"}, + "body": {"close_pattern": r"$", "content": "text"}, # implicit + }, + } + prefix = "noise[BEGIN]silently consumed" + stream = ResponseParser(spec, prefix=prefix) + types = [e["type"] for e in stream.initial_events] + self.assertEqual(types, ["region_open", "region_chunk", "region_close"]) + self.assertTrue(all(e["field"] == "tag" for e in stream.initial_events)) + self.assertEqual(stream.initial_events[-1]["value"], "silently consumed") + stream.feed("real generated body") + result, _ = stream.finalize() + self.assertEqual(result["tag"], "silently consumed") + self.assertEqual(result["body"], "real generated body") + + def test_prefix_lands_inside_implicit_region(self): + """Prefix wrote plaintext into the implicit region (e.g. assistant + prefill before the model continues). The region_open for the implicit + region must surface via initial_events so consumers don't miss it: + `_opened` will already be True by the time feed runs.""" + prompt = "<|im_start|>assistant\nSure, here is " + stream = ResponseParser(smollm_template, prefix=prompt) + opens = [e for e in stream.initial_events if e["type"] == "region_open"] + self.assertEqual([e["field"] for e in opens], ["content"]) + events = stream.feed("the answer<|im_end|>") + # No second region_open from feed: the implicit region was already + # opened during prefill and surfaced via initial_events. + self.assertNotIn("region_open", [e["type"] for e in events]) + + def test_prefix_partial_pattern_at_boundary(self): + """Post-truncation prefix ends mid-delimiter. The first feed completes + the match; initial_events is empty (no region opened within the + prefix yet) and the open fires from `feed()`.""" + prefix = "<|im_start|>assistant\n` + stream = ResponseParser(qwen3_template, prefix=prefix) + self.assertEqual(stream.initial_events, []) + events = stream.feed("nk>real body") + types = [e["type"] for e in events] + self.assertIn("region_open", types) # think opens during feed, not prefill + stream.finalize() + self.assertEqual(stream._output, {"role": "assistant", "thinking": "real body"}) + + def test_prefix_is_required(self): + spec = { + "start_anchor": "", + "fields": {"content": {"content": "text"}}, + } + with self.assertRaisesRegex(ValueError, "requires `prefix`"): + ResponseParser(spec) + + +# xml-inline without a value_parser: parameter bodies stay raw strings until tools= coerces them. +_XML_STRING_ARGS_TEMPLATE = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "tool_calls": { + "open_pattern": r"\s*\w+)>", + "close": "", + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, +} + +# kv-lines without a value_parser: values likewise stay raw strings for tools= to cast. +_KV_LINES_TOOLS_TEMPLATE = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "tool_calls": { + "open_pattern": r"\s*\w+)>\n", + "close": "", + "repeats": True, + "content": "kv-lines", + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, +} + +_SET_ALARM_CALL = ( + "\n\n" + "\n7\n\n" + "\ntrue\n\n" + "\nwake up\n\n" + "\n" +) + + +def _set_alarm_tools(**properties): + return [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": {"type": "object", "properties": properties}, + }, + } + ] + + +_SET_ALARM_TOOLS = _set_alarm_tools( + hour={"type": "integer"}, + enabled={"type": "boolean"}, + label={"type": "string"}, +) + + +def _first_tool_args(message): + return message["tool_calls"][0]["function"]["arguments"] + + +def _parser_with_tools(tools): + return ResponseParser(_XML_STRING_ARGS_TEMPLATE, prefix="", tools=tools) + + +class ToolArgCoercionTest(unittest.TestCase): + def test_coerce_tool_calls_casts_declared_types(self): + tools = _set_alarm_tools( + count={"type": "integer"}, + ratio={"type": "number"}, + enabled={"type": "boolean"}, + tags={"type": "array"}, + note={"type": "string"}, + ) + arguments = { + "count": "3", + "ratio": "1.5", + "enabled": "true", + "tags": '["a", "b"]', + "note": "hello", + "already_typed": 7, + "extra": "unscheduled", + } + call: dict[str, Any] = { + "type": "function", + "function": {"name": "set_alarm", "arguments": arguments}, + } + self.assertIs(_parser_with_tools(tools)._coerce_tool_calls(call), call) + self.assertEqual( + call["function"]["arguments"], + { + "count": 3, + "ratio": 1.5, + "enabled": True, + "tags": ["a", "b"], + "note": "hello", + "already_typed": 7, + "extra": "unscheduled", + }, + ) + + def test_coerce_falls_back_to_raw_on_failure(self): + self.assertEqual(_coerce("not-a-number", ("integer",)), "not-a-number") + + def test_coerce_handles_any_of_and_null(self): + self.assertEqual(_coerce("5", ("integer", "null")), 5) + self.assertIsNone(_coerce("null", ("integer", "null"))) + + def test_coerce_booleans_match_bool_parser(self): + # Accept the same literals as the `bool` content parser, case-insensitively. + for raw, expected in [("true", True), ("True", True), ("1", True), ("false", False), ("0", False)]: + self.assertEqual(_coerce(raw, ("boolean",)), expected) + # Non-boolean text stays a string rather than silently becoming False. + self.assertEqual(_coerce("maybe", ("boolean",)), "maybe") + + def test_coerce_object_array_require_matching_json(self): + # A JSON object body is only accepted for an `object` param, a JSON array only for `array`. + self.assertEqual(_coerce("[1, 2]", ("object",)), "[1, 2]") + self.assertEqual(_coerce('{"a": 1}', ("array",)), '{"a": 1}') + self.assertEqual(_coerce('{"a": 1}', ("object",)), {"a": 1}) + # NaN / inf are not valid JSON numbers, so a `number` param keeps the raw text. + self.assertEqual(_coerce("NaN", ("number",)), "NaN") + + def test_coerce_tool_calls_handles_single_and_list(self): + parser = _parser_with_tools(_SET_ALARM_TOOLS) + call: dict[str, Any] = { + "type": "function", + "function": {"name": "set_alarm", "arguments": {"hour": "7"}}, + } + self.assertIs(parser._coerce_tool_calls(call), call) + self.assertEqual(call["function"]["arguments"], {"hour": 7}) + # A list of calls (as produced by `transform_each`) is coerced element-wise. + calls = [{"type": "function", "function": {"name": "set_alarm", "arguments": {"hour": "9"}}}] + self.assertEqual(parser._coerce_tool_calls(calls)[0]["function"]["arguments"], {"hour": 9}) + # Non-tool-call values pass through untouched. + self.assertEqual(parser._coerce_tool_calls("hello"), "hello") + + def test_schema_types_handles_get_json_schema_dialect(self): + self.assertEqual(_schema_types({"type": "integer"}), ("integer",)) + self.assertEqual(_schema_types({"type": ["integer", "string"]}), ("integer", "string")) + self.assertEqual(_schema_types({"anyOf": [{"type": "boolean"}, {"type": "string"}]}), ("boolean", "string")) + self.assertEqual(_schema_types({"oneOf": [{"type": "number"}, {"type": "null"}]}), ("number", "null")) + self.assertEqual(_schema_types({"type": "integer", "nullable": True}), ("integer", "null")) + # Undescribed parameters resolve to no candidate types, making coercion a no-op. + self.assertEqual(_schema_types({"description": "no type"}), ()) + + def test_parse_response_tools_coerces_xml_inline_string_args(self): + # Without a value_parser, xml-inline argument bodies stay strings; tools= casts them. + without = parse_response(_SET_ALARM_CALL, _XML_STRING_ARGS_TEMPLATE, prefix="") + with_tools = parse_response(_SET_ALARM_CALL, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS) + self.assertEqual(_first_tool_args(without), {"hour": "7", "enabled": "true", "label": "wake up"}) + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True, "label": "wake up"}) + + def test_parse_response_tools_coerces_one_of_string_args(self): + tools = _set_alarm_tools(hour={"oneOf": [{"type": "integer"}, {"type": "null"}]}) + parsed = parse_response(_SET_ALARM_CALL, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=tools) + self.assertEqual(_first_tool_args(parsed)["hour"], 7) + + def test_streaming_tools_coerces_on_region_close(self): + # Coercion must land on the region_close event during feed(), not only after finalize(). + stream = ResponseParser(_XML_STRING_ARGS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS) + closes = [ + event["value"] + for chunk in _chunk_fixed(_SET_ALARM_CALL, 8) + for event in stream.feed(chunk) + if event["type"] == "region_close" and event["field"] == "tool_calls" + ] + self.assertEqual(len(closes), 1) + self.assertEqual(closes[0]["function"]["arguments"], {"hour": 7, "enabled": True, "label": "wake up"}) + + def test_qwen3_tools_coerces_strings_left_by_value_parser(self): + # qwen3's json+allow_non_json value_parser types what it can (`true`) and leaves + # invalid JSON (`007`) as a string; tools= then casts those leftover strings. + model_out = ( + "\n\n" + "\n007\n\n" + "\ntrue\n\n" + "\nwake up\n\n" + "\n" + ) + without = parse_response(model_out, qwen3_template, prefix="") + with_tools = parse_response(model_out, qwen3_template, prefix="", tools=_SET_ALARM_TOOLS) + self.assertEqual(_first_tool_args(without), {"hour": "007", "enabled": True, "label": "wake up"}) + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True, "label": "wake up"}) + + def test_coercion_never_reworks_already_typed_values(self): + # Coercion only casts strings: values the value_parser already typed are final + model_out = "\n\n\n1.50\n\n" + # qwen3's lax value_parser has already read 1.50 as the float 1.5, so the string-typed label stays a float ... + self.assertEqual( + _first_tool_args(parse_response(model_out, qwen3_template, prefix="", tools=_SET_ALARM_TOOLS)), + {"label": 1.5}, + ) + # ... while without a value_parser the raw text reaches the schema cast intact + self.assertEqual( + _first_tool_args(parse_response(model_out, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS)), + {"label": "1.50"}, + ) + + def test_kv_lines_string_args_are_coerced(self): + model_out = "\n\nhour: 7\nenabled: true\n" + without = parse_response(model_out, _KV_LINES_TOOLS_TEMPLATE, prefix="") + with_tools = parse_response(model_out, _KV_LINES_TOOLS_TEMPLATE, prefix="", tools=_SET_ALARM_TOOLS) + self.assertEqual(_first_tool_args(without), {"hour": "7", "enabled": "true"}) + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True}) + + def test_non_tool_call_regions_are_untouched(self): + # A field that captures a `name` but does not parse into a tool call must be left + # alone, even when the capture happens to match a tool: its keys are not arguments. + template = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "citation": { + "open_pattern": r"\w+)>", + "close": "", + "content": "xml-inline", + "content_args": { + "tag_pattern": r"<(?P\w+)>\s*(?P.*?)\s*", + "value_parser": {"name": "json", "args": {"allow_non_json": True}}, + }, + "transform": {"source": "{name}", "fields": "{content}"}, + }, + }, + } + model_out = "7" + expected = {"source": "set_alarm", "fields": {"label": 1.5, "hour": 7}} + self.assertEqual(parse_response(model_out, template, prefix="")["citation"], expected) + self.assertEqual( + parse_response(model_out, template, prefix="", tools=_SET_ALARM_TOOLS)["citation"], + expected, + ) + + def test_merge_duplicates_arguments_are_cast_element_wise(self): + # merge_duplicates collects repeated tags into a list, which is cast element-wise + template: dict[str, Any] = copy.deepcopy(_XML_STRING_ARGS_TEMPLATE) + template["fields"]["tool_calls"]["content_args"]["merge_duplicates"] = True + model_out = ( + "\n\n" + "\n7\n\n" + "\n9\n\n" + "\n" + ) + self.assertEqual(_first_tool_args(parse_response(model_out, template, prefix="")), {"hour": ["7", "9"]}) + self.assertEqual( + _first_tool_args(parse_response(model_out, template, prefix="", tools=_SET_ALARM_TOOLS)), + {"hour": [7, 9]}, + ) + # Elements that don't cast, and non-string elements, are left as they are. + parser = _parser_with_tools(_SET_ALARM_TOOLS) + call: dict[str, Any] = { + "type": "function", + "function": { + "name": "set_alarm", + "arguments": {"hour": ["7", "x", 9]}, + }, + } + parser._coerce_tool_calls(call) + self.assertEqual(call["function"]["arguments"], {"hour": [7, "x", 9]}) + + def test_already_decoded_array_is_not_cast_element_wise(self): + tools = _set_alarm_tools(groups={"type": "array", "items": {"type": "string"}}) + call: dict[str, Any] = { + "type": "function", + "function": { + "name": "set_alarm", + "arguments": {"groups": ["[1,2]", "[]"]}, + }, + } + _parser_with_tools(tools)._coerce_tool_calls(call) + self.assertEqual(call["function"]["arguments"], {"groups": ["[1,2]", "[]"]}) + + def test_coerce_tool_calls_ignores_unusable_function_name(self): + # A transform can hand us a name parsed from model output, so a non-string name + # must be ignored rather than raising on the schema lookup. + parser = _parser_with_tools(_SET_ALARM_TOOLS) + call: dict[str, Any] = { + "type": "function", + "function": {"name": ["set_alarm"], "arguments": {"hour": "7"}}, + } + self.assertEqual(parser._coerce_tool_calls(call), call) + self.assertEqual(call["function"]["arguments"], {"hour": "7"}) + + def test_union_with_container_still_keeps_scalar_text(self): + model_out = "\n\n\n1.50\n\n" + union = _set_alarm_tools(label={"anyOf": [{"type": "string"}, {"type": "object"}]}) + # `string` never casts, so the union keeps scalar-looking text as text ... + self.assertEqual( + _first_tool_args(parse_response(model_out, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=union)), + {"label": "1.50"}, + ) + # ... while the `object` branch still decodes a body that really is an object. + object_body = '\n\n\n{"a": 1}\n\n' + self.assertEqual( + _first_tool_args(parse_response(object_body, _XML_STRING_ARGS_TEMPLATE, prefix="", tools=union)), + {"label": {"a": 1}}, + ) + + def test_tools_is_a_noop_for_typed_json_tool_calls(self): + # JSON tool-call bodies already carry their types, so passing tools= must not change them. + model_out = ( + "<|START_THINKING|>x<|END_THINKING|>" + '<|START_ACTION|>[{"tool_call_id": "0", "tool_name": "set_alarm", ' + '"parameters": {"hour": 7, "enabled": true}}]<|END_ACTION|><|END_OF_TURN_TOKEN|>' + ) + without = parse_response(model_out, cohere_template, prefix="") + with_tools = parse_response(model_out, cohere_template, prefix="", tools=_SET_ALARM_TOOLS) + self.assertEqual(without, with_tools) + self.assertEqual(_first_tool_args(with_tools), {"hour": 7, "enabled": True}) + + +if __name__ == "__main__": + unittest.main() +# fmt: on diff --git a/vllm/parser/chat_parsing/__init__.py b/vllm/parser/chat_parsing/__init__.py new file mode 100644 index 0000000000..fad8a287e6 --- /dev/null +++ b/vllm/parser/chat_parsing/__init__.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright 2026 The HuggingFace Team. All rights reserved. +# ruff: noqa +# fmt: off +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Response parsing: convert model-emitted text into the assistant-message +dict used by chat templates, driven by a declarative `response_template` spec.""" + +from .response_parser import ResponseParser, parse_response + + +__all__ = ["ResponseParser", "parse_response"] +# fmt: on diff --git a/vllm/parser/chat_parsing/content_parsers.py b/vllm/parser/chat_parsing/content_parsers.py new file mode 100644 index 0000000000..21bd8447a9 --- /dev/null +++ b/vllm/parser/chat_parsing/content_parsers.py @@ -0,0 +1,255 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright 2026 The HuggingFace Team. All rights reserved. +# ruff: noqa +# fmt: off +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""This file contains the parsers used by chat response parsing. Each parser takes a chunk of captured text +and parses it into a single key in the output message dictionary. Functions are generally boilerplate.""" + +from __future__ import annotations + +import json +from typing import Any + +import regex as re + + +def _text(text: str, args: dict) -> str: + return text.strip() if args.get("strip", True) else text + + +def _int(text: str, args: dict) -> int: + return int(_text(text, args)) + + +def _float(text: str, args: dict) -> float: + return float(_text(text, args)) + + +def _bool(text: str, args: dict) -> bool: + return _text(text, args).lower() in ("true", "1") + + +# Sentinel characters for lax-JSON string pre-extraction — ASCII control chars +# that should never appear in real LLM output. +_LAX_OPEN, _LAX_CLOSE = "\x01", "\x02" + + +def _json(text: str, args: dict) -> Any: + """JSON parser with optional dialect knobs for LLM-emitted quirks. + + `args`: + - `unquoted_keys` (bool): quote bare-identifier keys before parsing. + - `string_delims` ([[open, close], ...]): strings delimited by these + custom markers are pre-extracted, then restored as standard JSON strings. + - `allow_non_json` (bool): return stripped text if parsing fails. + """ + string_delims = args.get("string_delims", []) + unquoted_keys = args.get("unquoted_keys", False) + + if string_delims and (_LAX_OPEN in text or _LAX_CLOSE in text): + raise ValueError("json: input contains reserved sentinel characters (\\x01/\\x02); cannot parse safely.") + + working = text + captured: list[str] = [] + for open_d, close_d in string_delims: + pattern = re.escape(open_d) + r"(.*?)" + re.escape(close_d) + + def _capture(m: Any) -> str: + captured.append(m.group(1)) + return f"{_LAX_OPEN}{len(captured) - 1}{_LAX_CLOSE}" + + working = re.sub(pattern, _capture, working, flags=re.DOTALL) + + if unquoted_keys: + working = re.sub(r"(?<=[{,])(\w+):", r'"\1":', working) + + for i, s in enumerate(captured): + working = working.replace(f"{_LAX_OPEN}{i}{_LAX_CLOSE}", json.dumps(s)) + + try: + return json.loads(working) + except json.JSONDecodeError as e: + if args.get("allow_non_json"): + return _text(text, args) + if working == text: + raise ValueError(f"json parser could not parse region as JSON.\nContent: {text!r}\nError: {e}") from e + raise ValueError( + f"json: could not parse after dialect transforms.\n" + f"Original: {text!r}\nTransformed: {working!r}\nError: {e}" + ) from e + + +def _sub_parse(raw: str, value_parser: dict | None) -> Any: + if value_parser is None: + return raw + return parse_content(raw, value_parser.get("name", "text"), value_parser.get("args", {})) + + +def _xml_inline(text: str, args: dict) -> dict: + """Parse shallow XML-ish tags into a dict. `tag_pattern` regex must have named + groups `key` and `value`. Optional `value_parser` recurses; `merge_duplicates` + collects duplicate keys into a list.""" + tag_pattern = args.get("tag_pattern") + if tag_pattern is None: + raise ValueError("xml-inline: 'tag_pattern' content_arg is required") + value_parser = args.get("value_parser") + merge = args.get("merge_duplicates", False) + + out: dict[str, Any] = {} + for m in re.finditer(tag_pattern, text, flags=re.DOTALL): + groups = m.groupdict() + key = groups.get("key") + if key is None: + raise ValueError(f"xml-inline: tag_pattern must have a named group 'key'. Pattern: {tag_pattern}") + value = _sub_parse(groups.get("value", ""), value_parser) + if key in out and merge: + if not isinstance(out[key], list): + out[key] = [out[key]] + out[key].append(value) + else: + out[key] = value + return out + + +def _kv_lines(text: str, args: dict) -> dict: + """Parse line-delimited `keyvalue` pairs into a dict.""" + line_sep = args.get("line_sep", "\n") + kv_sep = args.get("kv_sep", ":") + value_parser = args.get("value_parser") + + out: dict[str, Any] = {} + for line in text.split(line_sep): + line = _text(line, args) + if not line or kv_sep not in line: + continue + k, v = line.split(kv_sep, 1) + k, v = _text(k, args), _text(v, args) + out[k] = _sub_parse(v, value_parser) + return out + + +CONTENT_PARSERS = { + "text": _text, + "int": _int, + "float": _float, + "bool": _bool, + "json": _json, + "xml-inline": _xml_inline, + "kv-lines": _kv_lines, +} + +# Parsers whose output is the verbatim body text (modulo whitespace) — chunks +# from these fields stream with `dirty=False` because each chunk is part of +# the final value. Structured parsers (`json`, `xml-inline`, `kv-lines`) only +# produce a meaningful value on close, so their chunks stream raw bytes +# flagged `dirty=True` while the parsed value is delivered in `region_close`. +STREAMABLE_PARSERS = frozenset({"text", "int", "float", "bool"}) + + +def parse_content(text: str, name: str, args: dict) -> Any: + return CONTENT_PARSERS[name](text, args) + + +_PLACEHOLDER = re.compile(r"\{(\w+(?:\.\w+)*)\}") + + +def _apply_transform(transform: Any, scope: dict) -> Any: + """Recursively walk a transform template, which is used to restructure + parsed output into the actual shape we want. A dotted placeholder like + `{content.args}` descends into keys of the looked-up value.""" + if isinstance(transform, dict): + return {k: _apply_transform(v, scope) for k, v in transform.items()} + if isinstance(transform, list): + return [_apply_transform(v, scope) for v in transform] + if not isinstance(transform, str): + return transform + whole = _PLACEHOLDER.fullmatch(transform) + if not whole: + return transform + path = whole.group(1) + root, *keys = path.split(".") + if root not in scope: + raise KeyError(f"transform placeholder '{{{path}}}' is not defined. Available: {sorted(scope)}") + value = scope[root] + for key in keys: + if not isinstance(value, dict): + raise ValueError(f"transform placeholder '{{{path}}}' cannot index into {type(value).__name__} at '{key}'") + if key not in value: + raise ValueError(f"transform placeholder '{{{path}}}' is missing key '{key}'. Available: {sorted(value)}") + value = value[key] + return value + + +def validate_transform_strings(scope: str, transform: Any) -> None: + """Walk a transform template and reject any string that mixes a `{name}` + placeholder with literal text. Only whole-string placeholders (e.g. + `"{content}"`) and plain literals are supported. Called from the template + loader so authors get a clear error at load time, not at parse time.""" + if isinstance(transform, dict): + for v in transform.values(): + validate_transform_strings(scope, v) + return + if isinstance(transform, list): + for v in transform: + validate_transform_strings(scope, v) + return + if not isinstance(transform, str): + return + if _PLACEHOLDER.search(transform) and not _PLACEHOLDER.fullmatch(transform): + raise ValueError( + f"{scope}: transform string {transform!r} mixes a {{placeholder}} with literal text. " + 'Use either a whole-string placeholder (e.g. "{content}") or a plain literal; ' + "string interpolation is not supported." + ) + + +def process_field(body: str, field, captures: dict) -> Any: + """Run `body` through the field's content parser, then optionally apply the + transform template. When `transform_each` is set, the parsed content must + be a list and the template is applied to each element (with the element's + keys unpacked into the template scope, alongside any regex captures). + + `field` is a `spec.Field`; typed via duck-typing to avoid a cyclic import.""" + value = parse_content(body, field.content, field.content_args) + if field.transform is None: + return value + if field.transform_each: + if not isinstance(value, list): + raise ValueError( + f"Field '{field.name}': transform_each requires the parsed content to be a list, " + f"got {type(value).__name__}." + ) + out = [] + for item in value: + if not isinstance(item, dict): + raise ValueError( + f"Field '{field.name}': transform_each requires each list element to be a dict, " + f"got {type(item).__name__}." + ) + out.append(_apply_transform(field.transform, {**captures, **item})) + return out + return _apply_transform(field.transform, {**captures, "content": value}) + + +__all__ = [ + "CONTENT_PARSERS", + "STREAMABLE_PARSERS", + "parse_content", + "process_field", + "validate_transform_strings", +] +# fmt: on diff --git a/vllm/parser/chat_parsing/response_parser.py b/vllm/parser/chat_parsing/response_parser.py new file mode 100644 index 0000000000..b739572759 --- /dev/null +++ b/vllm/parser/chat_parsing/response_parser.py @@ -0,0 +1,408 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright 2026 The HuggingFace Team. All rights reserved. +# ruff: noqa +# fmt: off +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import copy +import json +from typing import Any + +from .content_parsers import STREAMABLE_PARSERS, process_field +from .response_templates import ( + ResponseTemplate, + ResponseTemplateField, + load_response_template, +) + + +def _schema_types(schema: Any) -> tuple[str, ...]: + if not isinstance(schema, dict): + return () + declared = schema.get("type") + types = [declared] if isinstance(declared, str) else [] + if isinstance(declared, list): + types.extend(t for t in declared if isinstance(t, str)) + for union_name in ("anyOf", "oneOf"): + for choice in schema.get(union_name) or []: + types.extend(_schema_types(choice)) + if schema.get("nullable") and "null" not in types: # `nullable` is how get_json_schema marks Optionals + types.append("null") + return tuple(types) + + +def _coerce(raw: str, types: tuple[str, ...]) -> Any: + for type_name in types: + try: + if type_name == "integer": + return int(raw) + if type_name == "number": + number = float(raw) + if number != number or number in (float("inf"), float("-inf")): + continue # NaN / inf are not valid JSON numbers + # Preserve ints when the source text had no fractional part. + return int(number) if number.is_integer() and "." not in raw else number + if type_name == "boolean" and raw.strip().lower() in ("true", "1", "false", "0"): + return raw.strip().lower() in ("true", "1") + if type_name == "null" and raw.strip() in ("null", "None"): + return None + if type_name in ("object", "array"): + decoded = json.loads(raw) + if isinstance(decoded, dict if type_name == "object" else list): + return decoded + except ValueError: + continue + return raw # `string` params, unknown types and failed casts all keep the original text + + +def parse_response( + text: str, + response_template: dict | ResponseTemplate, + *, + prefix: str | None = None, + tools: list[dict] | None = None, +) -> dict: + """The main function for response parsing when you don't want streaming. Takes generated output + and the prompt prefix and parses them without streaming any events, then returns the parsed message. + + Pass OpenAI-style `tools` dictionaries to cast tool-call arguments + using the calling tool's JSON schema. + """ + response_template = load_response_template(response_template) + stream = ResponseParser(response_template, prefix=prefix, tools=tools) + stream.feed(text) + message, _ = stream.finalize() + return message + + +class ResponseParser: + """This class implements a streaming parser with a `response_template`. If you don't need streaming and + just want to parse a complete message, use the `parse_response` function above. Streaming parsing emits + events indicating when regions (message fields) are opened and closed, with the model writing to the region + that is currently open. + + Usage: + parser = ResponseParser(response_template, prefix=chat_prompt) + for event in parser.initial_events: + handle(event) + for chunk in model_text_stream: + for event in parser.feed(chunk): + handle(event) + message, final_events = parser.finalize() + for event in final_events: + handle(event) + + Pass OpenAI-style `tools=` dictionaries to cast tool-call arguments + using the calling tool's JSON schema as each region closes. + + Events can be either "region_open", "region_chunk", or "region_close". + + ResponseParser requires the chat `prefix` (i.e. the chat history, the prefill before the current generation). + This is because chat templates or assistant prefills can sometimes write part of the message, and if we + only see the model output, and not the template, then we can't reliably parse the message in those cases. + Any events produced while consuming the prefix are exposed as `initial_events`, so renderers can show + prefill regions before the model writes anything; closed prefill regions also land in the output dict. + """ + + def __init__( + self, + response_template: dict | ResponseTemplate, + prefix: str | None = None, + *, + tools: list[dict] | None = None, + ): + self._spec = load_response_template(response_template) + if prefix is None: + raise ValueError( + "`ResponseParser`/`parse_response` requires `prefix` (the chat prompt sent to the model before " + "generation), because chat templates often pre-write part of the assistant message (e.g. an " + "opening `` tag) that the parser must see to parse the output correctly. If the generation " + 'already contains the complete message, pass `prefix=""` to opt out explicitly.' + ) + # Maps tool name -> schema `properties`, used to cast parsed tool-call arguments + self._tool_params: dict[str, dict] = {} + for tool in tools or []: + fn = tool.get("function", tool) if isinstance(tool, dict) else None + if isinstance(fn, dict) and isinstance(fn.get("name"), str): + parameters = fn.get("parameters") + properties = parameters.get("properties") if isinstance(parameters, dict) else None + self._tool_params[fn["name"]] = properties if isinstance(properties, dict) else {} + self._buffer: str = "" + self._pos: int = 0 + self._output: dict[str, Any] = copy.deepcopy(self._spec.defaults) + self._implicit_name: str | None = self._spec.implicit + # Unified current-region state: starts in the implicit region (or a + # null sink if none was declared), and returns there after every close. + # For explicit regions `_opened` flips to True eagerly on the open + # match; for the implicit region it flips lazily on the first byte. + self._current: str | None = self._implicit_name + self._captures: dict[str, str] = {} + self._body: str = "" + self._opened: bool = False + self._finalized: bool = False + self.initial_events: list[dict] = [] + if prefix: + self._consume_prefix(prefix) + + def _consume_prefix(self, prefix: str) -> None: + """Loads the prefix (the chat prefill sent to the model), right-truncates it to the start of the + assistant message (as determined by start_anchor) and then runs the remainder through the parser. + Events produced while processing the prefix are stashed on `initial_events` so callers can replay + them into a renderer before feeding model output. + + Think of this as the "get the parser up to speed on the story so far" method. + """ + truncated = self._spec.truncate_past_last_anchor(prefix) + if not truncated: + return + self._buffer = truncated + self._process(self.initial_events, eos=False) + + def feed(self, text: str) -> list[dict]: + """Feeds more text/tokens from the model output into the tokenizer, and returns any events that result + (regions entered or left). This is the method you want to call after each generation step.""" + if self._finalized: + raise RuntimeError("ResponseParser already finalized") + if text: + self._buffer += text + events: list[dict] = [] + self._process(events, eos=False) + return events + + def finalize(self) -> tuple[dict, list[dict]]: + """Close the stream and return the final message dict together with + any finalization events. This is necessary because some regions may only + end at the end of the sequence, so you won't see the event telling you they're + ready until the sequence is finalized.""" + + def _is_empty(v: Any) -> bool: + return v is None or (isinstance(v, (list, dict, str)) and not v) + + if self._finalized: + raise RuntimeError("ResponseParser already finalized") + events: list[dict] = [] + self._process(events, eos=True) + missing = [n for n, f in self._spec.fields.items() if not f.optional and n not in self._output] + if missing: + raise ValueError(f"Required response_template fields missing from parsed output: {missing}") + defaults = self._spec.defaults + self._output = {k: v for k, v in self._output.items() if k in defaults or not _is_empty(v)} + self._finalized = True + return self._output, events + + def _process(self, events: list[dict], eos: bool) -> None: + while True: + watch = self._watchlist() + best, hold_start = self._scan(watch, eos) + + if best is not None: + kind, field, m = best + if m.start() > self._pos: + self._accumulate(events, self._buffer[self._pos : m.start()]) + self._pos = m.end() + if kind == "open": + self._close_current(events) + self._open_explicit(events, field, m) + else: # "close" (always the implicit region's close here, + # since explicit regions only expose their own close) + had_content = self._opened + self._close_current(events) + # Zero-width close on an already-empty region would just + # re-fire next iteration -- bail out to make progress. + if not had_content and m.start() == m.end(): + break + continue + + # No committable match in the current buffer. + if eos: + if self._pos < len(self._buffer): + self._accumulate(events, self._buffer[self._pos :]) + self._pos = len(self._buffer) + self._close_current(events) + break + # Stream everything up to the earliest still-pending delimiter. When + # nothing is pending `hold_start == len(self._buffer)`, so this flushes + # the whole buffer; otherwise we hold the (possibly partial) delimiter + # bytes back until more input resolves them. + if hold_start > self._pos: + self._accumulate(events, self._buffer[self._pos : hold_start]) + self._pos = hold_start + break + + def _watchlist(self) -> list[tuple[str, ResponseTemplateField]]: + """Patterns we care about right now: the close of the currently-open + explicit region, or -- if we're in the implicit/null region -- every + explicit open plus the implicit's own close (if any).""" + if self._current is not None and self._current != self._implicit_name: + field = self._spec.fields[self._current] + return [("close", field)] if field.close_re is not None else [] + watch: list[tuple[str, ResponseTemplateField]] = [] + for field in self._spec.fields.values(): + if field.open_re is not None: + watch.append(("open", field)) + if self._implicit_name is not None: + impl = self._spec.fields[self._implicit_name] + if impl.close_re is not None: + watch.append(("close", impl)) + return watch + + def _scan( + self, watch: list[tuple[str, ResponseTemplateField]], eos: bool + ) -> tuple[tuple[str, ResponseTemplateField, Any] | None, int]: + """Single pass over the watched delimiters, using the `regex` module's + partial matching to decide -- per delimiter -- whether it can be committed + now or must be held. Returns `(best, hold_start)`: + + * `best` is the earliest-starting delimiter we can safely commit *now* + (longest on ties, opens before closes), or `None`. + * `hold_start` is the leftmost buffer position occupied by a still-pending + match: a partial (incomplete) delimiter, or a complete one ending at the + buffer edge that more input could still grow. Bytes before it are safe to + emit; bytes from it onward must be held. It stays `len(self._buffer)` when + nothing is pending, letting the caller flush the whole buffer. + + A complete match is committable only if it starts strictly before + `hold_start` -- otherwise an earlier (or co-located) pending delimiter could + turn out to be the real one. At EOS nothing can grow, so partial matching is + skipped and every complete match is committable. + + (The `regex` module always reports the empty string as a live prefix, so a + partial search with no real match returns a zero-width match at the buffer + end; that lands in the pending branch with `start == len(self._buffer)`, a + no-op for `hold_start`.) + """ + best_key: tuple | None = None + best: tuple[str, ResponseTemplateField, Any] | None = None + hold_start = len(self._buffer) + for kind, field in watch: + # The watchlist only includes fields whose delimiter regex is set, so pattern is never None here. + pattern = field.open_re if kind == "open" else field.close_re + if eos: + m = pattern.search(self._buffer, self._pos) + else: + m = pattern.search(self._buffer, self._pos, partial=True) + if m is None: + continue + if not eos and (m.partial or self._can_grow(kind, field, m)): + # Pending: can't commit, and blocks emitting from its start onward. + hold_start = min(hold_start, m.start()) + continue + key = (m.start(), -(m.end() - m.start()), 0 if kind == "open" else 1, field.name) + if best_key is None or key < best_key: + best_key, best = key, (kind, field, m) + # A committable match co-located with or after a pending one must wait too: + # the pending delimiter starts no later and might be the one that fires. + if best is not None and best[2].start() >= hold_start: + best = None + return best, hold_start + + def _can_grow(self, kind: str, field: ResponseTemplateField, m: Any) -> bool: + r"""Whether a *complete* match ending at the current buffer edge could still + change as more input arrives -- in which case we defer rather than commit. A + match ending before the edge has already seen its terminating byte and is + final. At the edge: zero-width matches (`$` / `\Z`) are only real at true + EOS; a fully-present literal that no other literal in its set extends cannot + grow (the fast path that keeps literal delimiters zero-latency); anything + else (regex delimiters, prefix-overlapping literal lists) might.""" + if m.end() != len(self._buffer): + return False + if m.start() == m.end(): + return True + literals, can_extend = ( + (field.open_literals, field.open_literal_can_extend) + if kind == "open" + else (field.close_literals, field.close_literal_can_extend) + ) + return literals is None or can_extend + + def _accumulate(self, events: list[dict], text: str) -> None: + """Route `text` into the currently active region. When the current + region is the null sink (no implicit declared, no explicit open), we + silently discard. Every routed chunk emits a `region_chunk` event so + consumers can render live; `dirty=True` flags chunks from structured + parsers (json, xml-inline, kv-lines) whose raw bytes will only be + parsed into the final value on close.""" + if not text or self._current is None: + return + field = self._spec.fields[self._current] + if not self._opened: + events.append({"type": "region_open", "field": self._current}) + self._opened = True + self._body += text + dirty = field.content not in STREAMABLE_PARSERS + events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty}) + + def _open_explicit(self, events: list[dict], field: ResponseTemplateField, m: Any) -> None: + self._current = field.name + self._captures = {k: v for k, v in m.groupdict().items() if v is not None} + self._body = "" + self._opened = True + events.append({"type": "region_open", "field": field.name}) + + def _close_current(self, events: list[dict]) -> None: + """Close the current region and reset to the implicit/null region. + Skipped (aside from the reset) when the current region never opened -- + avoids vacuous open/close pairs at every explicit boundary.""" + if self._current is None or not self._opened: + self._reset_to_implicit() + return + field = self._spec.fields[self._current] + value = process_field(self._body, field, self._captures) + if self._tool_params: + value = self._coerce_tool_calls(value) + if field.join is not None: + if not isinstance(value, str): + raise ValueError( + f"Field '{field.name}': 'join' requires each match to parse to a string, " + f"got {type(value).__name__}." + ) + previous = self._output.get(self._current) + self._output[self._current] = value if previous is None else previous + field.join + value + elif field.repeats: + self._output.setdefault(self._current, []).append(value) + else: + self._output[self._current] = value + events.append({"type": "region_close", "field": self._current, "value": value}) + self._reset_to_implicit() + + def _coerce_tool_calls(self, value: Any) -> Any: + if isinstance(value, list): + return [self._coerce_tool_calls(item) for item in value] + fn = value.get("function") if isinstance(value, dict) else None + if not isinstance(fn, dict): + return value + name, arguments = fn.get("name"), fn.get("arguments") + if not isinstance(name, str) or not isinstance(arguments, dict): + return value + if properties := self._tool_params.get(name): + for key, argument in arguments.items(): + if key not in properties or not (types := _schema_types(properties[key])): + continue + if isinstance(argument, str): + arguments[key] = _coerce(argument, types) + elif isinstance(argument, list) and "array" not in types: + # duplicate keys collected by `merge_duplicates` + arguments[key] = [_coerce(item, types) if isinstance(item, str) else item for item in argument] + return value + + def _reset_to_implicit(self) -> None: + self._current = self._implicit_name + self._captures = {} + self._body = "" + self._opened = False +# fmt: on diff --git a/vllm/parser/chat_parsing/response_templates.py b/vllm/parser/chat_parsing/response_templates.py new file mode 100644 index 0000000000..2636e32f71 --- /dev/null +++ b/vllm/parser/chat_parsing/response_templates.py @@ -0,0 +1,249 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Copyright 2026 The HuggingFace Team. All rights reserved. +# ruff: noqa +# fmt: off +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Template loading and validation for response_template dicts.""" + +from __future__ import annotations + +import copy +import logging +from dataclasses import dataclass +from typing import Any + +import regex as re + +from .content_parsers import CONTENT_PARSERS, validate_transform_strings + + +logger = logging.getLogger(__name__) + + +@dataclass +class ResponseTemplateField: + name: str + open_re: Any + open_literals: list[str] | None + open_literal_can_extend: bool + close_re: Any + close_literals: list[str] | None + close_literal_can_extend: bool + content: str + content_args: dict + repeats: bool + join: str | None + optional: bool + transform: Any + transform_each: bool + + +@dataclass +class ResponseTemplate: + defaults: dict + fields: dict[str, ResponseTemplateField] + start_anchor_re: Any + implicit: str | None = None + start_anchor_literals: list[str] | None = None + + def truncate_past_last_anchor(self, text: str) -> str: + """When parsing responses, we can't just parse the tokens generated + by the model. This is because chat templates often include early parts of the message such as + or tokens in the prefill. Therefore, we take the whole chat as input, but truncate to the start + of the most recent assistant message, which can include tokens from the template as well as the output.""" + last_end: int | None = None + for m in self.start_anchor_re.finditer(text): + last_end = m.end() + if last_end is None: + kind = "start_anchor" if self.start_anchor_literals is not None else "start_anchor_pattern" + logger.info( + "response_template defines %s but the anchor was not found " + "in the prefix; the parser will process the entire prefix instead.", + kind, + ) + return text + return text[last_end:] + + +def _compile_anchor(scope: str, field: dict, literal_key: str, pattern_key: str) -> tuple[Any, list[str] | None, bool]: + """Compile a region's start/end anchor. An anchor can be: + + - a regex (`pattern_key`), + - a single literal string (`literal_key`), or + - a list of literal strings (`literal_key`), in which case any of these match + + All forms compile to a regex for matching, but we keep the literal strings around too: when present, + the parser can emit regions faster, since it knows exactly how many trailing bytes might be a + partial-match prefix (and that a finished literal match can never be extended by future bytes, except + when a literal is a strict prefix of another in the same list). + + Returns `(compiled_re, literals, can_extend)`: + + - `compiled_re`: the compiled pattern. + - `literals`: the literal strings, or `None` for a regex anchor. + - `can_extend`: `True` iff one literal is a strict prefix of another, in which case a match at the + buffer edge could still be lengthened by future input. + """ + if literal_key in field and pattern_key in field: + raise ValueError(f"{scope}: cannot specify both '{literal_key}' and '{pattern_key}'") + if literal_key in field: + raw = field[literal_key] + if isinstance(raw, str): + literals = [raw] + elif isinstance(raw, list): + if not raw: + raise ValueError(f"{scope}: '{literal_key}' list must contain at least one literal") + if not all(isinstance(s, str) for s in raw): + raise ValueError(f"{scope}: '{literal_key}' list must contain only strings") + literals = list(dict.fromkeys(raw)) # dedupe, preserve first-seen order + else: + raise ValueError(f"{scope}: '{literal_key}' must be a string or list of strings, got {type(raw).__name__}") + if any(s == "" for s in literals): + raise ValueError(f"{scope}: '{literal_key}' literals cannot be empty strings") + # Sort longest-first so alternation prefers the longer alternative when both could match. + ordered = sorted(literals, key=len, reverse=True) + can_extend = any(a != b and a.startswith(b) for a in literals for b in literals) + pattern = "|".join(re.escape(s) for s in ordered) + return re.compile(pattern, re.DOTALL), literals, can_extend + if pattern_key in field: + try: + return re.compile(field[pattern_key], re.DOTALL), None, False + except re.error as e: + raise ValueError(f"{scope}: invalid {pattern_key} regex: {e}") from e + return None, None, False + + +def _validate_template_shape(spec: dict) -> None: + """Validate the top-level response_template structure: it must be a dict of the expected version with + only known keys, a dict of `defaults`, and a non-empty dict of `fields`. Per-field specs are checked + separately in `_build_field`.""" + if not isinstance(spec, dict): + raise ValueError(f"response_template must be a dict, got {type(spec).__name__}") + version = spec.get("version", 1) + if version != 1: + raise ValueError(f"Unsupported response_template version: {version}") + if unknown_template_keys := set(spec) - {"version", "defaults", "fields", "start_anchor", "start_anchor_pattern"}: + raise ValueError(f"Unknown keys in response_template: {sorted(unknown_template_keys)}") + if not isinstance(spec.get("defaults", {}), dict): + raise ValueError("response_template.defaults must be a dict") + fields_raw = spec.get("fields", {}) + if not isinstance(fields_raw, dict) or not fields_raw: + raise ValueError("response_template.fields must be a non-empty dict") + + +def _build_field(name: str, field: dict) -> ResponseTemplateField: + """Validate a single field spec and compile it into a `ResponseTemplateField`.""" + + _ALLOWED_FIELD_KEYS = { + "open", + "open_pattern", + "close", + "close_pattern", + "content", + "content_args", + "repeats", + "join", + "optional", + "transform", + "transform_each", + } + scope = f"Field '{name}'" + if not isinstance(field, dict): + raise ValueError(f"{scope} must be a dict") + if unknown_field_keys := set(field) - _ALLOWED_FIELD_KEYS: + raise ValueError(f"{scope}: unknown keys {sorted(unknown_field_keys)}") + content = field.get("content", "text") + if content not in CONTENT_PARSERS: + raise ValueError(f"{scope}: unknown content parser '{content}'. Available: {sorted(CONTENT_PARSERS)}") + open_re, open_literals, open_literal_can_extend = _compile_anchor(scope, field, "open", "open_pattern") + close_re, close_literals, close_literal_can_extend = _compile_anchor(scope, field, "close", "close_pattern") + join = field.get("join") + if join is not None and not isinstance(join, str): + raise ValueError(f"{scope}: 'join' must be a string, got {type(join).__name__}") + if join is not None and not field.get("repeats", False): + raise ValueError(f"{scope}: 'join' requires 'repeats': true") + transform = field.get("transform") + transform_each = field.get("transform_each", False) + if not isinstance(transform_each, bool): + raise ValueError(f"{scope}: transform_each must be a bool, got {type(transform_each).__name__}") + if transform_each and transform is None: + raise ValueError(f"{scope}: transform_each is set but no transform was provided") + if transform is not None: + validate_transform_strings(scope, transform) + else: + # Named captures only reach the output through a transform, so flag any that would be silently dropped. + captured_names = set() + if open_re is not None: + captured_names |= set(open_re.groupindex) + if close_re is not None: + captured_names |= set(close_re.groupindex) + if captured_names: + raise ValueError( + f"{scope}: open_pattern/close_pattern declares named group(s) " + f"{sorted(captured_names)}, but the field has no 'transform'. Named captures " + f"are only surfaced through a 'transform' template (where they appear " + f"alongside 'content'). Either add a 'transform' that uses the captures, or " + f"remove the named groups from the pattern." + ) + return ResponseTemplateField( + name=name, + open_re=open_re, + open_literals=open_literals, + open_literal_can_extend=open_literal_can_extend, + close_re=close_re, + close_literals=close_literals, + close_literal_can_extend=close_literal_can_extend, + content=content, + content_args=field.get("content_args", {}), + repeats=field.get("repeats", False), + join=join, + optional=field.get("optional", True), + transform=transform, + transform_each=transform_each, + ) + + +def load_response_template(spec: dict | ResponseTemplate) -> ResponseTemplate: + if isinstance(spec, ResponseTemplate): + return spec + _validate_template_shape(spec) + + fields = {name: _build_field(name, raw) for name, raw in spec["fields"].items()} + # A field without an open anchor is the implicit-open / leftover sink; at most one is allowed. + implicit_fields = [name for name, field in fields.items() if field.open_re is None] + if len(implicit_fields) > 1: + raise ValueError( + "At most one field may omit 'open'/'open_pattern' (that field becomes the " + f"implicit-open / leftover sink). Found: {', '.join(implicit_fields)}" + ) + + start_anchor_re, start_anchor_literals, _ = _compile_anchor( + "response_template", spec, "start_anchor", "start_anchor_pattern" + ) + if start_anchor_re is None: + raise ValueError("response_template must define 'start_anchor' or 'start_anchor_pattern'.") + + return ResponseTemplate( + defaults=copy.deepcopy(spec.get("defaults", {})), + fields=fields, + implicit=implicit_fields[0] if implicit_fields else None, + start_anchor_re=start_anchor_re, + start_anchor_literals=start_anchor_literals, + ) + + +__all__ = ["ResponseTemplate", "ResponseTemplateField", "load_response_template"] +# fmt: on From 0f8b398158f01fb3ea0e81b5adc93338fdf3db9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Rialland?= <36076211+TQCB@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:22:10 +0200 Subject: [PATCH 138/248] [watermarking] add context deduplication support to speculative decoding (#56807) Signed-off-by: Raphael Rialland Signed-off-by: Simon Veitner Co-authored-by: Simon Veitner Co-authored-by: Lucas Wilkinson --- docs/features/watermarking.md | 29 +- tests/test_config.py | 25 +- .../test_rejection_sampler_utils.py | 176 ++++++ tests/watermarking/test_gumbel.py | 62 ++ tests/watermarking/test_watermarking.py | 563 +++++++++++++++++- vllm/config/vllm.py | 7 - vllm/v1/watermarking/gpu_sampler.py | 13 +- vllm/v1/watermarking/gumbel.py | 4 + vllm/v1/watermarking/spec_decode.py | 122 +++- vllm/v1/watermarking/watermarker.py | 9 +- vllm/v1/worker/gpu/model_runner.py | 6 +- vllm/v1/worker/gpu/sample/gumbel.py | 30 +- vllm/v1/worker/gpu/sample/watermark.py | 322 +++++++++- .../gpu/spec_decode/dspark/speculator.py | 12 +- .../gpu/spec_decode/rejection_sampler.py | 13 +- .../spec_decode/rejection_sampler_utils.py | 47 +- vllm/v1/worker/gpu/spec_decode/speculator.py | 27 +- 17 files changed, 1379 insertions(+), 88 deletions(-) diff --git a/docs/features/watermarking.md b/docs/features/watermarking.md index 54c7e58ce7..7ab02d3363 100644 --- a/docs/features/watermarking.md +++ b/docs/features/watermarking.md @@ -77,30 +77,17 @@ watermarked. The watermark signal is diluted in proportion to the share of output tokens supplied by accepted drafts; rejected drafts do not dilute it because their recovery tokens are watermarked. -Speculative-decoding token paths do not currently support generation-side -context deduplication. The configured `deduplicate_contexts` policy is not -applied to accepted drafts, rejection-recovery tokens, or bonus tokens. - For `dual_key_gumbel`, `alpha` has no effect under speculative decoding. The speculative protocol selects the key for each token instead. -## Algorithms - -### Gumbel-max - -Gumbel-max derives a deterministic pseudorandom value from the key, prior token -context, and every candidate token, then uses the resulting Gumbel noise for -categorical sampling. See -[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf). - -Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`) -bypass watermarking and emit a warning once per worker. +## Context deduplication When a token context is repeated, generation can use ordinary sampling for that occurrence. Reusing the repeated context leads to a bias over the sequence, as certain token choices would be correlated. Ordinary sampling at these positions allows for single-sequence non-distortion (see section G.3 of the [SynthID-Text supplementary materials](https://media.springernature.com/original/springer-static/esm/art%3A10.1038%2Fs41586-024-08025-4/MediaObjects/41586_2024_8025_MOESM1_ESM.pdf)). +This also applies to drafter watermarking during speculative decoding. The detector independently deduplicates contexts so repeated keyed random vectors are not treated as independent evidence, meaning that context deduplication at generation time does not reduce the watermarking signal, unless @@ -143,6 +130,18 @@ vllm serve MODEL \ '{"algorithm":"gumbel","key":42,"deduplicate_contexts":"all"}' ``` +## Algorithms + +### Gumbel-max + +Gumbel-max derives a deterministic pseudorandom value from the key, prior token +context, and every candidate token, then uses the resulting Gumbel noise for +categorical sampling. See +[Aaronson's original presentation](https://simons.berkeley.edu/sites/default/files/2024-10/LLM24-2%20Slides%20-%20Scott%20Aaronson.pdf). + +Gumbel-max requires stochastic sampling. Greedy requests (`temperature=0`) +bypass watermarking and emit a warning once per worker. + ### Dual-key Gumbel-max Dual-key Gumbel-max derives independent keys A and B from one configured master diff --git a/tests/test_config.py b/tests/test_config.py index 30161e89fb..93578ab918 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -3526,7 +3526,6 @@ def test_target_only_gumbel_allows_speculative_decoding(caplog_vllm, disable_log config._check_watermarking_unsupported() assert "Target-only watermarking leaves accepted draft tokens" in caplog_vllm.text - assert "Context deduplication is not supported" in caplog_vllm.text def test_speculative_watermarking_without_context_dedup_does_not_warn( @@ -3543,10 +3542,32 @@ def test_speculative_watermarking_without_context_dedup_does_not_warn( parallel_drafting=False, ) + caplog_vllm.clear() with caplog_vllm.at_level(logging.WARNING): config._check_watermarking_unsupported() - assert "Context deduplication is not supported" not in caplog_vllm.text + assert "dedup" not in caplog_vllm.text.lower() + + +def test_speculative_context_dedup_logs_no_unsupported_warning( + caplog_vllm, disable_log_dedup +): + config = _watermarked_vllm_config() + config.watermark_config = WatermarkConfig( + algorithm="dual_key_gumbel", key=42, deduplicate_contexts="single_turn" + ) + config.speculative_config = SimpleNamespace( + method="mtp", + draft_sample_method="probabilistic", + rejection_sample_method="standard", + parallel_drafting=False, + ) + + caplog_vllm.clear() + with caplog_vllm.at_level(logging.WARNING): + config._check_watermarking_unsupported() + + assert "dedup" not in caplog_vllm.text.lower() def test_gumbel_rejects_speculative_decoding_without_target_only(): diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index 7fc6898142..e80f9e6d2c 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -203,6 +203,182 @@ def test_watermarked_recovery_uses_target_key(): assert sampled[0, 0] == expected.token_ids[0] +def test_repeated_draft_context_does_not_change_acceptance(): + torch.manual_seed(0) + num_trials = 8 + # A draft distribution of its own, so the drafts are not accepted outright. + target_logits = torch.randn(64, device="cuda") + draft_logits = torch.randn(64, device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + draft_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=num_trials, + ) + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 12]], device="cuda").repeat( + num_trials, 1 + ), + watermarking=torch.ones(num_trials, dtype=torch.bool, device="cuda"), + watermarking_skip_mask=torch.tensor([True, False], device="cuda").repeat( + num_trials + ), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + # Some drafts must actually be rejected, or the ordinary-residual branch + # the skip mask selects is never reached. + assert (expected_num_sampled == 1).any() + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0) + + +def test_repeated_context_skip_mask_is_read_per_row_in_a_block(): + """A block of three drafts, accepted at row 0 and rejected at row 1. + + The skip mask selects the sampler per row, so the row the target recovers + at decides which key draws the recovery token. + """ + num_speculative_steps = 3 + target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda") + draft_logits = torch.tensor([30.0, -30.0, -30.0, -30.0], device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + draft_logits, + num_speculative_steps=num_speculative_steps, + temperature=1.0, + num_trials=1, + ) + # Row 0 drafts a token the target gives 0.67 and the draft e**-60, so it is + # accepted; row 1 drafts a token the target gives no mass, so it is not. + inputs["draft_sampled"][1] = 3 + inputs["draft_sampled"][2] = 0 + contexts = torch.tensor( + [[10, 11], [11, 3], [3, 0], [0, 1]], dtype=torch.int64, device="cuda" + ) + expected, expected_num_sampled = rejection_sample( + **inputs, num_speculative_steps=num_speculative_steps + ) + assert expected_num_sampled.item() == 2 + assert expected[0, 0].item() == 3 + + watermarker = GumbelWatermarker(key=42, context_width=2) + skipped, skipped_num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=num_speculative_steps, + contexts=contexts, + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.tensor([False, True, False, False], device="cuda"), + watermarker=watermarker, + ) + + assert skipped_num_sampled.item() == 2 + torch.testing.assert_close(skipped[:, :2], expected[:, :2], rtol=0, atol=0) + + watermarked, watermarked_num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=num_speculative_steps, + contexts=contexts, + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.zeros( + num_speculative_steps + 1, dtype=torch.bool, device="cuda" + ), + watermarker=watermarker, + ) + # The residual equals the target to within e**-60, so the recovery token is + # the target key's draw on the rejected row's context. + recovery = GumbelWatermarker(key=42, context_width=2).sample( + target_logits.unsqueeze(0), contexts[1:2], lambda _: None + ) + + assert watermarked_num_sampled.item() == 2 + assert watermarked[0, 0].item() == 3 + assert watermarked[0, 1] == recovery.token_ids[0] + + +def test_repeated_recovery_context_uses_ordinary_residual_sample(): + target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0], device="cuda") + draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + draft_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=1, + ) + inputs["draft_sampled"][1] = 0 + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 0]], device="cuda"), + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.tensor([True, False], device="cuda"), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + torch.testing.assert_close(sampled[:, :1], expected[:, :1], rtol=0, atol=0) + + +def test_repeated_bonus_context_uses_ordinary_target_sample(): + target_logits = torch.randn(64, device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + target_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=1, + ) + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 12]], device="cuda"), + watermarking=torch.tensor([True], device="cuda"), + watermarking_skip_mask=torch.tensor([False, True], device="cuda"), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + torch.testing.assert_close(sampled[:, :2], expected[:, :2], rtol=0, atol=0) + + +def test_repeated_context_does_not_change_unwatermarked_or_greedy_rows(): + target_logits = torch.randn(64, device="cuda") + inputs = _build_rejection_sample_inputs( + target_logits, + target_logits, + num_speculative_steps=1, + temperature=1.0, + num_trials=2, + ) + inputs["temperature"][1] = 0 + expected, expected_num_sampled = rejection_sample(**inputs, num_speculative_steps=1) + + sampled, num_sampled = watermarked_rejection_sample( + **inputs, + num_speculative_steps=1, + contexts=torch.tensor([[10, 11], [11, 12], [20, 21], [21, 22]], device="cuda"), + watermarking=torch.tensor([False, True], device="cuda"), + watermarking_skip_mask=torch.ones(4, dtype=torch.bool, device="cuda"), + watermarker=GumbelWatermarker(key=42, context_width=2), + ) + + torch.testing.assert_close(num_sampled, expected_num_sampled, rtol=0, atol=0) + for row, length in enumerate(num_sampled): + torch.testing.assert_close( + sampled[row, : int(length)], expected[row, : int(length)], rtol=0, atol=0 + ) + + def test_watermarked_recovery_supports_smaller_draft_vocabulary(): target_logits = torch.tensor([float("-inf"), 0.0, 1.0, 2.0, 100.0], device="cuda") draft_logits = torch.tensor([100.0, -100.0, -100.0, -100.0], device="cuda") diff --git a/tests/watermarking/test_gumbel.py b/tests/watermarking/test_gumbel.py index dd32bda627..d789692a0e 100644 --- a/tests/watermarking/test_gumbel.py +++ b/tests/watermarking/test_gumbel.py @@ -12,6 +12,7 @@ from vllm.v1.watermarking import ( derive_watermark_key, ) from vllm.v1.watermarking.gumbel import _gamma_survival_integer_shape +from vllm.v1.watermarking.watermarker import RandomSampler from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.sample.watermark import philox_gumbel_sample @@ -197,3 +198,64 @@ def test_philox_gumbel_sample_skip_mask_matches_separate_samplers(use_fp64: bool ) torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_philox_mixed_draft_sampling_caches_unprocessed_logits(): + torch.manual_seed(0) + num_rows, vocab_size, num_steps = 5, 4099, 3 + logits = torch.randn(num_rows, vocab_size, device="cuda", dtype=torch.float16) + temperatures = torch.tensor([0.0, 0.3, 0.5, 1.0, 2.0], device="cuda") + processed_logits = logits / torch.where( + temperatures == 0, 1, temperatures + ).unsqueeze(-1) + contexts = torch.randint(0, vocab_size, (num_rows, 4), device="cuda") + idx_mapping = torch.arange(num_rows, dtype=torch.int32, device="cuda") + seeds = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 10 + positions = torch.arange(num_rows, dtype=torch.int64, device="cuda") + 20 + # The skipped rows must divide by a temperature other than 1, or the + # unprocessed logits the cache stores are the logits the sampler sees. + # Row 0 keeps a greedy row in the skipped set. + skip_mask = torch.tensor([True, True, False, False, True], device="cuda") + cols = torch.arange(num_rows, dtype=torch.int32, device="cuda") % num_steps + cache = torch.zeros( + num_rows, num_steps, vocab_size + 1, dtype=logits.dtype, device="cuda" + ) + + ordinary = gumbel_sample( + logits, + idx_mapping, + temperatures, + seeds, + positions, + apply_temperature=True, + is_drafting=True, + ) + watermarked = philox_gumbel_sample(processed_logits, contexts, 42) + expected = torch.where(skip_mask, ordinary, watermarked) + actual = ( + GumbelWatermarker(key=42, context_width=4) + .sample( + processed_logits, + contexts, + random_sampler=RandomSampler( + expanded_idx_mapping=idx_mapping, + temperatures=temperatures, + seeds=seeds, + positions=positions, + is_drafting=True, + logits_cache=cache, + logits_cache_col=cols, + logits_cache_source=logits, + ), + skip_mask=skip_mask, + ) + .token_ids + ) + + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + stored = cache[torch.arange(num_rows, device="cuda"), cols.long(), :vocab_size] + assert torch.equal(stored.view(torch.int16), logits.view(torch.int16)) + assert not cache[:, :, -1].any() diff --git a/tests/watermarking/test_watermarking.py b/tests/watermarking/test_watermarking.py index 232f19a9a6..03c01b2485 100644 --- a/tests/watermarking/test_watermarking.py +++ b/tests/watermarking/test_watermarking.py @@ -28,6 +28,7 @@ from vllm.v1.watermarking.spec_decode import ( from vllm.v1.watermarking.watermarker import Watermarker, WatermarkSample from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.sample.watermark import ( + draft_watermarking_mask, philox_gumbel_sample, repeated_context_mask, ) @@ -202,11 +203,17 @@ def test_speculative_decoding_uses_fixed_dual_key_roles(): max_num_reqs=1, device=torch.device("cpu"), allow_target_only=False, + num_speculative_steps=3, + deduplicate_contexts="all", + deduplicate_contexts_max_history=None, ) assert target.prf.key == derive_watermark_key(42, b"key_b") assert draft is not None assert draft.watermarker.prf.key == derive_watermark_key(42, b"key_a") + assert draft.num_speculative_steps == 3 + assert draft.deduplicate_contexts == "all" + assert draft.deduplicate_contexts_max_history is None def test_recovery_key_rejects_the_unsplit_dual_key_watermarker(): @@ -892,6 +899,187 @@ def test_gpu_sampler_builds_speculative_contexts_from_drafts(): assert torch.equal(contexts, torch.tensor([[10, 11], [11, 20], [20, 21]])) +def test_gpu_sampler_deduplicates_contexts_within_speculative_block(): + sampler = object.__new__(GPUWatermarkSampler) + sampler.watermarker = SimpleNamespace(context_width=2) + sampler.deduplicate_contexts = "single_turn" + sampler.deduplicate_contexts_max_history = None + sampler.num_speculative_tokens = 4 + sampler.req_states = SimpleNamespace( + all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 1, 2, 0, 0, 0]])), + prompt_len=SimpleNamespace(gpu=torch.tensor([1])), + total_len=SimpleNamespace(gpu=torch.tensor([3])), + ) + request_indices = torch.tensor([0, 0, 0, 0, 0]) + local_positions = torch.tensor([0, 1, 2, 3, 4]) + contexts = sampler._get_contexts( + request_indices, + local_positions, + torch.tensor([0, 3, 4, 1, 2]), + ) + + repeated = sampler._get_repeated_contexts( + request_indices, contexts, local_positions + ) + + assert torch.equal( + contexts, + torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]), + ) + assert torch.equal(repeated, torch.tensor([False, False, False, False, True])) + + sampler.deduplicate_contexts_max_history = 3 + repeated = sampler._get_repeated_contexts( + request_indices, contexts, local_positions + ) + assert not repeated.any() + + +def test_gpu_sampler_all_scope_skips_only_partial_speculative_contexts(): + sampler = object.__new__(GPUWatermarkSampler) + sampler.watermarker = SimpleNamespace(context_width=2) + sampler.deduplicate_contexts = "all" + sampler.deduplicate_contexts_max_history = None + sampler.num_speculative_tokens = 3 + sampler.req_states = SimpleNamespace( + all_token_ids=SimpleNamespace(gpu=torch.tensor([[100, 101, 10, 0, 0]])), + prompt_len=SimpleNamespace(gpu=torch.tensor([2])), + total_len=SimpleNamespace(gpu=torch.tensor([3])), + ) + request_indices = torch.tensor([0, 0, 0]) + local_positions = torch.tensor([0, 1, 2]) + contexts = sampler._get_contexts( + request_indices, + local_positions, + torch.tensor([10, 20, 30]), + ) + + repeated = sampler._get_repeated_contexts( + request_indices, contexts, local_positions + ) + + assert torch.equal(contexts, torch.tensor([[-1, 10], [10, 20], [20, 30]])) + assert torch.equal(repeated, torch.tensor([True, False, False])) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +@pytest.mark.parametrize("max_history,expected_last", [(None, True), (3, False)]) +def test_speculative_context_repetition_accelerator_parity( + max_history: int | None, expected_last: bool +): + all_token_ids = torch.tensor([[10, 11]]) + req_indices = torch.tensor([0, 0, 0, 0, 0]) + prompt_lens = torch.tensor([0]) + total_lens = torch.tensor([2]) + local_positions = torch.tensor([0, 1, 2, 3, 4]) + contexts = torch.tensor([[1, 2], [2, 3], [3, 4], [4, 1], [1, 2]]) + + expected = repeated_context_mask( + all_token_ids, + req_indices, + prompt_lens, + total_lens, + contexts, + history_offsets=local_positions, + local_positions=local_positions, + num_speculative_steps=4, + max_history=max_history, + ) + actual = repeated_context_mask( + all_token_ids.cuda(), + req_indices.cuda(), + prompt_lens.cuda(), + total_lens.cuda(), + contexts.cuda(), + history_offsets=local_positions.cuda(), + local_positions=local_positions.cuda(), + num_speculative_steps=4, + max_history=max_history, + ).cpu() + + assert torch.equal( + expected, torch.tensor([False, False, False, False, expected_last]) + ) + assert torch.equal(actual, expected) + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +@pytest.mark.parametrize("steps_layout", ["per_row", "broadcast"]) +@pytest.mark.parametrize("req_indices_layout", ["contiguous", "column_view"]) +def test_draft_context_deduplication_accelerator_parity( + steps_layout, req_indices_layout +): + """Parity with the CPU reference for the layouts the speculators pass. + + A draft step is one 0-d tensor broadcast over the batch, and DSpark samples + a step at a time, so its request indices are an `idx_map[:, i]` column. Both + are read per row, so a row that ignores the layout consults another row's + step or another request's history. + """ + num_rows, num_steps, step = 4, 3, 1 + all_token_ids = torch.tensor([[10, 11, 20, 11], [30, 31, 11, 20]]) + prompt_lens = torch.tensor([0, 0]) + total_lens = torch.tensor([4, 4]) + contexts = torch.tensor([[40, 41], [31, 11], [10, 11], [50, 51]]) + prior_contexts = torch.zeros(num_rows, num_steps, 2, dtype=torch.int64) + prior_contexts[0, 0] = torch.tensor([40, 41]) + enabled = torch.ones(num_rows, dtype=torch.bool) + req_indices = torch.tensor([0, 1, 0, 1]) + + def run(all_token_ids, req_indices, steps, prior): + return draft_watermarking_mask( + all_token_ids, + req_indices, + prompt_lens.to(all_token_ids.device), + total_lens.to(all_token_ids.device), + prior, + contexts.to(all_token_ids.device), + steps, + enabled.to(all_token_ids.device), + max_history=3, + include_prompt=False, + ) + + reference_prior = prior_contexts.clone() + reference = run( + all_token_ids, + req_indices, + torch.full((num_rows,), step), + reference_prior, + ) + + # Row 0 repeats its own step-0 context, rows 1 and 2 repeat a committed + # context of their own request, row 3 repeats nothing. + assert torch.equal(reference, torch.tensor([False, False, False, True])) + expected_prior = prior_contexts.clone() + expected_prior[:, step] = contexts + assert torch.equal(reference_prior, expected_prior) + + if steps_layout == "per_row": + steps = torch.full((num_rows,), step).cuda() + else: + steps = torch.tensor(step).cuda().expand(num_rows) + assert steps.stride(0) == 0 + + if req_indices_layout == "contiguous": + gpu_req_indices = req_indices.cuda() + else: + gpu_req_indices = ( + req_indices.repeat_interleave(num_steps).view(num_rows, num_steps).cuda() + )[:, 1] + assert gpu_req_indices.stride(0) == num_steps + + actual_prior = prior_contexts.cuda() + actual = run(all_token_ids.cuda(), gpu_req_indices, steps, actual_prior).cpu() + + assert torch.equal(actual, reference) + assert torch.equal(actual_prior.cpu(), expected_prior) + + def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts(): sampler = object.__new__(GPUWatermarkSampler) sampler.watermarker = SimpleNamespace(context_width=2) @@ -921,6 +1109,69 @@ def test_gpu_sampler_builds_chunked_multi_request_speculative_contexts(): ) +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_draft_context_deduplication_broadcasts_the_draft_step(): + """The draft step is one 0-d tensor broadcast over the batch. + + The context occurs four positions back, which `max_history - step` puts + just outside the scanned window, so a row that mistakes another row's step + for its own reports the opposite verdict. The neighbours leg is the + deterministic control: a row that reads a step of 0 instead of 2 widens its + window by two positions and finds the context. The production leg asserts + the layout the speculators pass, where the stride-less read lands outside + the allocation and can return the right value by chance. + """ + num_rows = 4 + all_token_ids = torch.tensor([[3, 4, 9, 9, 9, 9], [3, 4, 9, 9, 9, 9]]) + req_indices = torch.tensor([0, 0, 1, 1]) + prompt_lens = torch.tensor([0, 0]) + total_lens = torch.tensor([6, 6]) + prior_contexts = torch.zeros(num_rows, 3, 2, dtype=torch.int64) + contexts = torch.tensor([[3, 4]]).expand(num_rows, 2).contiguous() + enabled = torch.ones(num_rows, dtype=torch.bool) + + def run(steps, device): + prior = prior_contexts.clone().to(device) + active = draft_watermarking_mask( + all_token_ids.to(device), + req_indices.to(device), + prompt_lens.to(device), + total_lens.to(device), + prior, + contexts.to(device), + steps, + enabled.to(device), + max_history=4, + include_prompt=False, + ) + return active.cpu(), prior.cpu() + + expected, expected_prior = run(torch.full((num_rows,), 2), torch.device("cpu")) + assert torch.equal(expected, torch.ones(num_rows, dtype=torch.bool)) + + materialized, materialized_prior = run( + torch.full((num_rows,), 2).cuda(), torch.device("cuda") + ) + assert torch.equal(materialized, expected) + assert torch.equal(materialized_prior, expected_prior) + + # Neighbouring steps are the value a row reads when the broadcast stride is + # ignored; the production tensor reads uninitialized memory instead. + neighbours = torch.tensor([2, 0, 0, 0]).cuda() + broadcast, broadcast_prior = run( + neighbours[0].expand(num_rows), torch.device("cuda") + ) + assert torch.equal(broadcast, expected) + assert torch.equal(broadcast_prior, expected_prior) + + step = torch.tensor(2).cuda() + production, production_prior = run(step.expand(num_rows), torch.device("cuda")) + assert torch.equal(production, expected) + assert torch.equal(production_prior, expected_prior) + + def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): class StubSpeculator(DraftModelSpeculator): def capture(self): ... @@ -937,22 +1188,32 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): return torch.zeros(hidden_states.shape[0], 8) class StubWatermarker: + context_width = 2 + @staticmethod - def sample(logits, contexts, random_sampler=None): - assert random_sampler is None - return WatermarkSample(torch.tensor([7, 7]), logits) + def sample(logits, contexts, random_sampler=None, skip_mask=None): + assert random_sampler is not None + return WatermarkSample( + torch.where(skip_mask, random_sampler(logits), 7), logits + ) speculator = object.__new__(StubSpeculator) speculator.model = StubModel() speculator.use_fp64_gumbel = False speculator.acceptance_estimator = None - draft_watermarker = object.__new__(DraftWatermarker) - draft_watermarker.watermarker = StubWatermarker() - draft_watermarker.contexts = torch.tensor([[1, 2], [3, 4]]) + draft_watermarker = DraftWatermarker( + StubWatermarker(), + max_num_reqs=2, + device=torch.device("cpu"), + num_speculative_steps=1, + deduplicate_contexts="none", + deduplicate_contexts_max_history=None, + ) + draft_watermarker.contexts.copy_(torch.tensor([[1, 2], [3, 4]])) draft_watermarker.enabled = torch.tensor([True, False]) speculator.draft_watermarker = draft_watermarker monkeypatch.setattr( - "vllm.v1.worker.gpu.spec_decode.speculator.gumbel_sample", + "vllm.v1.watermarking.watermarker.gumbel_sample", lambda *args, **kwargs: torch.tensor([3, 4]), ) @@ -970,6 +1231,274 @@ def test_draft_sampler_uses_draft_key_and_advances_context(monkeypatch): assert torch.equal(draft_watermarker.contexts, torch.tensor([[2, 7], [4, 4]])) +def test_draft_sampler_uses_ordinary_samples_for_repeated_contexts(monkeypatch): + class StubWatermarker: + context_width = 1 + + def __init__(self): + self.samples = iter((2, 1, 8)) + + def sample(self, logits, contexts, random_sampler=None, skip_mask=None): + watermarked = torch.tensor([next(self.samples)]) + return WatermarkSample( + torch.where(skip_mask, random_sampler(logits), watermarked), logits + ) + + draft_watermarker = DraftWatermarker( + StubWatermarker(), + max_num_reqs=1, + device=torch.device("cpu"), + num_speculative_steps=3, + deduplicate_contexts="single_turn", + deduplicate_contexts_max_history=None, + ) + draft_watermarker.prepare( + contexts=torch.tensor([[1]]), + enabled=torch.tensor([True]), + all_token_ids=torch.tensor([[100, 1, 0, 0]]), + prompt_lens=torch.tensor([1]), + total_lens=torch.tensor([2]), + ) + idx_mapping = torch.tensor([0]) + temperature = torch.tensor([1.0]) + ordinary_samples = iter((torch.tensor([5]), torch.tensor([6]), torch.tensor([4]))) + monkeypatch.setattr( + "vllm.v1.watermarking.watermarker.gumbel_sample", + lambda *args, **kwargs: next(ordinary_samples), + ) + seeds = torch.tensor([0]) + draft_logits = torch.zeros(1, 3, 8) + + first = draft_watermarker.sample( + torch.zeros(1, 8), + idx_mapping, + temperature=temperature, + seed=seeds, + pos=torch.tensor([0]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(0), + logits_cache=draft_logits, + use_fp64=False, + ) + second = draft_watermarker.sample( + torch.zeros(1, 8), + idx_mapping, + temperature=temperature, + seed=seeds, + pos=torch.tensor([1]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(1), + logits_cache=draft_logits, + use_fp64=False, + ) + repeated = draft_watermarker.sample( + torch.zeros(1, 8), + idx_mapping, + temperature=temperature, + seed=seeds, + pos=torch.tensor([2]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(2), + logits_cache=draft_logits, + use_fp64=False, + ) + + assert first.item() == 2 + assert second.item() == 1 + assert repeated.item() == 4 + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +def test_draft_context_deduplication_reads_strided_request_indices(): + """DSpark samples a step at a time, so req_indices is an `idx_map[:, i]` column.""" + num_reqs = 4 + num_speculative_steps = 3 + all_token_ids = torch.tensor( + [[3, 4, 3, 4], [9, 9, 9, 9], [3, 4, 3, 4], [9, 9, 9, 9]] + ).cuda() + prompt_lens = torch.zeros(num_reqs, dtype=torch.int64).cuda() + total_lens = torch.full((num_reqs,), 4).cuda() + contexts = torch.tensor([[3, 4]]).expand(num_reqs, 2).contiguous().cuda() + steps = torch.zeros(num_reqs, dtype=torch.int64).cuda() + enabled = torch.ones(num_reqs, dtype=torch.bool).cuda() + + idx_map = ( + torch.arange(num_reqs) + .repeat_interleave(num_speculative_steps) + .view(num_reqs, num_speculative_steps) + .cuda() + ) + column = idx_map[:, 1] + assert column.stride(0) == num_speculative_steps + + def run(req_indices): + prior = torch.zeros( + num_reqs, num_speculative_steps, 2, dtype=torch.int64, device="cuda" + ) + return draft_watermarking_mask( + all_token_ids, + req_indices, + prompt_lens, + total_lens, + prior, + contexts, + steps, + enabled, + max_history=None, + include_prompt=False, + ).cpu() + + expected = torch.tensor([False, True, False, True]) + assert torch.equal(run(column.contiguous()), expected) + assert torch.equal(run(column), expected) + + +def test_draft_sampler_deduplicates_against_committed_history(monkeypatch): + class StubWatermarker: + context_width = 1 + + @staticmethod + def sample(logits, contexts, random_sampler=None, skip_mask=None): + return WatermarkSample( + torch.where(skip_mask, random_sampler(logits), 7), logits + ) + + draft_watermarker = DraftWatermarker( + StubWatermarker(), + max_num_reqs=1, + device=torch.device("cpu"), + num_speculative_steps=1, + deduplicate_contexts="single_turn", + deduplicate_contexts_max_history=None, + ) + draft_watermarker.prepare( + contexts=torch.tensor([[1]]), + enabled=torch.tensor([True]), + all_token_ids=torch.tensor([[100, 1, 2, 1]]), + prompt_lens=torch.tensor([1]), + total_lens=torch.tensor([4]), + ) + + monkeypatch.setattr( + "vllm.v1.watermarking.watermarker.gumbel_sample", + lambda *args, **kwargs: torch.tensor([4]), + ) + sampled = draft_watermarker.sample( + torch.zeros(1, 8), + torch.tensor([0]), + temperature=torch.tensor([1.0]), + seed=torch.tensor([0]), + pos=torch.tensor([0]), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(0), + logits_cache=torch.zeros(1, 1, 8), + use_fp64=False, + ) + + assert sampled.item() == 4 + + +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="requires a CUDA-like accelerator" +) +@pytest.mark.parametrize("idx_mapping_layout", ["contiguous", "column_view"]) +def test_draft_watermarker_deduplicates_across_a_block_on_cuda( + monkeypatch, idx_mapping_layout +): + """Drive the whole draft sampler over a block with the production shapes. + + `draft_step` is a 0-d tensor, and a speculator that samples one step at a + time hands over a column of its index map. Each step is checked against the + committed history the window still reaches and against the earlier steps of + the block. + """ + num_reqs, num_steps, vocab_size = 2, 3, 16 + watermarker = GumbelWatermarker(key=42, context_width=2) + draft_watermarker = DraftWatermarker( + watermarker, + max_num_reqs=num_reqs, + device=torch.device("cuda"), + num_speculative_steps=num_steps, + deduplicate_contexts="single_turn", + deduplicate_contexts_max_history=4, + ) + draft_watermarker.prepare( + contexts=torch.tensor([[5, 6], [6, 3]]).cuda(), + enabled=torch.ones(num_reqs, dtype=torch.bool).cuda(), + all_token_ids=torch.tensor([[5, 6, 7, 5, 6], [8, 9, 6, 6, 3]]).cuda(), + prompt_lens=torch.zeros(num_reqs, dtype=torch.int64).cuda(), + total_lens=torch.full((num_reqs,), 5).cuda(), + ) + + if idx_mapping_layout == "contiguous": + idx_mapping = torch.arange(num_reqs, dtype=torch.int32).cuda() + else: + idx_mapping = ( + torch.arange(num_reqs, dtype=torch.int32) + .repeat_interleave(num_steps) + .view(num_reqs, num_steps) + .cuda() + )[:, 1] + assert idx_mapping.stride(0) == num_steps + + skip_masks: list[torch.Tensor] = [] + original = philox_gumbel_sample + + def spy(*args, **kwargs): + skip_masks.append(kwargs["skip_mask"].cpu()) + return original(*args, **kwargs) + + monkeypatch.setattr("vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample", spy) + + draft_logits = torch.zeros(num_reqs, num_steps, vocab_size + 1).cuda() + step_logits = [] + for step in range(num_steps): + # Token 6 wins by a margin no Gumbel draw can close, on either path. + logits = torch.zeros(num_reqs, vocab_size).cuda() + logits[:, 6] = 100.0 + logits[:, 0] = float(step) + 1.0 + step_logits.append(logits) + sampled = draft_watermarker.sample( + logits, + idx_mapping, + temperature=torch.ones(num_reqs).cuda(), + seed=torch.zeros(num_reqs, dtype=torch.int64).cuda(), + pos=torch.full((num_reqs,), 5 + step, dtype=torch.int64).cuda(), + apply_temperature=True, + is_drafting=True, + logits_cache_col=torch.tensor(step).cuda(), + logits_cache=draft_logits, + use_fp64=False, + ) + assert torch.equal(sampled.cpu(), torch.tensor([6, 6])) + + # Step 0: request 0's [5, 6] is committed at position 2, request 1's [6, 3] + # is new. Step 1: [6, 6] and [3, 6] are both new. Step 2: request 0's + # [6, 6] repeats its own step-1 context and request 1's repeats position 4. + assert [mask.tolist() for mask in skip_masks] == [ + [True, False], + [False, False], + [True, True], + ] + assert torch.equal( + draft_watermarker.prior_contexts[:num_reqs].cpu(), + torch.tensor([[[5, 6], [6, 6], [6, 6]], [[6, 3], [3, 6], [6, 6]]]), + ) + assert torch.equal( + draft_watermarker.contexts[:num_reqs].cpu(), torch.tensor([[6, 6], [6, 6]]) + ) + for step, logits in enumerate(step_logits): + # The 0-d draft step selects the logits cache column. + assert torch.equal(draft_logits[:, step, :vocab_size], logits) + assert not draft_logits[:, :, -1].any() + + def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch): speculator = object.__new__(DSparkSpeculator) speculator.draft_logits = torch.zeros(2, 1, 8) @@ -982,15 +1511,23 @@ def test_dspark_reduced_vocab_draft_sampler_applies_watermarking(monkeypatch): speculator.acceptance_estimator = None watermark_logits: list[torch.Tensor] = [] - def sample(logits, sampled, idx_map, temperature): + def sample( + logits, + idx_mapping, + temperature, + seed, + pos, + apply_temperature, + is_drafting, + logits_cache=None, + logits_cache_col=None, + use_fp64=False, + logits_cache_source=None, + ): watermark_logits.append(logits.clone()) - return sampled + 1 + return torch.tensor([4, 5]) speculator.draft_watermarker = SimpleNamespace(sample=sample) - monkeypatch.setattr( - "vllm.v1.worker.gpu.spec_decode.dspark.speculator.gumbel_sample", - lambda *args, **kwargs: torch.tensor([3, 4]), - ) sampled = speculator._sample_logits( torch.tensor([[10.0, 20.0], [30.0, 40.0]]), diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 824c49efa0..2da17707e5 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1344,13 +1344,6 @@ class VllmConfig: "share of output tokens supplied by accepted drafts.", scope="global", ) - if watermark_config.deduplicate_contexts != "none": - logger.warning_once( - "Context deduplication is not supported with speculative " - "decoding and will not be applied to accepted drafts, " - "rejection-recovery tokens, or bonus tokens.", - scope="global", - ) if watermark_config.algorithm == "dual_key_gumbel" and ( watermark_config.alpha != get_field(WatermarkConfig, "alpha").default ): diff --git a/vllm/v1/watermarking/gpu_sampler.py b/vllm/v1/watermarking/gpu_sampler.py index a4e6e2dd5b..776dece208 100644 --- a/vllm/v1/watermarking/gpu_sampler.py +++ b/vllm/v1/watermarking/gpu_sampler.py @@ -119,8 +119,9 @@ class GPUWatermarkSampler(Sampler): self, expanded_idx_mapping: torch.Tensor, contexts: torch.Tensor, + expanded_local_pos: torch.Tensor | None = None, ) -> torch.Tensor: - return repeated_context_mask( + repeated = repeated_context_mask( self.req_states.all_token_ids.gpu, expanded_idx_mapping, self.req_states.prompt_len.gpu, @@ -128,8 +129,16 @@ class GPUWatermarkSampler(Sampler): contexts, self.deduplicate_contexts_max_history, include_prompt=self.deduplicate_contexts == "all", - skip_partial_context=self.deduplicate_contexts == "all", + skip_partial_context=( + self.deduplicate_contexts == "all" and expanded_local_pos is None + ), + history_offsets=expanded_local_pos, + local_positions=expanded_local_pos, + num_speculative_steps=( + self.num_speculative_tokens if expanded_local_pos is not None else 0 + ), ) + return repeated def _get_contexts( self, diff --git a/vllm/v1/watermarking/gumbel.py b/vllm/v1/watermarking/gumbel.py index cacb620b5e..655267d1e4 100644 --- a/vllm/v1/watermarking/gumbel.py +++ b/vllm/v1/watermarking/gumbel.py @@ -94,6 +94,10 @@ class GumbelWatermarker(Watermarker): seeds=random_sampler.seeds, positions=random_sampler.positions, use_fp64=random_sampler.use_fp64, + is_drafting=random_sampler.is_drafting, + logits_cache=random_sampler.logits_cache, + logits_cache_col=random_sampler.logits_cache_col, + logits_cache_source=random_sampler.logits_cache_source, ) return WatermarkSample(token_ids, logits) diff --git a/vllm/v1/watermarking/spec_decode.py b/vllm/v1/watermarking/spec_decode.py index c3675a3ab9..36d00313ae 100644 --- a/vllm/v1/watermarking/spec_decode.py +++ b/vllm/v1/watermarking/spec_decode.py @@ -3,14 +3,16 @@ import torch -from vllm.config.watermarking import WatermarkConfig +from vllm.config.watermarking import WatermarkConfig, WatermarkContextScope from vllm.v1.watermarking.factory import create_watermarker from vllm.v1.watermarking.gumbel import GumbelWatermarker from vllm.v1.watermarking.prfs import PhiloxPRF from vllm.v1.watermarking.watermarker import ( + RandomSampler, SupportsSpeculativeDecoding, Watermarker, ) +from vllm.v1.worker.gpu.sample.watermark import draft_watermarking_mask from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import rejection_sample @@ -20,8 +22,14 @@ class DraftWatermarker: watermarker: Watermarker, max_num_reqs: int, device: torch.device, + num_speculative_steps: int, + deduplicate_contexts: WatermarkContextScope, + deduplicate_contexts_max_history: int | None, ) -> None: self.watermarker = watermarker + self.num_speculative_steps = num_speculative_steps + self.deduplicate_contexts = deduplicate_contexts + self.deduplicate_contexts_max_history = deduplicate_contexts_max_history self.contexts = torch.zeros( max_num_reqs, watermarker.context_width, @@ -29,30 +37,112 @@ class DraftWatermarker: device=device, ) self.enabled = torch.zeros(max_num_reqs, dtype=torch.bool, device=device) + self.prior_contexts = torch.zeros( + max_num_reqs, + num_speculative_steps, + watermarker.context_width, + dtype=torch.int64, + device=device, + ) + self.all_token_ids: torch.Tensor | None = None + self.prompt_lens: torch.Tensor | None = None + self.total_lens: torch.Tensor | None = None - def prepare(self, contexts: torch.Tensor, enabled: torch.Tensor) -> None: + def prepare( + self, + contexts: torch.Tensor, + enabled: torch.Tensor, + all_token_ids: torch.Tensor, + prompt_lens: torch.Tensor, + total_lens: torch.Tensor, + ) -> None: num_reqs = contexts.shape[0] self.contexts[:num_reqs].copy_(contexts) self.enabled[:num_reqs].copy_(enabled) + self.all_token_ids = all_token_ids + self.prompt_lens = prompt_lens + self.total_lens = total_lens + + def _sampling_state( + self, + logits: torch.Tensor, + idx_mapping: torch.Tensor, + request_temperatures: torch.Tensor, + draft_step: int | torch.Tensor, + contexts: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + num_rows = logits.shape[0] + steps = torch.as_tensor(draft_step, device=logits.device, dtype=torch.int64) + if steps.ndim == 0: + steps = steps.expand(num_rows) + enabled = self.enabled[:num_rows] & (request_temperatures != 0) + if self.deduplicate_contexts == "none": + return steps, enabled + assert self.all_token_ids is not None + assert self.prompt_lens is not None + assert self.total_lens is not None + enabled = draft_watermarking_mask( + self.all_token_ids, + idx_mapping, + self.prompt_lens, + self.total_lens, + self.prior_contexts, + contexts, + steps, + enabled, + self.deduplicate_contexts_max_history, + include_prompt=self.deduplicate_contexts == "all", + ) + return steps, enabled def sample( self, logits: torch.Tensor, - ordinary_sampled: torch.Tensor, idx_mapping: torch.Tensor, temperature: torch.Tensor, + seed: torch.Tensor, + pos: torch.Tensor, + apply_temperature: bool, + is_drafting: bool, + logits_cache: torch.Tensor | None = None, + logits_cache_col: torch.Tensor | None = None, + use_fp64: bool = False, + logits_cache_source: torch.Tensor | None = None, ) -> torch.Tensor: + assert apply_temperature + assert is_drafting + assert logits_cache is not None + assert logits_cache_col is not None + num_rows = logits.shape[0] request_temperatures = temperature[idx_mapping] + contexts = self.contexts[:num_rows] + steps, enabled = self._sampling_state( + logits, idx_mapping, request_temperatures, logits_cache_col, contexts + ) + processed_logits = logits / torch.where( request_temperatures == 0, 1, request_temperatures ).unsqueeze(-1) - watermarked = self.watermarker.sample( + random_sampler = RandomSampler( + expanded_idx_mapping=idx_mapping, + temperatures=temperature, + seeds=seed, + positions=pos, + use_fp64=use_fp64, + is_drafting=True, + logits_cache=logits_cache, + logits_cache_col=steps, + logits_cache_source=( + logits if logits_cache_source is None else logits_cache_source + ), + ) + sampled = self.watermarker.sample( processed_logits, - self.contexts[: logits.shape[0]], + contexts, + random_sampler=random_sampler, + skip_mask=~enabled, ).token_ids - enabled = self.enabled[: logits.shape[0]] & (request_temperatures != 0) - sampled = torch.where(enabled, watermarked, ordinary_sampled) - contexts = self.contexts[: logits.shape[0]] + contexts.copy_(torch.cat((contexts[:, 1:], sampled.unsqueeze(-1)), dim=-1)) return sampled @@ -68,9 +158,19 @@ def create_speculative_draft_watermarker( max_num_reqs: int, device: torch.device, allow_target_only: bool, + num_speculative_steps: int = 1, + deduplicate_contexts: WatermarkContextScope = "none", + deduplicate_contexts_max_history: int | None = 8192, ) -> DraftWatermarker | None: if isinstance(watermarker, SupportsSpeculativeDecoding): - return DraftWatermarker(watermarker.draft_watermarker, max_num_reqs, device) + return DraftWatermarker( + watermarker.draft_watermarker, + max_num_reqs, + device, + num_speculative_steps, + deduplicate_contexts, + deduplicate_contexts_max_history, + ) if allow_target_only: return None raise ValueError( @@ -130,9 +230,10 @@ def watermarked_rejection_sample( contexts: torch.Tensor, watermarking: torch.Tensor, watermarker: Watermarker, + watermarking_skip_mask: torch.Tensor | None = None, use_fp64: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: - """Use the target key for rejection recovery and bonus tokens.""" + """Use the target key except at contexts selected for ordinary sampling.""" assert contexts.shape == (target_logits.shape[0], watermarker.context_width) return rejection_sample( target_logits, @@ -149,5 +250,6 @@ def watermarked_rejection_sample( use_fp64=use_fp64, contexts=contexts, watermarking=watermarking, + watermarking_skip_mask=watermarking_skip_mask, watermark_key=_resolve_watermark_key(watermarker), ) diff --git a/vllm/v1/watermarking/watermarker.py b/vllm/v1/watermarking/watermarker.py index 12570f4ae5..15917d2520 100644 --- a/vllm/v1/watermarking/watermarker.py +++ b/vllm/v1/watermarking/watermarker.py @@ -23,6 +23,10 @@ class RandomSampler: seeds: torch.Tensor positions: torch.Tensor use_fp64: bool = False + is_drafting: bool = False + logits_cache: torch.Tensor | None = None + logits_cache_col: torch.Tensor | None = None + logits_cache_source: torch.Tensor | None = None def __call__(self, logits: torch.Tensor) -> torch.Tensor: return gumbel_sample( @@ -32,7 +36,10 @@ class RandomSampler: self.seeds, self.positions, apply_temperature=False, - is_drafting=False, + is_drafting=self.is_drafting, + logits_cache=self.logits_cache, + logits_cache_col=self.logits_cache_col, + logits_cache_source=self.logits_cache_source, use_fp64=self.use_fp64, ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 3bdeda23dd..fed9e80d0c 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -914,8 +914,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec_hidden_states = pre_hc_hidden_states[: hidden_states.shape[0]] # type: ignore[union-attr] if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( - self.sampler._get_contexts(input_batch.idx_mapping), - self.sampler.watermarking.gpu[input_batch.idx_mapping], + self.sampler, input_batch.idx_mapping ) with use_workspace_lane(self._draft_workspace_lane): self.speculator.propose( @@ -2194,8 +2193,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): spec_hidden_states = pre_hc_hidden_states[: draft_hidden_states.size(0)] if isinstance(self.sampler, GPUWatermarkSampler): self.speculator.prepare_watermarking( - self.sampler._get_contexts(input_batch.idx_mapping), - self.sampler.watermarking.gpu[input_batch.idx_mapping], + self.sampler, input_batch.idx_mapping ) with use_workspace_lane(self._draft_workspace_lane): draft_tokens = self.speculator.propose( diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index e54a458709..a34169b733 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -220,6 +220,8 @@ def gumbel_block_argmax( logits_cache_stride_0, logits_cache_stride_1, logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, vocab_size, IS_DRAFTING: tl.constexpr, APPLY_TEMPERATURE: tl.constexpr, @@ -240,12 +242,16 @@ def gumbel_block_argmax( col = tl.load(logits_cache_col_ptr + token_idx) else: col = tl.load(logits_cache_col_ptr) + cached_logits = tl.load( + logits_cache_source_ptr + token_idx * logits_cache_source_stride + block, + mask=mask, + ) tl.store( logits_cache_ptr + req_state_idx * logits_cache_stride_0 + col * logits_cache_stride_1 + block, - logits, + cached_logits, mask=mask & is_valid_req, ) @@ -275,6 +281,8 @@ def _gumbel_sample_kernel( logits_cache_stride_0, logits_cache_stride_1, logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, logits_ptr, logits_stride, expanded_idx_mapping_ptr, @@ -312,6 +320,8 @@ def _gumbel_sample_kernel( logits_cache_stride_0, logits_cache_stride_1, logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, vocab_size, IS_DRAFTING=IS_DRAFTING, APPLY_TEMPERATURE=APPLY_TEMPERATURE, @@ -334,6 +344,7 @@ def gumbel_sample( logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size] logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens] use_fp64: bool = False, + logits_cache_source: torch.Tensor | None = None, ) -> torch.Tensor: # Enforce contiguity on non-strided input tensors expanded_idx_mapping = expanded_idx_mapping.contiguous() @@ -342,11 +353,26 @@ def gumbel_sample( logits_cache_col = logits_cache_col.contiguous() num_tokens, vocab_size = logits.shape if logits_cache is not None: + if logits_cache_source is None: + logits_cache_source = logits + assert logits_cache_source.shape == logits.shape, ( + "logits cache source must match sampled logits shape" + ) + assert logits_cache_source.device == logits.device, ( + "logits cache source must be on the sampled logits device" + ) + assert logits_cache_source.dtype == logits_cache.dtype, ( + "logits cache source and destination must have the same dtype" + ) assert logits_cache.size(-1) >= vocab_size, ( f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower " f"than the sampled logits ({vocab_size}). Cached logits would be " "truncated." ) + elif logits_cache_source is not None: + raise ValueError("logits_cache_source requires logits_cache") + if logits_cache_source is not None and logits_cache_source.stride(-1) != 1: + logits_cache_source = logits_cache_source.contiguous() BLOCK_SIZE = 1024 num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE) local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64) @@ -362,6 +388,8 @@ def gumbel_sample( logits_cache.stride(0) if logits_cache is not None else 0, logits_cache.stride(1) if logits_cache is not None else 0, logits_cache_col, + logits_cache_source, + logits_cache_source.stride(0) if logits_cache_source is not None else 0, logits, logits.stride(0), expanded_idx_mapping, diff --git a/vllm/v1/worker/gpu/sample/watermark.py b/vllm/v1/worker/gpu/sample/watermark.py index b70d8ae271..216d189f8d 100644 --- a/vllm/v1/worker/gpu/sample/watermark.py +++ b/vllm/v1/worker/gpu/sample/watermark.py @@ -51,12 +51,25 @@ def _repeated_context_mask_kernel( req_indices_ptr, prompt_lens_ptr, total_lens_ptr, + history_offsets_ptr, + history_offsets_stride, contexts_ptr, context_stride, + local_positions_ptr, + prior_contexts_ptr, + prior_contexts_stride_0, + prior_contexts_stride_1, + steps_ptr, + steps_stride, + enabled_ptr, CONTEXT_WIDTH: tl.constexpr, + NUM_SPECULATIVE_STEPS: tl.constexpr, MAX_HISTORY: tl.constexpr, INCLUDE_PROMPT: tl.constexpr, SKIP_PARTIAL_CONTEXT: tl.constexpr, + HAS_HISTORY_OFFSETS: tl.constexpr, + SPECULATIVE_CONTEXTS: tl.constexpr, + DRAFT_CONTEXTS: tl.constexpr, BLOCK: tl.constexpr, ): row = tl.program_id(0).to(tl.int64) @@ -71,12 +84,16 @@ def _repeated_context_mask_kernel( return sequence_start = 0 if INCLUDE_PROMPT else prompt_len history_len = total_len - sequence_start + history_offset = 0 + if HAS_HISTORY_OFFSETS: + history_offset = tl.load(history_offsets_ptr + row * history_offsets_stride) offsets = tl.arange(0, BLOCK) repeated = tl.full((), 0, tl.int32) scan_start = 0 if MAX_HISTORY > 0: - scan_start = tl.maximum(history_len - MAX_HISTORY, 0) + remaining_history = tl.maximum(MAX_HISTORY - history_offset, 0) + scan_start = tl.maximum(history_len - remaining_history, 0) # Aligned loop bounds let the per-lane offsets fold into immediates. aligned_start = (scan_start // BLOCK) * BLOCK for block_start in tl.range(aligned_start, history_len, BLOCK): @@ -96,6 +113,93 @@ def _repeated_context_mask_kernel( ) matches &= historical_token == context_token repeated |= tl.max(matches.to(tl.int32), axis=0) + + if SPECULATIVE_CONTEXTS: + local_pos = tl.load(local_positions_ptr + row) + if INCLUDE_PROMPT: + partial_context = tl.full((), 0, tl.int32) + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + partial_context |= context_token < 0 + repeated |= partial_context + for offset in range(1, NUM_SPECULATIVE_STEPS + 1): + prior_row = row - offset + in_history = (prior_row >= 0) & (local_pos >= offset) + if MAX_HISTORY > 0: + in_history &= offset <= MAX_HISTORY + prior_req_idx = tl.load( + req_indices_ptr + tl.maximum(prior_row, 0), + mask=in_history, + other=-1, + ) + prior_local_pos = tl.load( + local_positions_ptr + tl.maximum(prior_row, 0), + mask=in_history, + other=-1, + ) + matches = ( + in_history + & (req_idx == prior_req_idx) + & (local_pos == prior_local_pos + offset) + ) + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + prior_token = tl.load( + contexts_ptr + + tl.maximum(prior_row, 0) * context_stride + + context_offset, + mask=in_history, + other=-2, + ) + matches &= context_token == prior_token + repeated |= matches + + if DRAFT_CONTEXTS: + step = tl.load(steps_ptr + row * steps_stride).to(tl.int64) + partial_context = tl.full((), 0, tl.int32) + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + partial_context |= context_token < 0 + if INCLUDE_PROMPT: + repeated |= partial_context + + for prior_step in range(NUM_SPECULATIVE_STEPS): + in_history = prior_step < step + if MAX_HISTORY > 0: + in_history &= prior_step >= step - MAX_HISTORY + matches = in_history + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + prior_token = tl.load( + prior_contexts_ptr + + row * prior_contexts_stride_0 + + prior_step * prior_contexts_stride_1 + + context_offset + ) + matches &= context_token == prior_token + repeated |= matches + + for context_offset in range(CONTEXT_WIDTH): + context_token = tl.load( + contexts_ptr + row * context_stride + context_offset + ) + tl.store( + prior_contexts_ptr + + row * prior_contexts_stride_0 + + step * prior_contexts_stride_1 + + context_offset, + context_token, + ) + enabled = tl.load(enabled_ptr + row) + repeated = enabled & (repeated == 0) tl.store(output_ptr + row, repeated) @@ -108,6 +212,7 @@ def _repeated_context_mask_cpu( max_history: int | None = None, include_prompt: bool = False, skip_partial_context: bool = False, + history_offsets: torch.Tensor | None = None, ) -> torch.Tensor: """Reference implementation; the parity tests check the Triton kernel against it.""" repeated = torch.zeros(len(req_indices), dtype=torch.bool) @@ -124,8 +229,11 @@ def _repeated_context_mask_cpu( history_tokens = all_token_ids[req_idx, sequence_start:total_len].tolist() prefix = [-1] * contexts.shape[-1] current_context = tuple(contexts[row].tolist()) + history_offset = 0 if history_offsets is None else int(history_offsets[row]) history_start = ( - max(0, len(history_tokens) - max_history) if max_history is not None else 0 + max(0, len(history_tokens) - max(0, max_history - history_offset)) + if max_history is not None + else 0 ) for history_pos, token_id in enumerate(history_tokens): if history_pos >= history_start and ( @@ -146,6 +254,9 @@ def repeated_context_mask( max_history: int | None = None, include_prompt: bool = False, skip_partial_context: bool = False, + history_offsets: torch.Tensor | None = None, + local_positions: torch.Tensor | None = None, + num_speculative_steps: int = 0, ) -> torch.Tensor: """Return, per row, whether the row's context already occurred in its history. @@ -163,12 +274,23 @@ def repeated_context_mask( include_prompt: Search the prompt as well as the generated tokens. skip_partial_context: Mark contexts containing start padding so they use ordinary sampling. + history_offsets: Number of newer, non-committed positions preceding each + context. These positions count toward `max_history`. Must be 1-D + over rows; a stride-0 broadcast row is fine. + local_positions: Position within each request's speculative block. When + provided, compare each context with earlier contexts in that block. + num_speculative_steps: Maximum number of draft tokens in the block. """ if max_history is not None and max_history < 1: raise ValueError("max_history must be positive or None") + if (local_positions is None) != (num_speculative_steps == 0): + raise ValueError( + "local_positions and positive num_speculative_steps must be " + "provided together" + ) if all_token_ids.device.type == "cpu": - return _repeated_context_mask_cpu( + repeated = _repeated_context_mask_cpu( all_token_ids, req_indices, prompt_lens, @@ -177,10 +299,42 @@ def repeated_context_mask( max_history, include_prompt, skip_partial_context, + history_offsets, ) + if local_positions is not None: + max_offset = num_speculative_steps + 1 + if max_history is not None: + max_offset = min(max_offset, max_history + 1) + for offset in range(1, max_offset): + prior_contexts = torch.cat( + (contexts[:offset], contexts[:-offset]), dim=0 + ) + prior_requests = torch.cat( + (req_indices[:offset], req_indices[:-offset]), dim=0 + ) + prior_local_pos = torch.cat( + (local_positions[:offset], local_positions[:-offset]), dim=0 + ) + repeated |= ( + (req_indices >= 0) + & (local_positions >= offset) + & (req_indices == prior_requests) + & (local_positions == prior_local_pos + offset) + & (contexts == prior_contexts).all(dim=-1) + ) + if include_prompt: + repeated |= (contexts < 0).any(dim=-1) + return repeated if contexts.stride(-1) != 1: contexts = contexts.contiguous() + # Only history_offsets passes its row stride to the kernel. + req_indices = req_indices.contiguous() + if local_positions is not None: + local_positions = local_positions.contiguous() + # `prompt_lens` and `total_lens` are indexed flat by request slot, and + # `all_token_ids` takes a row stride but is read flat within the row. Those + # three are contiguous by contract; every caller owns them outright. repeated = torch.empty(len(req_indices), dtype=torch.bool, device=contexts.device) _repeated_context_mask_kernel[(len(req_indices),)]( repeated, @@ -189,17 +343,104 @@ def repeated_context_mask( req_indices, prompt_lens, total_lens, + history_offsets, + 0 if history_offsets is None else history_offsets.stride(0), contexts, contexts.stride(0), + local_positions, + None, + 0, + 0, + None, + 0, + None, CONTEXT_WIDTH=contexts.shape[-1], + NUM_SPECULATIVE_STEPS=num_speculative_steps, MAX_HISTORY=0 if max_history is None else max_history, INCLUDE_PROMPT=include_prompt, SKIP_PARTIAL_CONTEXT=skip_partial_context, + HAS_HISTORY_OFFSETS=history_offsets is not None, + SPECULATIVE_CONTEXTS=local_positions is not None, + DRAFT_CONTEXTS=False, BLOCK=512, ) return repeated +def draft_watermarking_mask( + all_token_ids: torch.Tensor, + req_indices: torch.Tensor, + prompt_lens: torch.Tensor, + total_lens: torch.Tensor, + prior_contexts: torch.Tensor, + contexts: torch.Tensor, + steps: torch.Tensor, + enabled: torch.Tensor, + max_history: int | None, + include_prompt: bool, +) -> torch.Tensor: + if contexts.device.type == "cpu": + repeated = repeated_context_mask( + all_token_ids, + req_indices, + prompt_lens, + total_lens, + contexts, + max_history, + include_prompt=include_prompt, + history_offsets=steps, + ) + prior_steps = torch.arange(prior_contexts.shape[1]).unsqueeze(0) + in_history = prior_steps < steps.unsqueeze(1) + if max_history is not None: + in_history &= prior_steps >= steps.unsqueeze(1) - max_history + repeated |= ( + (prior_contexts[: len(contexts)] == contexts.unsqueeze(1)).all(dim=-1) + & in_history + ).any(dim=-1) + if include_prompt: + repeated |= (contexts < 0).any(dim=-1) + row_indices = torch.arange(len(contexts)) + prior_contexts.index_put_((row_indices, steps), contexts) + return enabled & ~repeated + + if contexts.stride(-1) != 1: + contexts = contexts.contiguous() + # Only `steps` passes its row stride to the kernel. + req_indices = req_indices.contiguous() + enabled = enabled.contiguous() + active = torch.empty(len(contexts), dtype=torch.bool, device=contexts.device) + _repeated_context_mask_kernel[(len(contexts),)]( + active, + all_token_ids, + all_token_ids.stride(0), + req_indices, + prompt_lens, + total_lens, + steps, + steps.stride(0), + contexts, + contexts.stride(0), + None, + prior_contexts, + prior_contexts.stride(0), + prior_contexts.stride(1), + steps, + steps.stride(0), + enabled, + NUM_SPECULATIVE_STEPS=prior_contexts.shape[1], + CONTEXT_WIDTH=contexts.shape[1], + MAX_HISTORY=0 if max_history is None else max_history, + INCLUDE_PROMPT=include_prompt, + SKIP_PARTIAL_CONTEXT=False, + HAS_HISTORY_OFFSETS=True, + SPECULATIVE_CONTEXTS=False, + DRAFT_CONTEXTS=True, + BLOCK=512, + ) + return active + + @triton.jit def _mulhilo32(multiplier: tl.constexpr, value): high = tl_math.umulhi(multiplier, value) @@ -351,28 +592,58 @@ def _philox_gumbel_kernel( seeds_ptr, pos_ptr, temp_ptr, + logits_cache_ptr, + logits_cache_stride_0, + logits_cache_stride_1, + logits_cache_col_ptr, + logits_cache_source_ptr, + logits_cache_source_stride, key_0_value, key_1_value, vocab_size, CONTEXT_WIDTH: tl.constexpr, BLOCK_SIZE: tl.constexpr, USE_FP64: tl.constexpr, + IS_DRAFTING: tl.constexpr, + PER_TOKEN_COL: tl.constexpr, ): row = tl.program_id(0).to(tl.int64) block_index = tl.program_id(1) groups = block_index * (BLOCK_SIZE // 4) + tl.arange(0, BLOCK_SIZE // 4) + candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = candidate < vocab_size - if skip_mask_ptr is not None: + req_state_idx = row + valid_req = True + if expanded_idx_mapping_ptr is not None: req_state_idx = tl.load(expanded_idx_mapping_ptr + row).to(tl.int64) valid_req = req_state_idx >= 0 + + if logits_cache_ptr is not None: + if PER_TOKEN_COL: + col = tl.load(logits_cache_col_ptr + row) + else: + col = tl.load(logits_cache_col_ptr) + cached_logits = tl.load( + logits_cache_source_ptr + row * logits_cache_source_stride + candidate, + mask=mask, + ) + tl.store( + logits_cache_ptr + + req_state_idx * logits_cache_stride_0 + + col * logits_cache_stride_1 + + candidate, + cached_logits, + mask=mask & valid_req, + ) + + if skip_mask_ptr is not None: # noqa: SIM102 if tl.load(skip_mask_ptr + row): temp = tl.load(temp_ptr + req_state_idx, mask=valid_req, other=0.0).to( tl.float32 ) seed = tl.load(seeds_ptr + req_state_idx, mask=valid_req, other=0) pos = tl.load(pos_ptr + row) - candidate = block_index * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask = candidate < vocab_size logits_row = logits_ptr + row * logits_stride logits = tl.load(logits_row + candidate, mask=mask, other=float("-inf")).to( tl.float32 @@ -384,7 +655,7 @@ def _philox_gumbel_kernel( seed, pos, temp, - IS_DRAFTING=False, + IS_DRAFTING=IS_DRAFTING, USE_FP64=USE_FP64, APPLY_TEMPERATURE=False, ) @@ -443,12 +714,37 @@ def philox_gumbel_sample( seeds: torch.Tensor | None = None, positions: torch.Tensor | None = None, use_fp64: bool = False, + is_drafting: bool = False, + logits_cache: torch.Tensor | None = None, + logits_cache_col: torch.Tensor | None = None, + logits_cache_source: torch.Tensor | None = None, ) -> torch.Tensor: sampling_state = (expanded_idx_mapping, temperatures, seeds, positions) if skip_mask is None and any(value is not None for value in sampling_state): raise ValueError("sampling state requires skip_mask") if skip_mask is not None and any(value is None for value in sampling_state): raise ValueError("skip_mask requires complete sampling state") + if logits_cache is not None: + if logits_cache_col is None or logits_cache_source is None: + raise ValueError("logits cache requires its column and source logits") + if expanded_idx_mapping is None: + raise ValueError("logits cache requires expanded_idx_mapping") + assert logits_cache_source.shape == logits.shape, ( + "logits cache source must match sampled logits shape" + ) + assert logits_cache_source.device == logits.device, ( + "logits cache source must be on the sampled logits device" + ) + assert logits_cache_source.dtype == logits_cache.dtype, ( + "logits cache source and destination must have the same dtype" + ) + assert logits_cache.size(-1) >= logits.shape[-1], ( + f"draft logits cache vocab dim ({logits_cache.size(-1)}) is narrower " + f"than the sampled logits ({logits.shape[-1]}). Cached logits would be " + "truncated." + ) + elif logits_cache_col is not None or logits_cache_source is not None: + raise ValueError("logits cache column and source require logits_cache") if logits.stride(-1) != 1: logits = logits.contiguous() @@ -460,6 +756,10 @@ def philox_gumbel_sample( expanded_idx_mapping = expanded_idx_mapping.contiguous() if positions is not None: positions = positions.contiguous() + if logits_cache_col is not None: + logits_cache_col = logits_cache_col.contiguous() + if logits_cache_source is not None and logits_cache_source.stride(-1) != 1: + logits_cache_source = logits_cache_source.contiguous() num_tokens, vocab_size = logits.shape block_size = 1024 num_blocks = triton.cdiv(vocab_size, block_size) @@ -483,12 +783,20 @@ def philox_gumbel_sample( seeds, positions, temperatures, + logits_cache, + logits_cache.stride(0) if logits_cache is not None else 0, + logits_cache.stride(1) if logits_cache is not None else 0, + logits_cache_col, + logits_cache_source, + logits_cache_source.stride(0) if logits_cache_source is not None else 0, key & _UINT32_MASK_VALUE, key >> 32, vocab_size, CONTEXT_WIDTH=contexts.shape[-1], BLOCK_SIZE=block_size, USE_FP64=use_fp64, + IS_DRAFTING=is_drafting, + PER_TOKEN_COL=logits_cache_col is not None and logits_cache_col.dim() > 0, ) max_block_index = local_max.argmax(dim=-1, keepdim=True) return local_argmax.gather(dim=-1, index=max_block_index).view(-1) diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index b86f58a450..02dd336a6b 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -135,7 +135,12 @@ class DSparkSpeculator(DFlashSpeculator): # sample_pos is the predicted token's position P. Sampling keys a draw # by the position before the sampled token, P-1. - sampled = gumbel_sample( + sampler = ( + gumbel_sample + if self.draft_watermarker is None + else self.draft_watermarker.sample + ) + return sampler( logits, idx_map, self.temperature, @@ -147,11 +152,6 @@ class DSparkSpeculator(DFlashSpeculator): logits_cache_col=self._step_cols[step], use_fp64=self.use_fp64_gumbel, ) - if self.draft_watermarker is not None: - sampled = self.draft_watermarker.sample( - logits, sampled, idx_map, self.temperature - ) - return sampled def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: if self._draft_topk is not None: diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 61eb3fb49a..a9ad00e89f 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -226,11 +226,18 @@ class RejectionSampler: if self.watermark_key is None: return {} assert isinstance(self.sampler, GPUWatermarkSampler) + contexts = self.sampler._get_contexts( + expanded_idx_mapping, expanded_local_pos, draft_sampled + ) + watermarking_skip_mask = None + if self.sampler.deduplicate_contexts != "none": + watermarking_skip_mask = self.sampler._get_repeated_contexts( + expanded_idx_mapping, contexts, expanded_local_pos + ) return { - "contexts": self.sampler._get_contexts( - expanded_idx_mapping, expanded_local_pos, draft_sampled - ), + "contexts": contexts, "watermarking": self.sampler.watermarking.gpu, + "watermarking_skip_mask": watermarking_skip_mask, "watermark_key": self.watermark_key, } diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index cf804e6a6b..b2efa6432e 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -564,7 +564,6 @@ def _rejection_kernel( # stores the target argmax upon first rejection, so it rejects the # placeholder via `accepted` instead. verifying &= is_valid_draft - if verifying: pos = tl.load(pos_ptr + logit_idx) u = tl_rand32(seed, pos, includes_zero=False) @@ -719,6 +718,8 @@ def _seeded_resample_argmax( 0, # logits_cache_stride_0 0, # logits_cache_stride_1 None, # logits_cache_col_ptr + None, # logits_cache_source_ptr + 0, # logits_cache_source_stride vocab_size, IS_DRAFTING=False, APPLY_TEMPERATURE=False, @@ -766,6 +767,7 @@ def _resample_kernel( contexts_stride, # [max_num_reqs], uint8 view of a bool tensor watermarking_ptr, + watermarking_skip_mask_ptr, watermark_key_0, watermark_key_1, vocab_size, @@ -775,6 +777,7 @@ def _resample_kernel( USE_BLOCK_VERIFICATION: tl.constexpr, CONTEXT_WIDTH: tl.constexpr, WATERMARK: tl.constexpr, + DEDUPLICATE_CONTEXTS: tl.constexpr, ): req_idx = tl.program_id(0) resample_idx = tl.load(rejected_step_ptr + req_idx) @@ -807,6 +810,25 @@ def _resample_kernel( ).to(tl.float32) # Compute the residual logits to resample the rejected token from. + is_watermarked = False + if WATERMARK: + is_watermarked = ( + tl.load( + watermarking_ptr + req_state_idx, + mask=req_state_idx >= 0, + other=0, + ) + != 0 + ) + + skip_watermarking = False + if DEDUPLICATE_CONTEXTS: + skip_watermarking = ( + (tl.load(watermarking_skip_mask_ptr + resample_token_idx) != 0) + & is_watermarked + & (temp != 0.0) + ) + if is_bonus or not is_valid_rejected_draft: # Bonus token (no rejections) or -1 placeholder token. In either case, # directly use the target logits. @@ -868,15 +890,7 @@ def _resample_kernel( # Resample the rejected/bonus token. if WATERMARK: # Padded and greedy rows retain the stock draw. - is_watermarked = ( - tl.load( - watermarking_ptr + req_state_idx, - mask=req_state_idx >= 0, - other=0, - ) - != 0 - ) - if is_watermarked & (temp != 0.0): + if is_watermarked & (temp != 0.0) & ~skip_watermarking: watermark_value, idx = philox_gumbel_block_argmax( residual_logits, mask, @@ -1023,6 +1037,7 @@ def rejection_sample( use_block_verification: bool = False, contexts: torch.Tensor | None = None, watermarking: torch.Tensor | None = None, + watermarking_skip_mask: torch.Tensor | None = None, watermark_key: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: assert target_logits.ndim == 2 and target_logits.stride(-1) == 1 @@ -1036,9 +1051,13 @@ def rejection_sample( assert watermark == (watermarking is not None) == (watermark_key is not None), ( "contexts, watermarking and watermark_key must be set together." ) + assert watermarking_skip_mask is None or watermark, ( + "watermarking_skip_mask requires watermarking." + ) contexts_stride = 0 context_width = 1 watermarking_bytes: torch.Tensor | None = None + watermarking_skip_mask_bytes: torch.Tensor | None = None watermark_key_0 = 0 watermark_key_1 = 0 if contexts is not None: @@ -1053,6 +1072,12 @@ def rejection_sample( contexts_stride = contexts.stride(0) context_width = contexts.shape[-1] watermarking_bytes = watermarking.view(torch.uint8) + if watermarking_skip_mask is not None: + assert watermarking_skip_mask.shape == (num_logits,) + assert watermarking_skip_mask.dtype == torch.bool + if not watermarking_skip_mask.is_contiguous(): + watermarking_skip_mask = watermarking_skip_mask.contiguous() + watermarking_skip_mask_bytes = watermarking_skip_mask.view(torch.uint8) watermark_key_0 = watermark_key & 0xFFFFFFFF watermark_key_1 = watermark_key >> 32 draft_logits_stride_0 = 0 @@ -1269,6 +1294,7 @@ def rejection_sample( contexts, contexts_stride, watermarking_bytes, + watermarking_skip_mask_bytes, watermark_key_0, watermark_key_1, vocab_size, @@ -1278,6 +1304,7 @@ def rejection_sample( USE_BLOCK_VERIFICATION=use_block_verification, CONTEXT_WIDTH=context_width, WATERMARK=watermark, + DEDUPLICATE_CONTEXTS=watermarking_skip_mask is not None, ) # Insert the resampled tokens into the output sampled. diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index c30b653196..17baddf0a8 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -38,6 +38,7 @@ from vllm.v1.worker.gpu.spec_decode.acceptance_estimator import ( from vllm.v1.worker.utils import AttentionGroup if TYPE_CHECKING: + from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler from vllm.v1.worker.gpu.pcp_manager import PCPManager logger = init_logger(__name__) @@ -198,6 +199,9 @@ class DraftModelSpeculator(BaseSpeculator): self.max_num_reqs, device, watermark_config.allow_target_only_watermarking, + self.num_speculative_steps, + watermark_config.deduplicate_contexts, + watermark_config.deduplicate_contexts_max_history, ) self.supports_mm_inputs = False @@ -411,7 +415,12 @@ class DraftModelSpeculator(BaseSpeculator): ) -> torch.Tensor: if draft_logits is not None: logits = self.model.compute_logits(hidden_states) - sampled = gumbel_sample( + sampler = ( + gumbel_sample + if self.draft_watermarker is None + else self.draft_watermarker.sample + ) + sampled = sampler( logits, idx_mapping, temperature, @@ -423,10 +432,6 @@ class DraftModelSpeculator(BaseSpeculator): logits_cache_col=draft_step, use_fp64=self.use_fp64_gumbel, ) - if self.draft_watermarker is not None: - sampled = self.draft_watermarker.sample( - logits, sampled, idx_mapping, temperature - ) elif self.use_local_argmax_reduction: return self.model.get_top_tokens(hidden_states) else: @@ -462,11 +467,19 @@ class DraftModelSpeculator(BaseSpeculator): self.acceptance_estimator.step(idx_mapping, num_sampled, num_rejected) def prepare_watermarking( - self, contexts: torch.Tensor, watermarking: torch.Tensor + self, + sampler: "GPUWatermarkSampler", + idx_mapping: torch.Tensor, ) -> None: if self.draft_watermarker is None: return - self.draft_watermarker.prepare(contexts, watermarking) + self.draft_watermarker.prepare( + sampler._get_contexts(idx_mapping), + sampler.watermarking.gpu[idx_mapping], + sampler.req_states.all_token_ids.gpu, + sampler.req_states.prompt_len.gpu, + sampler.req_states.total_len.gpu, + ) def _copy_request_inputs( self, From 8873ac83bbd279aaf1aeacca4050e0baa8e56da2 Mon Sep 17 00:00:00 2001 From: Thien Tran Date: Thu, 1 Oct 2026 00:23:40 +0800 Subject: [PATCH 139/248] [Docs] Add @gau-nernst to CODEOWNERS and committers (#59369) Signed-off-by: Thien Tran Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/CODEOWNERS | 8 ++++++++ docs/governance/committers.md | 2 ++ 2 files changed, 10 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 55acde281a..b483e1f8b6 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -158,6 +158,7 @@ mkdocs.yaml @hmellor # Qwen-specific files /vllm/model_executor/models/qwen* @sighingnow @vadiklyutiy /vllm/transformers_utils/configs/qwen* @sighingnow @vadiklyutiy +/vllm/models/qwen4_exp @gau-nernst # MTP-specific files /vllm/model_executor/models/deepseek_mtp.py @luccafong @@ -166,6 +167,9 @@ mkdocs.yaml @hmellor /vllm/models/deepseek_v4 @zyongye /vllm/model_executor/layers/sparse_attn_indexer.py @zyongye +# Kimi-K3-specific files +/vllm/models/kimi_k3/nvidia/kda.py @gau-nernst + # Mistral-specific files /vllm/model_executor/models/mistral*.py @patrickvonplaten /vllm/model_executor/models/mixtral*.py @patrickvonplaten @@ -178,6 +182,10 @@ mkdocs.yaml @hmellor /vllm/v1/attention/ops/chunked_prefill_paged_decode.py @tdoublep /vllm/v1/attention/ops/triton_unified_attention.py @tdoublep /vllm/third_party/flash_linear_attention @ZJY0516 @vadiklyutiy +/vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl @gau-nernst +/vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py @gau-nernst +/vllm/model_executor/kernels/linear/cute_dsl/gemm_rs_ar.py @gau-nernst +/vllm/cute_utils @gau-nernst # ROCm related: specify owner with write access to notify AMD folks for careful code review /vllm/**/*rocm* @tjtanaa @dllehr-amd @shen-shanshan diff --git a/docs/governance/committers.md b/docs/governance/committers.md index 9613534538..ff5cb2e099 100644 --- a/docs/governance/committers.md +++ b/docs/governance/committers.md @@ -22,6 +22,7 @@ Sorted alphabetically by GitHub handle: - [@DarkLight1337](https://github.com/DarkLight1337): Multimodality, API server - [@dllehr-amd](https://github.com/dllehr-amd): AMD integration - [@esmeetu](https://github.com/esmeetu): developer marketing, community +- [@gau-nernst](https://github.com/gau-nernst): Kernels and performance - [@gshtras](https://github.com/gshtras): ROCm / AMD GPU integration - [@Harry-Chen](https://github.com/Harry-Chen): Build, CI, CUDA architecture - [@heheda12345](https://github.com/heheda12345): Hybrid memory allocator @@ -177,6 +178,7 @@ If you have PRs touching the area, please feel free to ping the area owner for r - FlashInfer: @LucasWilkinson, @mgoin, @WoosukKwon, @MatthewBonanni, @wzhao18 - Blackwell Kernels: @mgoin, @yewentao256 - DeepEP/DeepGEMM: @mgoin, @yewentao256 +- FlashKDA: @gau-nernst ### Integrations From d2fb35f66e6ce9fbd0e32c6129b130ad90c1754c Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Wed, 30 Sep 2026 18:24:08 +0200 Subject: [PATCH 140/248] [CI][ROCm] Drop the duplicate OAI Triton MoE run from FP8 MoE Kernels (#59237) Signed-off-by: Stefan Koncarevic --- .buildkite/test-amd.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index fd2398e759..5620bc5f45 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -5056,7 +5056,6 @@ steps: - vllm/platforms/rocm.py - vllm/envs.py - tests/kernels/moe/test_gpt_oss_triton_kernels.py - - tests/kernels/moe/test_modular_oai_triton_moe.py - tests/kernels/moe/test_moe.py - tests/kernels/moe/test_block_int8.py - tests/kernels/moe/test_triton_moe_no_act_mul.py @@ -5064,7 +5063,6 @@ steps: optional: true commands: - pytest -v -s kernels/moe/test_gpt_oss_triton_kernels.py - - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py - pytest -v -s kernels/moe/test_moe.py - pytest -v -s kernels/moe/test_block_int8.py - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py From 8f24ab30a36d62386bb0d30d2aa1c689806604af Mon Sep 17 00:00:00 2001 From: Rita Brugarolas Date: Wed, 30 Sep 2026 10:20:05 -0700 Subject: [PATCH 141/248] [ROCm][Kimi-K3][Perf] Fuse MLA decode KV-cache write and Q-prep via AITER (#57640) Signed-off-by: Rita Brugarolas Brufau Co-authored-by: Claude Co-authored-by: Andreas Karatzas --- vllm/models/kimi_k3/amd/mla.py | 178 +++++++++++++++++++++++++++++++-- 1 file changed, 170 insertions(+), 8 deletions(-) diff --git a/vllm/models/kimi_k3/amd/mla.py b/vllm/models/kimi_k3/amd/mla.py index e76c0e80ad..11df7d035f 100644 --- a/vllm/models/kimi_k3/amd/mla.py +++ b/vllm/models/kimi_k3/amd/mla.py @@ -7,16 +7,47 @@ from typing import cast import torch from vllm._aiter_ops import rocm_aiter_ops +from vllm.model_executor.layers.attention.attention import get_attention_context from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.mla import MultiHeadLatentAttentionWrapper +from vllm.platforms import current_platform + +_OPT_KV_LORA_RANK = 512 +_OPT_ROT_DIM = 64 +_OPT_MIN_SIZE = 2048 class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper): - """Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion.""" + """Kimi-K3 MLA wrapper with eager AITER q/kv RMSNorm fusion and a fused + decode Q-prep path.""" def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._use_eager_qk_rmsnorm_fusion = bool(rocm_aiter_ops.is_enabled()) + self._fused_qk_prep = self._fused_qk_prep_supported() + self._identity_rope: tuple[torch.Tensor, torch.Tensor] | None = None + + def _fused_qk_prep_supported(self) -> bool: + attn = self.mla_attn + if not ( + rocm_aiter_ops.is_mla_enabled() + and attn.kv_cache_dtype.startswith("fp8") + and self.kv_lora_rank == _OPT_KV_LORA_RANK + and self.qk_rope_head_dim == _OPT_ROT_DIM + and self.kv_lora_rank * self.num_heads >= _OPT_MIN_SIZE + ): + return False + # The fused kernel writes q_out only for slot >= 0, so padded rows would + # be garbage after a DCP query all-gather. PCP and HiSparse rewrite the + # KV inputs/slots before the standalone write, which this path bypasses. + if attn.impl.dcp_world_size > 1 or attn.use_pcp: + return False + if attn.hisparse_cache is not None or attn.q_pad_num_heads is not None: + return False + # Only the W_UK variants reproduced in _fused_decode are supported. + return attn.is_aiter_triton_fp4_bmm_enabled or ( + not attn.is_aiter_triton_fp8_bmm_enabled and attn.W_UK_T is not None + ) def _normalize_q_kv( self, @@ -38,6 +69,107 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper): return q_layernorm(q_c), kv_layernorm(kv_c) + def _fused_decode( + self, + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + positions: torch.Tensor, + output_shape: tuple[int, int], + attn_metadata, + layer, + kv_cache: torch.Tensor, + slot_mapping: torch.Tensor, + ) -> torch.Tensor: + """Decode-only MLA through one fused AITER launch. + + Replaces the fp8 KV-cache write plus the ``[ql_nope | q_pe]`` concat and + that query's static fp8 quant with a single + ``fused_qk_rope_concat_and_cache_mla``. On an untouched upstream build + the concat and quant already arrive as one inductor-fused kernel, so + this collapses two launches into one. + """ + from aiter.ops.cache import fused_qk_rope_concat_and_cache_mla + + attn = self.mla_attn + q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) + # (B, N, P) -> (N, B, P) + q_nope_t = q_nope.transpose(0, 1) + + # Mirror MLAAttention.forward_impl's W_UK dispatch, *including its + # precedence* -- fp4 is checked before fp8 there, and both flags can be + # set at once. ql_nope must come out bit-identical to what the unfused + # path feeds its concat+quant, because this only replaces what happens + # after the bmm. Both variants return bf16, which the kernel consumes. + if attn.is_aiter_triton_fp4_bmm_enabled: + from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4 + + ql_nope = batched_gemm_a16wfp4( + q_nope_t, + attn.W_K, + attn.W_K_scale, + transpose_bm=True, + prequant=True, + y_scale=layer._q_scale, + ) + else: + assert attn.W_UK_T is not None # guaranteed by _fused_qk_prep_supported + B, N = q_nope.shape[0], q_nope.shape[1] + L = attn.W_UK_T.shape[-1] + ql_nope = q_nope_t.new_empty((B, N, L)) + torch.bmm(q_nope_t, attn.W_UK_T, out=ql_nope.transpose(0, 1)) + + if self._identity_rope is None: + # cos = 1, sin = 0 makes the kernel's RoPE the identity, which is + # what a NoPE model needs. One row suffices because the `_opt` + # kernel clamps `pos` into [0, cos_cache.size(0)); a full-length + # cache would cost ~268 MB at K3's 1M max_position_embeddings. + half = self.qk_rope_head_dim // 2 + opts = {"dtype": kv_c_normed.dtype, "device": kv_c_normed.device} + self._identity_rope = ( + torch.ones(1, half, **opts), + torch.zeros(1, half, **opts), + ) + cos_cache, sin_cache = self._identity_rope + + # An fp8 KV cache is allocated as uint8 and re-viewed as fp8 before use; + # the AITER kernel rejects the raw uint8 dtype outright. + fp8_dtype = current_platform.fp8_dtype() + if kv_cache.dtype != fp8_dtype: + kv_cache = kv_cache.view(fp8_dtype) + + head_size = attn.kv_lora_rank + self.qk_rope_head_dim + num_tokens, num_heads = ql_nope.shape[:2] + q_out = torch.empty( + (num_tokens, num_heads, head_size), + dtype=fp8_dtype, + device=ql_nope.device, + ) + fused_qk_rope_concat_and_cache_mla( + # the W_UK bmm hands back a transposed view; the kernel reads dense + ql_nope.contiguous(), + q_pe, + kv_c_normed, + k_pe.squeeze(1), + kv_cache.view(kv_cache.shape[0], -1, head_size), + q_out, + slot_mapping.flatten(), + layer._k_scale, + layer._q_scale, + positions, + cos_cache, + sin_cache, + is_neox=True, + is_nope_first=True, + ) + + # forward_mqa is MLA-specific and not on AttentionImpl; MLAAttention + # annotates its own call to it the same way. + attn_out, _ = attn.impl.forward_mqa(q_out, kv_cache, attn_metadata, attn) # type: ignore[attr-defined] + output = q.new_empty(output_shape) + attn._v_up_proj(attn_out, out=output) + return output + def forward( self, positions: torch.Tensor, @@ -106,13 +238,43 @@ class KimiK3MultiHeadLatentAttentionWrapper(MultiHeadLatentAttentionWrapper): if self.dcp_q_replicate: q_dcp_replicated, q = q, q_proj_layer._local_view(q) - attn_out = self.mla_attn( - q, - kv_c_normed, - k_pe, - output_shape=(hidden_states.shape[0], self.num_heads * self.v_head_dim), - q_dcp_replicated=q_dcp_replicated, - ) + output_shape = (hidden_states.shape[0], self.num_heads * self.v_head_dim) + + # The fused decode path covers only the MQA slice, so it is taken for + # decode-only batches; anything else falls through to MLAAttention. + attn_metadata = layer = kv_cache = slot_mapping = None + fuse = self._fused_qk_prep and q_dcp_replicated is None + if fuse: + attn_metadata, layer, kv_cache, slot_mapping = get_attention_context( + self.mla_attn.layer_name + ) + fuse = ( + attn_metadata is not None + and slot_mapping is not None + and kv_cache.numel() > 0 + and attn_metadata.num_actual_tokens == attn_metadata.num_decode_tokens + ) + + if fuse: + attn_out = self._fused_decode( + q, + kv_c_normed, + k_pe, + positions, + output_shape, + attn_metadata, + layer, + kv_cache, + slot_mapping, + ) + else: + attn_out = self.mla_attn( + q, + kv_c_normed, + k_pe, + output_shape=output_shape, + q_dcp_replicated=q_dcp_replicated, + ) if self.g_proj is not None: attn_out = attn_out * self.g_proj(hidden_states)[0].sigmoid() From b33e1cb172609343166ee485bf97c0a6f11587cf Mon Sep 17 00:00:00 2001 From: Aarushi Jain <142941703+aarushjain29@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:44:40 -0500 Subject: [PATCH 142/248] [ROCm]Transpose compressed-tensors MoE weights on device (#59253) Co-authored-by: Andreas Karatzas --- vllm/model_executor/layers/fused_moe/routed_experts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 3de0aa8dab..de08d9aa54 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -691,7 +691,10 @@ class RoutedExperts(PluggableLayer): ) and is_transposed ): - loaded_weight = loaded_weight.t().contiguous() + # Transpose on the parameter's device: a CPU transpose of the + # mmap view reads the file column-wise, which is very slow on + # NFS. The contiguous .to() reads it sequentially. + loaded_weight = loaded_weight.to(param.device).t().contiguous() if shard_id not in ("w1", "w2", "w3"): raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.") From ed3f6d1a56272a966bd9e514cb262c0ce3ee6822 Mon Sep 17 00:00:00 2001 From: Aarushi Jain <142941703+aarushjain29@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:51:20 -0500 Subject: [PATCH 143/248] [ROCm][CI] Remove duplicate MI355 DPX jobs (#59256) Signed-off-by: aarushjain29 Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 384 +-------------------------------------- 1 file changed, 3 insertions(+), 381 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 5620bc5f45..2b20e60d61 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3177,6 +3177,7 @@ steps: dind: false agent_pool: mi355_2 num_gpus: 2 + optional: true working_dir: "/vllm-workspace/" source_file_dependencies: - benchmarks/attention_benchmarks/ @@ -3921,78 +3922,6 @@ steps: - bash ../tools/install_torchcodec_rocm.sh || exit 1 - pytest -s entrypoints/openai/correctness/ -- label: ":amd: (MI355 DPX) Entrypoints Integration (API Server) %N (MI355 suite)" - timeout_in_minutes: 65 - parallelism: 4 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - fast_check: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/serve - - tests/entrypoints/scale_out - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc; fi - - pytest -v -s entrypoints/scale_out --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - -- label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API completion) (MI355 suite)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - fast_check: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/openai - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness - -- label: ":amd: (MI355 DPX) Entrypoints Integration (OpenAI API chat_completion) (MI355 suite)" # TBD - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - fast_check: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/entrypoints/openai - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s entrypoints/openai/chat_completion - -- label: ":amd: (MI355 DPX) Entrypoints Integration (API Server Generate) (MI355 suite)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - fast_check: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/tool_use - - tests/entrypoints/tool_parsers - - tests/entrypoints/anthropic - - tests/entrypoints/generate - - tests/entrypoints/cohere - commands: - - pytest -v -s tool_use - - pytest -v -s entrypoints/tool_parsers - - pytest -v -s entrypoints/generate - - pytest -v -s entrypoints/anthropic - - pytest -v -s entrypoints/cohere - - label: ":amd: (MI355 DPX) Entrypoints Integration (Speech to Text)" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -4435,6 +4364,7 @@ steps: dind: false agent_pool: mi355_2 num_gpus: 2 + optional: true working_dir: "/vllm-workspace" source_file_dependencies: - vllm/model_executor/models/ @@ -4898,7 +4828,7 @@ steps: commands: - pytest -v -s kernels/attention --ignore=kernels/attention/test_triton_unified_attention_diffkv.py --ignore=kernels/attention/test_rocm_aiter_mla_decode.py --ignore=kernels/attention/test_rocm_aiter_mla_decode_metadata.py --ignore=kernels/attention/test_rocm_aiter_mla_fp8_support.py --ignore=kernels/attention/test_rocm_aiter_mla_op_registration.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355 DPX) MoE Kernels Shard %N (MI355 suite)" # TBD +- label: ":amd: (MI355 DPX) MoE Kernels Shard %N" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] dind: false @@ -5313,20 +5243,6 @@ steps: commands: - pytest -v -s models/language/generation_ppl_test -- label: ":amd: (MI355 DPX) Language Models (Standard) (MI355 suite)" # TBD - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/language - optional: true - commands: - - pip freeze | grep -E 'torch' - - pytest -v -s models/language -m 'core_model and (not slow_test)' - #---------------------------------------------------- mi355 · models / multimodal ----------------------------------------------------# - label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2" # TBD @@ -5399,20 +5315,6 @@ steps: commands: - pytest -v -s models/multimodal/pooling -m 'not core_model' -- label: ":amd: (MI355 DPX) Multimodal Models (Standard) 1: qwen2 (MI355 suite)" # TBD - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" - - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model - - label: ":amd: (MI355 DPX) Multimodal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -5933,224 +5835,6 @@ steps: - pytest -v -s v1/attention --ignore=v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - VLLM_ROCM_USE_AITER=1 pytest -v -s v1/attention/test_mla_backends.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT -- label: ":amd: (MI355 DPX) V1 Core (MI355 suite)" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/cudagraph - - tests/v1/kv_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/simple_kv_offload - - tests/v1/streaming_input - - tests/v1/ec_connector/unit - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not cpu_test' v1/core - -- label: ":amd: (MI355 DPX) V1 Executor + Worker (MI355 suite)" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/cudagraph - - tests/v1/kv_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/simple_kv_offload - - tests/v1/streaming_input - - tests/v1/ec_connector/unit - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s v1/executor - - pytest -v -s v1/worker - - pytest -v -s v1/streaming_input - -- label: ":amd: (MI355 DPX) V1 KV Offload (MI355 suite)" - timeout_in_minutes: 60 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/cudagraph - - tests/v1/kv_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/simple_kv_offload - - tests/v1/streaming_input - - tests/v1/ec_connector/unit - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - bash /vllm-workspace/.buildkite/scripts/install-kv-offload.sh - - pytest -v -s v1/kv_offload - - pytest -v -s v1/simple_kv_offload - -- label: ":amd: (MI355 DPX) V1 KV Connectors Shard %N (MI355 suite)" - timeout_in_minutes: 60 - parallelism: 4 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/cudagraph - - tests/v1/kv_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/simple_kv_offload - - tests/v1/streaming_input - - tests/v1/ec_connector/unit - commands: - - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - - pytest -v -s -m 'not cpu_test' v1/ec_connector/unit --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - -- label: ":amd: (MI355 DPX) V1 Metrics + LM Eval (MI355 suite)" - timeout_in_minutes: 65 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/core - - tests/v1/executor - - tests/v1/kv_offload - - tests/v1/worker - - tests/v1/cudagraph - - tests/v1/kv_connector/unit - - tests/v1/metrics - - tests/entrypoints/openai/correctness/test_lmeval.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/entrypoints/pooling/ - - vllm/inputs/ - - vllm/lora/ - - vllm/model_executor/ - - vllm/multimodal/ - - vllm/outputs.py - - vllm/platforms/ - - vllm/pooling_params.py - - vllm/profiler/ - - vllm/sampling_params.py - - vllm/tokenizers/ - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - - tests/v1/simple_kv_offload - - tests/v1/streaming_input - - tests/v1/ec_connector/unit - commands: - - export GIT_CONFIG_COUNT=1 - - export GIT_CONFIG_KEY_0=http.version - - export GIT_CONFIG_VALUE_0=HTTP/1.1 - - export GIT_TERMINAL_PROMPT=0 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s -m 'not cpu_test' v1/metrics - - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine - - label: ":amd: (MI355 DPX) CUDAGraph" timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] @@ -6181,68 +5865,6 @@ steps: - pytest -v -s v1/cudagraph/test_encoder_cudagraph.py -- label: ":amd: (MI355 DPX) V1 Sample (MI355 suite)" - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1/sample - - tests/v1/logits_processors - - tests/v1/test_oracle.py - - tests/v1/test_request.py - - tests/v1/test_outputs.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/logger.py - - vllm/model_executor/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s v1/sample - -- label: ":amd: (MI355 DPX) V1 Logits + Oracle (MI355 suite)" - timeout_in_minutes: 70 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] - dind: false - agent_pool: mi355_dpx - optional: true - working_dir: /vllm-workspace/tests - source_file_dependencies: - - vllm/ - - tests/v1/sample - - tests/v1/logits_processors - - tests/v1/test_oracle.py - - tests/v1/test_request.py - - tests/v1/test_outputs.py - - vllm/config/ - - vllm/distributed/ - - vllm/engine/ - - vllm/inputs/ - - vllm/logger.py - - vllm/model_executor/ - - vllm/platforms/ - - vllm/sampling_params.py - - vllm/transformers_utils/ - - vllm/utils/ - - vllm/v1/ - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - pytest -v -s v1/logits_processors - - pytest -v -s v1/test_oracle.py - - pytest -v -s v1/test_request.py - - pytest -v -s v1/test_outputs.py - - - label: ":amd: (MI355 DPX) V1 Spec Decode" # TBD timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] From aba01ef1f73d5bf24952c8f7d61febc509bbfefc Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Thu, 1 Oct 2026 02:23:02 +0800 Subject: [PATCH 144/248] [XPU][CI] Skip test_core_engine_actor_manager.py on Intel CI (#59338) Signed-off-by: Chaojun Zhang Co-authored-by: Kunshang Ji --- .buildkite/intel_jobs/engine_intel.yaml | 2 +- .buildkite/intel_jobs/test-intel.yaml | 2 +- .buildkite/scripts/hardware_ci/run-intel-ci-test.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/intel_jobs/engine_intel.yaml b/.buildkite/intel_jobs/engine_intel.yaml index 98218f23b7..fd028142cc 100644 --- a/.buildkite/intel_jobs/engine_intel.yaml +++ b/.buildkite/intel_jobs/engine_intel.yaml @@ -62,7 +62,7 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py && + pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py --ignore v1/engine/test_core_engine_actor_manager.py && pytest -v -s test_config.py' - label: V1 e2e (2 GPUs) diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 7b9bd1e873..08f8dcdeab 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -99,7 +99,7 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pip install dpctl && pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py && - pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py && + pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py --ignore=v1/engine/test_core_engine_actor_manager.py && pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" && pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && pytest -v -s v1/structured_output && diff --git a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh index 997244bf59..c679d0aff2 100644 --- a/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-ci-test.sh @@ -37,7 +37,7 @@ case "${test_suite}" in cd tests pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py - pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py + pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py --ignore=v1/engine/test_core_engine_actor_manager.py pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py pytest -v -s v1/structured_output From a21a3f2ec76a02fada576b7e581a56307fb3e78f Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Wed, 30 Sep 2026 15:04:47 -0400 Subject: [PATCH 145/248] [CI] Add supports_multimodal_inputs to test_executor_replace's mock config (#59476) Signed-off-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- tests/renderers/test_executor_replace.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/renderers/test_executor_replace.py b/tests/renderers/test_executor_replace.py index fc0175e1e2..7aad93123a 100644 --- a/tests/renderers/test_executor_replace.py +++ b/tests/renderers/test_executor_replace.py @@ -34,6 +34,7 @@ class MockModelConfig: skip_tokenizer_init: bool = False is_encoder_decoder: bool = False is_multimodal_model: bool = False + supports_multimodal_inputs: bool = False renderer_num_workers: int = 1 hidden_size: int = 768 dtype: torch.dtype = torch.float32 From b56b54bc54073223d8acc04368fb004859df882e Mon Sep 17 00:00:00 2001 From: djramic Date: Wed, 30 Sep 2026 21:27:21 +0200 Subject: [PATCH 146/248] [ROCm][CI] Relax simple-nemotron-h-8b GSM8K threshold on ROCm (#59477) Signed-off-by: Djordje Ramic Co-authored-by: Andreas Karatzas --- tests/evals/gsm8k/test_gsm8k_offloading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/evals/gsm8k/test_gsm8k_offloading.py b/tests/evals/gsm8k/test_gsm8k_offloading.py index bcc001e2bd..9df8498d45 100644 --- a/tests/evals/gsm8k/test_gsm8k_offloading.py +++ b/tests/evals/gsm8k/test_gsm8k_offloading.py @@ -191,7 +191,7 @@ MODELS = [ id="simple-nemotron-h-8b", model="nvidia/Nemotron-H-8B-Base-8K", connector="SimpleCPUOffloadConnector", - accuracy_threshold=0.45, + accuracy_threshold=0.43 if current_platform.is_rocm() else 0.45, ), OffloadingModelConfig( id="simple-gemma-4-e4b-it", From 84f738f1878ff2ab7664f1a4464b7f9f3cdb063d Mon Sep 17 00:00:00 2001 From: Venky <23023424+venkywonka@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:33:23 -0700 Subject: [PATCH 147/248] [Perf][MiniMax-M3] Triton indexer: decode grid retune + SM12.0 split-K (#56151) Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Signed-off-by: Zijing Liu Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Zijing Liu Co-authored-by: Kyle Liang <199154931+kyleliang-nv@users.noreply.github.com> --- tests/kernels/attention/test_minimax_m3.py | 12 +++++-- .../minimax_m3/common/ops/index_topk.py | 33 ++++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index e3e4897e15..83eef72e31 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -614,14 +614,22 @@ def _reference_decode_index_score( return out -def test_prefill_index_topk_correctness(): +# 3600: 29 blocks over 12 splits of 3, so the last busy split has 2 blocks. +@pytest.mark.parametrize("prefix_len", [1024, 3600]) +@pytest.mark.parametrize("force_split_k", [False, True]) +def test_prefill_index_topk_correctness(monkeypatch, prefix_len, force_split_k): + if force_split_k: + # Take the SM12.0-only split-K launch on any CUDA GPU. + monkeypatch.setattr( + current_platform, "is_device_capability", lambda cap, *_: cap == (12, 0) + ) topk = 6 init_blocks = 0 local_blocks = 1 num_idx_heads = 2 head_dim = 16 q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32) - prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32) + prefix_lens = torch.tensor((0, prefix_len), device="cuda", dtype=torch.int32) seq_lens = prefix_lens + q_lens batch = q_lens.numel() max_seq_len = seq_lens.max().item() diff --git a/vllm/models/minimax_m3/common/ops/index_topk.py b/vllm/models/minimax_m3/common/ops/index_topk.py index 5123dabe90..94b74363e7 100644 --- a/vllm/models/minimax_m3/common/ops/index_topk.py +++ b/vllm/models/minimax_m3/common/ops/index_topk.py @@ -101,6 +101,7 @@ def _index_block_score_kernel( stride_bt_b, BLOCK_SIZE_Q: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128) + USE_SPLIT_K: tl.constexpr, ): pid_q = tl.program_id(0) pid_bh = tl.program_id(1) @@ -136,8 +137,19 @@ def _index_block_score_kernel( bt_row = block_table_ptr + pid_b * stride_bt_b # Causal window: only blocks up to the last query token's position. hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q) - for i in tl.range(0, hi, BLOCK_SIZE_K): - blk = i // BLOCK_SIZE_K + num_blocks = tl.cdiv(hi, BLOCK_SIZE_K) + if USE_SPLIT_K: + pid_k = tl.program_id(2) + blocks_per_split = tl.cdiv(num_blocks, tl.num_programs(2)) + block_start = pid_k * blocks_per_split + block_end = tl.minimum(block_start + blocks_per_split, num_blocks) + if block_start >= block_end: + return + else: + block_start = 0 + block_end = num_blocks + for blk in tl.range(block_start, block_end): + i = blk * BLOCK_SIZE_K page = tl.load(bt_row + blk).to(tl.int64) pos = i + off_k # index-K for this page: [BLOCK_SIZE_D, BLOCK_SIZE_K] (transposed) @@ -686,7 +698,17 @@ def minimax_m3_index_score( device=idx_q.device, ) BLOCK_SIZE_Q = 64 - grid_score = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads) + n_q_tiles = triton.cdiv(max_query_len, BLOCK_SIZE_Q) + SCORE_TARGET_GRID = 48 + split_k = max( + 1, + min(max_block, SCORE_TARGET_GRID // max(1, n_q_tiles * batch * num_idx_heads)), + ) + if not ( + current_platform.is_cuda() and current_platform.is_device_capability((12, 0)) + ): + split_k = 1 + grid_score = (n_q_tiles, batch * num_idx_heads, split_k) _index_block_score_kernel[grid_score]( idx_q, index_kv_cache, @@ -709,6 +731,7 @@ def minimax_m3_index_score( block_table.stride(0), BLOCK_SIZE_Q=BLOCK_SIZE_Q, BLOCK_SIZE_K=SPARSE_BLOCK_SIZE, + USE_SPLIT_K=split_k > 1, ) return score @@ -822,7 +845,7 @@ def minimax_m3_index_decode_score( ) # split-K over seq blocks; chunk count depends only on shape constants so # the grid is fixed within a cuda graph. - TARGET_GRID = 512 + TARGET_GRID = 4096 MAX_NUM_KV_CHUNKS = 256 # Use the configured max decode length to avoid Triton recompiles when # switching between qlen=1 and spec-decode verification batches. @@ -921,7 +944,7 @@ def minimax_m3_index_decode( ) # Chunk count is shape-constant (cudagraph-safe), capped so the merge sorts # pow2(num_topk_chunks * pow2(topk)) candidates. - TOPK_TARGET_GRID = 64 + TOPK_TARGET_GRID = 512 MAX_NUM_TOPK_CHUNKS = 16 topk_target = max( 1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, batch * num_idx_heads)) From 3eb6cec22ad9bb098393021b956feaa97081fc85 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Wed, 30 Sep 2026 15:43:09 -0400 Subject: [PATCH 148/248] [Bugfix][HiSparse] Adopt GPU prefix copies after the hit's allocation (#59282) Signed-off-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/v1/core/test_prefix_caching.py | 46 ++++++++++++++++++++++++++++ vllm/v1/hisparse/coordinator.py | 18 +++++++++-- 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index fab623eb84..e4c54e880c 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1139,6 +1139,52 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages(): ) +@pytest.mark.parametrize("free_blocks,adopted_pages", [(7, []), (10, [0, 1])]) +def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies( + free_blocks, adopted_pages +): + """A host prefix hit must not starve its own allocation of free blocks. + + Admission counts free GPU copies as free, but adopting a copy pins it. When + adoption ran before the hit's allocation, it took blocks the allocation was + promised and the pool ran dry (``Cannot get N free blocks``). The allocation + now goes first, evicting a prefix's copies tail first; the copies it leaves + are adopted. + """ + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) + original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + _publish_hisparse_pages(manager) + copy_ids = [block.block_id for block in manager.get_blocks("original").blocks[2]] + manager.free(original) + pool = manager.block_pool + pool.get_new_blocks(pool.get_num_free_blocks() - free_blocks) + + resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) + computed, num_computed, _ = manager.get_computed_blocks(resumed) + assert num_computed == 3 * HISPARSE_BLOCK_SIZE + assert manager.allocate_slots( + resumed, + num_new_tokens=len(tokens) - num_computed, + num_new_computed_tokens=num_computed, + new_computed_blocks=computed, + ) + + resumed_blocks = manager.get_blocks("resumed").blocks + assert [block.block_id for block in resumed_blocks[2][:3]] == [ + copy_ids[page] if page in adopted_pages else pool.null_block.block_id + for page in range(3) + ] + gpu_ids = [ + block.block_id + for group in resumed_blocks[1:] + for block in group + if not block.is_null + ] + assert len(gpu_ids) == len(set(gpu_ids)) + + def test_hisparse_host_backed_request_accepts_local_prefix_hit(): """Local group-completion hits need no external host import allocation.""" manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index 26357e0a2a..01a6392964 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -66,6 +66,8 @@ class _HiSparseRequestState: copies_recorded_blocks: int = 0 pinned_clean: set[int] = field(default_factory=set) unpinned_pages: set[int] = field(default_factory=set) + # Prefix pages whose GPU copies are adopted after the admitting allocation. + pages_to_adopt: int = 0 @dataclass @@ -195,7 +197,9 @@ class HiSparseCoordinator: state = self._get_request_state(request_id) state.valid_pages.update(range(num_host_pages)) state.ready_prefix_pages = max(num_host_pages, state.ready_prefix_pages) - self._adopt_copies(request_id, state, host_blocks[:num_host_pages]) + # Adopting pins copies out of the free pool, so it must wait until the + # allocation admission counted on has run (see update_residency). + state.pages_to_adopt = max(state.pages_to_adopt, num_host_pages) state.copies_recorded_blocks = max(state.copies_recorded_blocks, num_host_pages) def take_host_block_copies(self) -> tuple[KVCacheBlockCopy, ...]: @@ -345,7 +349,8 @@ class HiSparseCoordinator: return True def _unpin_clean_pages(self, request_id: str, state: _HiSparseRequestState) -> None: - for page_idx in sorted(state.pinned_clean): + # Tail first, so a prefix loses its tail pages before its head. + for page_idx in sorted(state.pinned_clean, reverse=True): self._unpin_page(request_id, state, page_idx) def _page_became_clean( @@ -391,6 +396,11 @@ class HiSparseCoordinator: if not self.resident_managers: return state = self._get_request_state(request_id) + if state.pages_to_adopt: + assert self.host_manager is not None + host_blocks = self.host_manager.req_to_blocks.get(request_id, ()) + self._adopt_copies(request_id, state, host_blocks[: state.pages_to_adopt]) + state.pages_to_adopt = 0 if not self._can_read_from_host(request_id): assert self.gpu_pool is not None if self.gpu_pool.get_num_free_blocks() >= self.transition_watermark: @@ -791,7 +801,9 @@ class HiSparseCoordinator: blocks = manager.req_to_blocks.get(request_id) if not blocks: continue - for page_idx, block in enumerate(blocks): + # Tail first, so a prefix loses its tail pages before its head. + for page_idx in reversed(range(len(blocks))): + block = blocks[page_idx] if block.is_null: continue if page_idx in state.unpinned_pages: From 51048b13ce4229b85b980b582fda07c0aad81415 Mon Sep 17 00:00:00 2001 From: Nils Matteson Date: Wed, 30 Sep 2026 13:31:21 -0700 Subject: [PATCH 149/248] [Docs] Clarify snapshot runtime image support (#59127) Signed-off-by: Nils Matteson --- docs/features/initialized_snapshots.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/features/initialized_snapshots.md b/docs/features/initialized_snapshots.md index 538695d0a8..15cce7554b 100644 --- a/docs/features/initialized_snapshots.md +++ b/docs/features/initialized_snapshots.md @@ -42,10 +42,10 @@ Snapshots currently require: and CRIU still reopens it by path, so replacing that file permanently invalidates the artifact without an early error. -The official `vllm/vllm-openai` Linux x86-64 image includes the snapshot -runtime. It still requires a compatible host driver, kernel, and privileges. -Arm64 images omit it. Source installs must set `CRIU_CUDA_PLUGIN_DIR` to the -directory containing `cuda_plugin.so`. +The official CUDA 13 `vllm/vllm-openai` Linux x86-64 images include the +snapshot runtime. CUDA 12.x and Arm64 images omit it. A compatible host +driver, kernel, and privileges are still required. Source installs must set +`CRIU_CUDA_PLUGIN_DIR` to the directory containing `cuda_plugin.so`. Run snapshot commands with `docker exec` inside a long-lived container. Restore hands the API server off as a detached process, so a one-shot container would From 73c7cae4d746f64c22677348d1cc120eea6f7439 Mon Sep 17 00:00:00 2001 From: "Yueh-Ting (eop) Chen" Date: Thu, 1 Oct 2026 04:50:01 +0800 Subject: [PATCH 150/248] [Bugfix][HiSparse] Stop the host pool feeding device KV cache residency metrics (#58725) Signed-off-by: Yueh-Ting Chen Signed-off-by: Matthew Bonanni Co-authored-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- vllm/v1/core/single_type_kv_cache_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 31e610b578..3d784891c5 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2247,7 +2247,7 @@ class HiSparseSourceManager(FullAttentionManager): enable_caching=self.enable_caching and self.kv_cache_spec.prefix_cacheable, hash_block_size=device_pool.hash_block_size, enable_kv_cache_events=device_pool.enable_kv_cache_events, - metrics_collector=device_pool.metrics_collector, + metrics_collector=None, medium=MEDIUM_CPU, event_owner=device_pool, ) From 42f0c17ea755a70656f92d278ba086e31fc72086 Mon Sep 17 00:00:00 2001 From: Venky <23023424+venkywonka@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:14:24 -0700 Subject: [PATCH 151/248] [Bugfix][PP][Spec Decode] MiniMax-M3 EAGLE3 aux-state relay at PP > 1 and per-stage FlashInfer autotune (#57197) Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) --- .../test_flashinfer_autotune_warmup.py | 192 +++++++++++++++++- .../test_eagle3_aux_hidden_states_pp.py | 152 ++++++++++++++ vllm/model_executor/warmup/kernel_warmup.py | 30 ++- vllm/models/minimax_m3/nvidia/model.py | 23 ++- 4 files changed, 382 insertions(+), 15 deletions(-) diff --git a/tests/model_executor/test_flashinfer_autotune_warmup.py b/tests/model_executor/test_flashinfer_autotune_warmup.py index dd162629b8..22bb54d71f 100644 --- a/tests/model_executor/test_flashinfer_autotune_warmup.py +++ b/tests/model_executor/test_flashinfer_autotune_warmup.py @@ -1,16 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json +import sys +from collections import defaultdict +from contextlib import nullcontext from inspect import signature -from types import SimpleNamespace +from pathlib import Path +from types import ModuleType, SimpleNamespace from typing import Any from unittest.mock import Mock, call, patch import pytest +from vllm.model_executor.warmup import kernel_warmup as warmup from vllm.model_executor.warmup.kernel_warmup import ( _flashinfer_autotune_token_counts, _run_flashinfer_autotune_dummy_runs, + flashinfer_autotune, ) pytestmark = pytest.mark.cpu_test @@ -37,7 +44,8 @@ def _make_runner(modules, *, max_tokens: int = 8192, linear_backend: str = "auto return SimpleNamespace( scheduler_config=SimpleNamespace(max_num_batched_tokens=max_tokens), vllm_config=SimpleNamespace( - kernel_config=SimpleNamespace(linear_backend=linear_backend) + kernel_config=SimpleNamespace(linear_backend=linear_backend), + attention_config=SimpleNamespace(hisparse_config=None), ), get_model=Mock( return_value=SimpleNamespace(modules=Mock(return_value=modules)) @@ -125,3 +133,183 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn): **({"skip_attn": True} if skip_attn else {}), ), ] + + +class _AutotuneGroup: + def __init__(self, run, ranks): + self.run = run + self.ranks = tuple(ranks) + self.world_size = len(self.ranks) + self.rank_in_group = self.ranks.index(run.rank) + self.cpu_group = self + + def record(self, operation): + self.run.collectives[self.ranks][self.run.rank].append(operation) + + def broadcast_object(self, obj, src=0): + assert src == 0 + self.record(("broadcast", src)) + if self.rank_in_group == src: + self.run.broadcasts[self.ranks] = obj + return self.run.broadcasts[self.ranks] + + def barrier(self): + self.record(("barrier",)) + + +class _AutotuneTuner: + def __init__(self, run): + self.run = run + self.cache = {} + self.loaded = None + + def load_configs(self, path): + self.loaded = json.loads(Path(path).read_text()) + self.cache.update(self.loaded) + + def save_configs(self, path): + self.run.saves.append((self.run.rank, Path(path), dict(self.cache))) + Path(path).write_text(json.dumps(self.cache)) + + def profile(self, operation): + if operation in self.cache: + return + group = self.run.tuning_group + self.run.profile_groups[self.run.rank].append( + None if group is None else group.ranks + ) + for tactic in range(2): + if group is not None: + group.record(("all_reduce", operation, tactic)) + self.cache[operation] = self.run.rank // self.run.tp + + +class _AutotuneRun: + def __init__(self, pp, tp): + self.pp, self.tp = pp, tp + self.rank = 0 + self.tuning_group = None + self.collectives: dict[tuple[int, ...], dict[int, list[tuple[Any, ...]]]] = ( + defaultdict(lambda: defaultdict(list)) + ) + self.broadcasts = {} + self.tuners = {} + self.saves = [] + self.profile_groups = defaultdict(list) + + def world(self): + return _AutotuneGroup(self, range(self.pp * self.tp)) + + def tensor_group(self): + start = self.rank // self.tp * self.tp + return _AutotuneGroup(self, range(start, start + self.tp)) + + def pipeline_group(self): + return SimpleNamespace(world_size=self.pp) + + def set_group(self, group): + self.tuning_group = group + + def dummy_runs(self, runner, *, skip_attn=False): + self.tuners[self.rank].profile("shared_gemm") + if self.pp > 1 and self.rank // self.tp == 0: + self.tuners[self.rank].profile("pp0_extra_gemm") + + def execute(self): + for rank in range(self.pp * self.tp): + self.rank = rank + self.tuners[rank] = _AutotuneTuner(self) + flashinfer_autotune(_make_runner([])) + return self + + def assert_collectives_match(self): + for ranks, traces in self.collectives.items(): + assert set(traces) == set(ranks) + expected = traces[ranks[0]] + assert all(trace == expected for trace in traces.values()), dict(traces) + + +@pytest.fixture +def autotune_run(monkeypatch, tmp_path): + import vllm.utils.flashinfer as fi_utils + from vllm.distributed import parallel_state + + def make_run(*, pp=2, tp=4): + run = _AutotuneRun(pp, tp) + autotuner = ModuleType("flashinfer.autotuner") + monkeypatch.setattr( + autotuner, + "AutoTuner", + SimpleNamespace(get=lambda: run.tuners[run.rank]), + raising=False, + ) + monkeypatch.setattr( + autotuner, "set_autotune_process_group", run.set_group, raising=False + ) + monkeypatch.setitem(sys.modules, "flashinfer.autotuner", autotuner) + monkeypatch.setattr(parallel_state, "get_world_group", run.world) + monkeypatch.setattr(parallel_state, "get_tp_group", run.tensor_group) + monkeypatch.setattr(parallel_state, "get_pp_group", run.pipeline_group) + monkeypatch.setattr(fi_utils, "autotune", lambda **kwargs: nullcontext()) + monkeypatch.setattr( + warmup, + "resolve_flashinfer_autotune_file", + lambda runner: tmp_path / "autotune_configs.json", + ) + monkeypatch.setattr( + warmup, "_flashinfer_autotune_skip_ops", lambda runner: None + ) + monkeypatch.setattr( + warmup, "_run_flashinfer_autotune_dummy_runs", run.dummy_runs + ) + monkeypatch.setattr(warmup, "replayssm_autotune_warmup", lambda runner: None) + monkeypatch.setattr(warmup, "_autotune_kimi_k3_kda_qkvg", lambda model: None) + return run + + return make_run + + +@pytest.mark.parametrize("tp", [1, 4]) +def test_heterogeneous_pp_stages_have_compatible_collectives(autotune_run, tp): + run = autotune_run(tp=tp).execute() + run.assert_collectives_match() + for rank, groups in run.profile_groups.items(): + expected = tuple(range(rank // tp * tp, (rank // tp + 1) * tp)) + assert groups and all( + group == (expected if tp > 1 else None) for group in groups + ) + + +def test_pp_stage_cache_roundtrip_isolated_and_asymmetric_hits_safe( + autotune_run, tmp_path +): + legacy = tmp_path / "autotune_configs.json" + legacy.write_text('{"legacy_world_cache": 99}') + cold = autotune_run().execute() + assert [rank for rank, _, _ in cold.saves] == [0, 4] + paths = [path for _, path, _ in cold.saves] + assert len(set(paths)) == 2 and legacy not in paths + assert json.loads(legacy.read_text()) == {"legacy_world_cache": 99} + assert cold.saves[0][2] == {"shared_gemm": 0, "pp0_extra_gemm": 0} + assert cold.saves[1][2] == {"shared_gemm": 1} + cold.assert_collectives_match() + warm = autotune_run().execute() + warm.assert_collectives_match() + assert not warm.profile_groups + for rank, tuner in warm.tuners.items(): + assert tuner.loaded == cold.saves[rank // 4][2] + paths[1].unlink() + mixed = autotune_run().execute() + mixed.assert_collectives_match() + assert set(mixed.profile_groups) == {4, 5, 6, 7} + assert all(mixed.tuners[rank].loaded is not None for rank in range(4)) + assert all(mixed.tuners[rank].loaded is None for rank in range(4, 8)) + + +def test_pp1_retains_world_synchronization_and_existing_cache_name(autotune_run): + run = autotune_run(pp=1, tp=4).execute() + run.assert_collectives_match() + assert [(rank, path.name) for rank, path, _ in run.saves] == [ + (0, "autotune_configs.json") + ] + assert all(groups == [(0, 1, 2, 3)] for groups in run.profile_groups.values()) diff --git a/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py b/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py index b22da39c91..309a81b36c 100644 --- a/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py +++ b/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py @@ -3,10 +3,15 @@ from types import SimpleNamespace import pytest +import torch +from vllm.distributed import parallel_state from vllm.model_executor.models.interfaces import EagleModelMixin from vllm.model_executor.models.mimo import MiMoModel +from vllm.models.minimax_m3.nvidia import model as minimax_m3 +from vllm.sequence import IntermediateTensors from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( + reserve_aux_intermediate_tensor_slots, verify_supports_aux_hidden_states_over_pp, ) @@ -24,3 +29,150 @@ def test_mimo_does_not_inherit_aux_hidden_state_pp_support(): assert not inner.supports_aux_hidden_states_over_pp with pytest.raises(ValueError, match="does not support eagle3"): verify_supports_aux_hidden_states_over_pp(target, "eagle3") + + +class _PPAuxLayer(torch.nn.Module): + """Aliases and mutates residual in place like MiniMaxM3DecoderLayer.""" + + ffn_all_reduce_deferred = False + + def __init__(self, index): + super().__init__() + self.index = index + + def forward(self, positions, hidden_states, residual): + total = hidden_states if residual is None else hidden_states + residual + residual = hidden_states if residual is None else residual + residual.copy_(total + self.index + 1) + return torch.full_like(hidden_states, 10 * (self.index + 1)), residual + + +class _PPAuxNorm(torch.nn.Module): + """Clobbers its inputs so aux taps that alias them would fail.""" + + def forward(self, hidden_states, residual): + output = hidden_states + residual + hidden_states.fill_(-101) + residual.fill_(-103) + return output, residual + + +@pytest.fixture +def minimax_pp_stage(monkeypatch): + """Use the real MiniMax constructor/forward and aux helpers on CPU tensors.""" + group = SimpleNamespace(world_size=2, is_first_rank=True, is_last_rank=False) + bounds = [0, 3] + monkeypatch.setattr(minimax_m3, "get_pp_group", lambda: group) + monkeypatch.setattr(parallel_state, "get_pp_group", lambda: group) + monkeypatch.setattr(parallel_state, "model_parallel_is_initialized", lambda: True) + monkeypatch.setattr( + minimax_m3, + "VocabParallelEmbedding", + lambda *args, **kwargs: torch.nn.Identity(), + ) + monkeypatch.setattr( + minimax_m3, "MiniMAXGemmaRMSNorm", lambda *args, **kwargs: _PPAuxNorm() + ) + monkeypatch.setattr( + minimax_m3, + "MiniMaxM3DecoderLayer", + lambda **kwargs: _PPAuxLayer(int(kwargs["prefix"].rsplit(".", 1)[1])), + ) + + def make_layers(num_layers, build, prefix): + return ( + *bounds, + torch.nn.ModuleList( + build(f"{prefix}.{index}") + if bounds[0] <= index < bounds[1] + else torch.nn.Identity() + for index in range(num_layers) + ), + ) + + monkeypatch.setattr(minimax_m3, "make_layers", make_layers) + + def make_stage(start, end, taps, world_size=2): + bounds[:] = [start, end] + group.world_size = world_size + group.is_first_rank = start == 0 + group.is_last_rank = end == 6 + config = SimpleNamespace( + vocab_size=16, hidden_size=6144, num_hidden_layers=6, rms_norm_eps=1e-6 + ) + model = minimax_m3.MiniMaxM3Model( + vllm_config=SimpleNamespace( + model_config=SimpleNamespace(hf_text_config=config), quant_config=None + ) + ) + model._set_aux_hidden_state_layers(taps) + target = SimpleNamespace( + model=model, + make_empty_intermediate_tensors=model.make_empty_intermediate_tensors, + ) + reserve_aux_intermediate_tensor_slots(target) + return model, target + + return make_stage + + +# Input 7; _PPAuxLayer i adds 11 * (i + 1) to hidden + residual. +def _expected_aux_value(tap): + return 7 + 11 * tap * (tap + 1) // 2 + + +def _pp_aux_state(value): + return torch.full((2, 6144), float(value), device="cpu") + + +@pytest.mark.parametrize("taps", [(0, 3, 6), (0, 1, 3), (4, 5, 6)]) +def test_minimax_pp2_preserves_ordered_global_aux_taps(minimax_pp_stage, taps): + first, _ = minimax_pp_stage(0, 3, taps) + sent = first(None, None, None, _pp_aux_state(7)) + last, target = minimax_pp_stage(3, 6, taps) + reserved = target.make_empty_intermediate_tensors(2, torch.float32, "cpu") + assert set(sent.tensors) == set(reserved.tensors) + result = last(None, None, sent) + assert isinstance(result, tuple) + output, aux = result + torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6))) + for hidden, tap in zip(aux, taps, strict=True): + torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap))) + + +def test_minimax_missing_upstream_aux_slot_is_not_silently_dropped(minimax_pp_stage): + model, _ = minimax_pp_stage(3, 6, (0, 3, 6)) + incoming = IntermediateTensors( + { + "hidden_states": _pp_aux_state(30), + "residual": _pp_aux_state(43), + "aux_hidden_states_0": _pp_aux_state(7), + } + ) + with pytest.raises(RuntimeError, match="Missing aux_hidden_states_1"): + model(None, None, incoming) + + +@pytest.mark.parametrize("taps", [(), (0, 3, 6)]) +def test_minimax_pp1_output_and_aux_taps(minimax_pp_stage, taps): + model, _ = minimax_pp_stage(0, 6, taps, world_size=1) + output = model(None, None, None, _pp_aux_state(7)) + if taps: + output, aux = output + for hidden, tap in zip(aux, taps, strict=True): + torch.testing.assert_close(hidden, _pp_aux_state(_expected_aux_value(tap))) + torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6))) + + +def test_minimax_pp2_without_aux_keeps_hidden_and_residual_transport(minimax_pp_stage): + first, _ = minimax_pp_stage(0, 3, ()) + sent = first(None, None, None, _pp_aux_state(7)) + assert set(sent.tensors) == {"hidden_states", "residual"} + torch.testing.assert_close(sent["hidden_states"], _pp_aux_state(30)) + torch.testing.assert_close(sent["residual"], _pp_aux_state(43)) + last, target = minimax_pp_stage(3, 6, ()) + assert set( + target.make_empty_intermediate_tensors(12, torch.bfloat16, "cpu").tensors + ) == {"hidden_states", "residual"} + output = last(None, None, sent) + torch.testing.assert_close(output, _pp_aux_state(_expected_aux_value(6))) diff --git a/vllm/model_executor/warmup/kernel_warmup.py b/vllm/model_executor/warmup/kernel_warmup.py index 33fe7002aa..228ce73803 100644 --- a/vllm/model_executor/warmup/kernel_warmup.py +++ b/vllm/model_executor/warmup/kernel_warmup.py @@ -410,17 +410,24 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: Without autotuning, FlashInfer will rely on heuristics, which may be significantly slower. - Every rank profiles the same tactics. When distributed, per-tactic - timings are averaged over the world CPU group so all ranks select the - same tactic. + With PP > 1, stages run different layers and may profile different ops, + so each stage's TP group tunes separately with its own cache file; + otherwise the world group tunes together. Per-tactic timings are + averaged over the tuning group so all its ranks select the same tactic. """ from flashinfer.autotuner import AutoTuner, set_autotune_process_group import vllm.utils.flashinfer as fi_utils - from vllm.distributed.parallel_state import get_world_group + from vllm.distributed.parallel_state import ( + get_pp_group, + get_tp_group, + get_world_group, + ) world = get_world_group() - is_leader = world.rank_in_group == 0 + pp_size = get_pp_group().world_size + tune_group = get_tp_group() if pp_size > 1 else world + is_leader = tune_group.rank_in_group == 0 tuner = AutoTuner.get() autotune_kwargs: dict = {} @@ -433,6 +440,11 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: autotune_kwargs["skip_ops"] = skip_ops cache_path = resolve_flashinfer_autotune_file(runner) + if pp_size > 1: + ranks = "-".join(str(rank) for rank in tune_group.ranks) + cache_path = cache_path.with_name( + f"{cache_path.stem}_tp_{ranks}{cache_path.suffix}" + ) if is_leader: logger.info_once("Using FlashInfer autotune cache file: %s", cache_path) @@ -441,18 +453,18 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None: # which lead to some EP ranks receiving no tokens and skipping their # MoE kernel entirely, and cause hang due to all-reduce collective # during synchronized autotuning. - # Read cached autotune results and broadcast to all ranks. + # Read cached autotune results and broadcast within the tuning group. cached_results: bytes | None = None if is_leader and cache_path.exists(): with open(cache_path, "rb") as f: cached_results = f.read() - cached_results = world.broadcast_object(cached_results, src=0) + cached_results = tune_group.broadcast_object(cached_results, src=0) if cached_results is not None: write_flashinfer_autotune_cache(cache_path, cached_results) - world.barrier() + tune_group.barrier() tuner.load_configs(str(cache_path)) - group = world.cpu_group if world.world_size > 1 else None + group = tune_group.cpu_group if tune_group.world_size > 1 else None set_autotune_process_group(group) try: with ( diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index d1c0a01c40..1c0138946f 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -794,6 +794,7 @@ class MiniMaxM3DecoderLayer(nn.Module): class MiniMaxM3Model(nn.Module, EagleModelMixin): fall_back_to_pt_during_load = False + supports_aux_hidden_states_over_pp = True def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -886,9 +887,18 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - # EAGLE3 is not yet compatible with pipeline parallel - aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) - for idx, layer in enumerate(self.layers[self.start_layer : self.end_layer]): + remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors) + aux_hidden_states: list[torch.Tensor] = [] + if get_pp_group().is_first_rank: + self._maybe_add_hidden_state( + aux_hidden_states, self.start_layer, hidden_states, residual + ) + # Preserve the entry tap before in-place residual updates. + if aux_hidden_states: + aux_hidden_states[0] = aux_hidden_states[0].clone() + for idx, layer in enumerate( + self.layers[self.start_layer : self.end_layer], start=self.start_layer + ): hidden_states, residual = layer(positions, hidden_states, residual) self._maybe_add_hidden_state( aux_hidden_states, idx + 1, hidden_states, residual @@ -896,7 +906,11 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): if not get_pp_group().is_last_rank: return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} + { + "hidden_states": hidden_states, + "residual": residual, + **self.pack_local_aux_hidden_states(aux_hidden_states), + } ) if self.fuse_final_norm_allreduce: @@ -906,6 +920,7 @@ class MiniMaxM3Model(nn.Module, EagleModelMixin): else: hidden_states, _ = self.norm(hidden_states, residual) + aux_hidden_states = remote_aux + aux_hidden_states if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states return hidden_states From 7e583e615c20ee4ff0cd82aac592c7d6310aa7c3 Mon Sep 17 00:00:00 2001 From: kliuae <17350011+kliuae@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:26:54 +0800 Subject: [PATCH 152/248] [Bugfix][Core] Fix mamba prefill checkpoint block reservation and prompt-end eviction in align mode (#59175) Signed-off-by: kliuae Co-authored-by: Nick Hill --- .../test_partial_prefix_cache_hits.py | 147 ++++++++++++++++++ vllm/v1/core/kv_cache_coordinator.py | 9 ++ vllm/v1/core/kv_cache_manager.py | 6 + vllm/v1/core/sched/scheduler.py | 1 + vllm/v1/core/single_type_kv_cache_manager.py | 19 +++ 5 files changed, 182 insertions(+) diff --git a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py index 3b5fd218a6..76beffce45 100644 --- a/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py +++ b/tests/v1/core/prefix_cache/test_partial_prefix_cache_hits.py @@ -13,6 +13,9 @@ import torch from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request from vllm.distributed.kv_transfer.kv_connector.v1.base import SupportsHMA +from vllm.model_executor.layers.mamba.checkpoint import ( + compute_mamba_prefill_checkpoints, +) from vllm.utils.hashing import sha256 from vllm.v1.core.kv_cache_utils import ( KVCacheBlockCopy, @@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, MambaSpec, SlidingWindowSpec, + get_mamba_prefill_checkpoint_position, ) @@ -686,6 +690,81 @@ def test_transient_checkpoint_evicts_retained_boundary_hash(): assert manager.block_pool.get_cached_block(checkpoint_hash, [1]) is None +@pytest.mark.parametrize("prompt_len", [120, 104, 96]) +def test_prompt_end_checkpoint_survives_block_aligned_cache_count(prompt_len): + """At the default match unit the hybrid coordinator hands ``cache_blocks`` + a block-aligned token count, which is below the prompt length whenever the + prompt is not block-aligned. The prompt-end checkpoint is the replay + boundary and must stay published under retention_interval=0; only a + mid-prompt checkpoint is transient. + """ + hash_block_size = mamba_block_size = 32 + manager = make_full_mamba_manager( + dcp_world_size=1, + hash_block_size=hash_block_size, + full_block_size=hash_block_size, + mamba_block_size=mamba_block_size, + num_prefill_checkpoint_blocks=1, + ) + manager.coordinator.retention_interval = 0 + assert not manager.coordinator.enable_partial_hash_hits + + request = make_request("producer", list(range(prompt_len)), hash_block_size, sha256) + assert manager.allocate_slots(request, request.num_tokens) is not None + checkpoint_position = get_mamba_prefill_checkpoint_position( + prompt_len, hash_block_size, drop_eagle_block=False + ) + checkpoint_hash = request.block_hashes[checkpoint_position // hash_block_size - 1] + hit = manager.block_pool.get_cached_block(checkpoint_hash, [1]) + assert hit is not None + assert hit[0].block_hash_num_tokens == checkpoint_position + + manager.free(request) + replay = make_request("replay", list(range(prompt_len)), hash_block_size, sha256) + _, num_computed, _ = manager.get_computed_blocks(replay) + assert num_computed == checkpoint_position + + +@pytest.mark.parametrize("retention_interval,reserved", [(None, True), (0, False)]) +def test_intermediate_chunk_checkpoint_reserved_only_under_dense_retention( + retention_interval, reserved +): + """A checkpoint inside a chunk that does not finish the prefill is only + publishable under dense retention. With retention_interval=0 it would be + evicted as transient, so the block is not reserved in the first place; + the prefill-end checkpoint is reserved either way. + """ + hash_block_size = 16 + manager = make_full_mamba_manager( + dcp_world_size=1, + hash_block_size=hash_block_size, + full_block_size=hash_block_size, + mamba_block_size=32, + num_prefill_checkpoint_blocks=1, + ) + manager.coordinator.retention_interval = retention_interval + mamba_manager = manager.coordinator.single_type_managers[1] + + request = make_request("producer", list(range(240)), hash_block_size, sha256) + computed_blocks, num_computed, _ = manager.get_computed_blocks(request) + free_before = manager.block_pool.get_num_free_blocks() + + # Mid-prompt chunk: the chunk-keyed position (112) is inside it. Blocks: + # the attention blocks, the running mamba block, plus the checkpoint. + assert manager.allocate_slots(request, 128, num_computed, computed_blocks) + assert (request.request_id in mamba_manager._checkpoints) == reserved + allocated = free_before - manager.block_pool.get_num_free_blocks() + assert allocated == 128 // hash_block_size + 1 + int(reserved) + + request.num_computed_tokens = 128 + manager.new_step_starts() + # Final chunk: the prefill-end checkpoint at 224 is reserved either way. + assert manager.allocate_slots(request, 112) is not None + assert mamba_manager._checkpoints[request.request_id][0] == 224 + end_hash = request.block_hashes[224 // hash_block_size - 1] + assert manager.block_pool.get_cached_block(end_hash, [1]) is not None + + def test_eagle_block_aligned_checkpoint_replaces_newer_hash(): hash_block_size = mamba_block_size = 32 manager = make_full_mamba_manager( @@ -2565,3 +2644,71 @@ def test_prompt_end_checkpoint_survives_sparse_retention_with_coarse_hash(): follower = make_request("follower", list(range(240)) + [7] * 40, block_size, sha256) _, num_hit, _ = manager.get_computed_blocks(follower) assert num_hit == 224 + + +@pytest.mark.parametrize( + "hash_block_size,mamba_block_size,use_eagle", + [(32, 32, False), (16, 32, False), (32, 32, True)], +) +@pytest.mark.parametrize("retention_interval", [0, None]) +@pytest.mark.parametrize( + "num_prompt_tokens,num_output_tokens", + [(104, 0), (200, 0), (40, 24), (40, 25), (150, 43)], +) +@pytest.mark.parametrize("chunk_blocks", [2, 100]) +def test_checkpoint_reservation_matches_worker( + hash_block_size, + mamba_block_size, + use_eagle, + retention_interval, + num_prompt_tokens, + num_output_tokens, + chunk_blocks, +): + """The worker exports a checkpoint wherever `compute_mamba_prefill_checkpoints` + says, into whatever block sits at its column. Each chunk, the manager must + either reserve exactly that position and column, or leave the column null so + the export is masked; anything else publishes a hash over the wrong state. + Under sparse retention, only the prefill-end chunk may reserve one. + """ + manager = make_full_mamba_manager( + dcp_world_size=1, + hash_block_size=hash_block_size, + full_block_size=hash_block_size, + mamba_block_size=mamba_block_size, + num_blocks=64, + use_eagle=use_eagle, + num_speculative_blocks=3 if use_eagle else 0, + num_prefill_checkpoint_blocks=1, + ) + manager.coordinator.retention_interval = retention_interval + mamba_manager = manager.coordinator.single_type_managers[1] + + request = make_request( + "req", list(range(num_prompt_tokens)), hash_block_size, sha256 + ) + request.append_output_token_ids(list(range(1000, 1000 + num_output_tokens))) + num_tokens = request.num_tokens + prefill_end = max(num_prompt_tokens, num_tokens - 1) + + start = 0 + while start < num_tokens: + end = min(start + chunk_blocks * mamba_block_size, num_tokens) + assert manager.allocate_slots(request, end - start) is not None + offsets, cols = compute_mamba_prefill_checkpoints( + [end], + [end - start], + hash_block_size=hash_block_size, + mamba_block_size=mamba_block_size, + checkpoint_alignment=16, + drop_eagle_block=use_eagle, + ) + reserved = mamba_manager._checkpoints.get(request.request_id) + if reserved is not None: + assert reserved == (start + offsets[0], cols[0]) + assert retention_interval is None or end >= prefill_end + elif offsets[0]: + assert mamba_manager.req_to_blocks[request.request_id][cols[0]].is_null + request.num_computed_tokens = end + manager.new_step_starts() + start = end diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index d69b2f5fb3..daf0cd12c0 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -174,6 +174,7 @@ class KVCacheCoordinator(ABC): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: """Get the number of device blocks needed to be allocated for the request. @@ -195,12 +196,19 @@ class KVCacheCoordinator(ABC): per-request admission cap (SWA / chunked-local). Set only by the full-sequence admission gate; per-step allocation must leave it False so the predictor matches `allocate_new_blocks`. + prefill_end: The token index the request's prefill ends at, the + same value the scheduler splits chunks against. Under sparse + retention Mamba reserves a prefill checkpoint only on the chunk + reaching it; under dense retention every chunk publishes a + state, so it is ignored (0). Returns: The number of blocks to allocate. """ num_blocks_to_allocate = 0 + if self.retention_interval != 0: + prefill_end = 0 for i, manager in enumerate(self.single_type_managers): if isinstance(manager, CrossAttentionManager): # For cross-attention, we issue a single static allocation @@ -223,6 +231,7 @@ class KVCacheCoordinator(ABC): num_local_computed_tokens, num_tokens_main_model, apply_admission_cap=apply_admission_cap, + prefill_end=prefill_end, ) return num_blocks_to_allocate diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 30bf584f8c..2d39a38651 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -512,6 +512,10 @@ class KVCacheManager: ): watermark_blocks = self.watermark_blocks + # Matches the scheduler's own prefill boundary: `num_tokens - 1` + # extends it to resumed requests replaying their output tokens. + prefill_end = max(request.num_prompt_tokens, request.num_tokens - 1) + if full_sequence_must_fit: # First check and fail if the full request sequence won't fit. full_num_tokens = min(request.num_tokens, self.max_model_len) @@ -525,6 +529,7 @@ class KVCacheManager: num_local_computed_tokens=num_local_computed_tokens, num_tokens_main_model=full_num_tokens, apply_admission_cap=True, + prefill_end=prefill_end, ) required_blocks = num_blocks_to_allocate + watermark_blocks if required_blocks > self.block_pool.get_num_free_blocks(): @@ -559,6 +564,7 @@ class KVCacheManager: + num_external_computed_tokens, num_local_computed_tokens=num_local_computed_tokens, num_tokens_main_model=num_tokens_main_model, + prefill_end=prefill_end, ) # Keep `reserved_blocks` free for other in-flight sequences, and an diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index fc924b876c..494dba5e46 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -3003,6 +3003,7 @@ class Scheduler(SchedulerInterface): num_local_computed_tokens=request.num_computed_tokens, num_tokens_main_model=full_num_tokens, apply_admission_cap=True, + prefill_end=max(request.num_prompt_tokens, request.num_tokens - 1), ) return num_blocks + self._spec_decode_step_blocks() diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 3d784891c5..c8b9c54435 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -184,6 +184,7 @@ class SingleTypeKVCacheManager(ABC): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: """Get the number of blocks needed to be allocated for the request. @@ -203,6 +204,10 @@ class SingleTypeKVCacheManager(ABC): apply_admission_cap: If True, clamp by `num_required_blocks` by `_max_admission_blocks_per_request`for recycling-aware specs (SWA, chunked-local). + prefill_end: The token index the request's prefill ends at, the + same value the scheduler splits chunks against. Mamba reserves + a prefill checkpoint only on the chunk reaching it; 0 (dense + retention) reserves one on every chunk. Returns: The number of blocks to allocate. @@ -1215,6 +1220,7 @@ class CircularBufferManager(FullAttentionManager): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: return 0 if self.req_to_blocks.get(request_id) else 1 @@ -1678,12 +1684,14 @@ class MambaManager(SingleTypeKVCacheManager): query_start: int, query_end: int, checkpoint_position: int, + prefill_end: int, ) -> bool: assert isinstance(self.kv_cache_spec, MambaSpec) checkpoint_idx = cdiv(query_end, self.block_size) - 2 blocks = self.req_to_blocks[request_id] return ( self.has_prefill_checkpoint_blocks + and query_end >= prefill_end and is_mamba_prefill_checkpoint_valid( query_start=query_start, query_end=query_end, @@ -1712,6 +1720,7 @@ class MambaManager(SingleTypeKVCacheManager): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: assert isinstance(self.kv_cache_spec, MambaSpec) if ( @@ -1738,6 +1747,7 @@ class MambaManager(SingleTypeKVCacheManager): num_local_computed_tokens, num_tokens_main_model, apply_admission_cap=apply_admission_cap, + prefill_end=prefill_end, ) else: # We don't allocate blocks for lookahead tokens in align mode, because if @@ -1765,6 +1775,11 @@ class MambaManager(SingleTypeKVCacheManager): ) if has_partial_hit: num_new_blocks = max(num_new_blocks, 0) + 1 + # Keyed on the chunk end like the worker's + # `compute_mamba_prefill_checkpoints`, which writes the state there. + # The helper returns a boundary strictly below its input, so every + # chunk has one; under sparse retention only the prefill-end chunk's + # is kept, so `prefill_end` skips reserving blocks for the others. checkpoint_position = get_mamba_prefill_checkpoint_position( num_tokens, self.block_pool.hash_block_size, @@ -1775,6 +1790,7 @@ class MambaManager(SingleTypeKVCacheManager): total_computed_tokens, num_tokens, checkpoint_position, + prefill_end, ): checkpoint_position = 0 checkpoint_block = int(checkpoint_position > 0) @@ -2272,6 +2288,7 @@ class HiSparseSourceManager(FullAttentionManager): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: required = super().get_num_blocks_to_allocate( request_id, @@ -2407,6 +2424,7 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: assert not new_computed_blocks # A hot region is needed to read host-backed history: an external @@ -2479,6 +2497,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): num_local_computed_tokens: int, num_tokens_main_model: int, apply_admission_cap: bool = False, + prefill_end: int = 0, ) -> int: del num_tokens_main_model assert not new_computed_blocks From deb148e730225c64427f8ccf1f313cb4b33325b0 Mon Sep 17 00:00:00 2001 From: Andrii Skliar Date: Wed, 30 Sep 2026 23:41:17 +0200 Subject: [PATCH 153/248] [Mamba] Add FlashInfer ReplaySSM support for MTP (#52928) Signed-off-by: Andrii Skliar Co-authored-by: Andrii Skliar --- .buildkite/test_areas/engine.yaml | 21 + tests/kernels/mamba/test_ssu_dispatch.py | 398 +++++++++++++++++- tests/model_executor/test_replayssm_warmup.py | 23 +- tests/test_config.py | 6 +- .../test_attention_backends_selection.py | 28 ++ .../test_replayssm_metadata_builder.py | 94 ++++- tests/v1/e2e/test_replayssm_decode.py | 146 +++++-- tests/v1/worker/test_utils.py | 26 +- tools/pre_commit/test_tethering_allowlist.txt | 1 - vllm/config/cache.py | 11 +- vllm/config/vllm.py | 47 ++- vllm/model_executor/layers/mamba/abstract.py | 4 +- .../layers/mamba/mamba_mixer2.py | 109 ++++- .../layers/mamba/mamba_utils.py | 5 +- .../layers/mamba/ops/ssu_dispatch.py | 143 ++++++- vllm/model_executor/models/nemotron_h.py | 1 + .../model_executor/warmup/replayssm_warmup.py | 37 +- vllm/v1/attention/backends/mamba_attn.py | 42 +- 18 files changed, 1029 insertions(+), 113 deletions(-) diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index da02f47827..fa71c9f737 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -133,6 +133,27 @@ steps: - pytest -v -s v1/e2e/general/test_kv_sharing_fast_prefill.py - pytest -v -s v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2 v1/e2e/general/test_mamba_prefix_cache.py::test_mamba_prefix_cache_mrv2_async +- label: ":nvidia: (H100) ReplaySSM E2E" + device: h100 + num_devices: 2 + key: replayssm-e2e + timeout_in_minutes: 90 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/v1/e2e/test_replayssm_decode.py + - vllm/config/cache.py + - vllm/config/vllm.py + - vllm/model_executor/layers/mamba/ + - vllm/model_executor/models/nemotron_h.py + - vllm/model_executor/warmup/replayssm_warmup.py + - vllm/v1/attention/backends/mamba_attn.py + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_model_runner.py + - vllm/v1/worker/gpu_worker.py + commands: + - pytest -v -s v1/e2e/test_replayssm_decode.py + - label: ":nvidia: (L4) V1 E2E" key: v1-e2e-2-gpus timeout_in_minutes: 25 # TODO: Fix timeout after we have more confidence in the test stability diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index 6226340a99..082b61c43d 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -1,24 +1,30 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace from unittest.mock import Mock import pytest import torch from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm +from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2 from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( FlashInferSSUBackend, TritonSSUBackend, + commit_replayssm_ring_trackers, get_mamba_ssu_backend, initialize_mamba_ssu_backend, reset_replayssm_ring_trackers, selective_state_update, + selective_state_update_replayssm_flashinfer, update_replayssm_ring_trackers, ) from vllm.utils.torch_utils import set_random_seed +from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, @@ -32,6 +38,13 @@ try: except ImportError: HAS_FLASHINFER = False +try: + from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner + + HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None +except ImportError: + HAS_FLASHINFER_CHECKPOINTING_SSU = False + @pytest.fixture(autouse=True) def restore_backend_state(): @@ -47,6 +60,7 @@ def restore_backend_state(): def test_flashinfer_replayssm_ring_tracker_lifecycle(): ring_start = torch.zeros(2, dtype=torch.int32, device="cuda") prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda") + prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda") state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda") observed = [] @@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle(): update_replayssm_ring_trackers( ring_start, prev_num_accepted, + prev_query_len, state_batch_indices, logical_window=16, ring_buffer_len=17, @@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle(): reset_replayssm_ring_trackers( ring_start, prev_num_accepted, + prev_query_len, state_batch_indices, ) - assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0) + assert ( + ring_start[1].item(), + prev_num_accepted[1].item(), + prev_query_len[1].item(), + ) == (0, 0, 0) + + +@pytest.mark.parametrize( + ("accepted_sequence", "expected"), + [ + pytest.param( + [4] * 22, + [ + (0, 0, 4), + (0, 4, 4), + (0, 8, 4), + (0, 12, 4), + (0, 16, 4), + (16, 4, 4), + (16, 8, 4), + (16, 12, 4), + (16, 16, 4), + (12, 4, 4), + (12, 8, 4), + (12, 12, 4), + (12, 16, 4), + (8, 4, 4), + (8, 8, 4), + (8, 12, 4), + (8, 16, 4), + (4, 4, 4), + (4, 8, 4), + (4, 12, 4), + (4, 16, 4), + (0, 4, 4), + ], + id="all-accepted", + ), + pytest.param( + [4, 4, 0, 3, 4, 2, 4, 1], + [ + (0, 0, 4), + (0, 4, 4), + (0, 4, 4), + (0, 7, 4), + (0, 11, 4), + (0, 13, 4), + (13, 4, 4), + (13, 5, 4), + ], + id="mixed", + ), + ], +) +def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected): + logical_window = 16 + num_speculative_tokens = 3 + query_len = 1 + num_speculative_tokens + ring_buffer_len = logical_window + 1 + num_speculative_tokens + ring_start = torch.zeros(2, dtype=torch.int32, device="cuda") + prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda") + prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda") + state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda") + query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda") + + observed = [] + for accepted in accepted_sequence: + commit_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + torch.tensor([accepted], dtype=torch.int32, device="cuda"), + query_start_loc, + logical_window, + ring_buffer_len, + ) + snapshot = ( + ring_start[1].item(), + prev_num_accepted[1].item(), + prev_query_len[1].item(), + ) + observed.append(snapshot) + assert snapshot[1] + snapshot[2] <= ring_buffer_len + + assert observed == expected + + +def test_replayssm_resume_resets_commit_history(): + ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda") + prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda") + prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda") + state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda") + + reset_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + ) + assert ( + ring_start[1].item(), + prev_num_accepted[1].item(), + prev_query_len[1].item(), + ) == (0, 0, 0) + + commit_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + torch.tensor([3], dtype=torch.int32, device="cuda"), + torch.tensor([0, 4], dtype=torch.int32, device="cuda"), + logical_window=16, + ring_buffer_len=20, + ) + assert ( + ring_start[1].item(), + prev_num_accepted[1].item(), + prev_query_len[1].item(), + ) == (0, 0, 4) + + +def test_replayssm_commit_tracker_ragged_query_lengths(): + ring_start = torch.zeros(3, dtype=torch.int32, device="cuda") + prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda") + prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda") + state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda") + query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda") + + observed = [] + for accepted in ([4, 2], [3, 1]): + commit_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + torch.tensor(accepted, dtype=torch.int32, device="cuda"), + query_start_loc, + logical_window=16, + ring_buffer_len=20, + ) + observed.append( + [ + ( + ring_start[slot].item(), + prev_num_accepted[slot].item(), + prev_query_len[slot].item(), + ) + for slot in (1, 2) + ] + ) + + assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]] + + +@pytest.mark.parametrize("operation", ["commit", "reset"]) +def test_replayssm_tracker_kernels_mask_invalid_slots(operation): + num_states = 3 + ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda") + prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda") + prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda") + state_batch_indices = torch.tensor( + [-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda" + ) + + if operation == "commit": + commit_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"), + torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"), + logical_window=16, + ring_buffer_len=20, + ) + expected_valid = (2, 5, 4) + else: + reset_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + ) + expected_valid = (0, 0, 0) + + assert ( + ring_start.tolist(), + prev_num_accepted.tolist(), + prev_query_len.tolist(), + ) == ( + [11, expected_valid[0], 33], + [11, expected_valid[1], 33], + [11, expected_valid[2], 33], + ) def _kv_cache_config_with_ssu( @@ -231,14 +442,190 @@ def test_triton_basic_call(): assert not torch.isnan(out).any() +@pytest.mark.parametrize("layout", ["packed", "dense"]) +def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout): + import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod + + kernel = Mock(return_value=torch.empty(0)) + monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel) + + batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1 + state = torch.empty(2, nheads, dim, dstate) + x_shape: tuple[int, ...] + B_shape: tuple[int, ...] + expected_x_shape: tuple[int, ...] + expected_B_shape: tuple[int, ...] + if layout == "packed": + x_shape = (6, nheads, dim) + B_shape = (6, ngroups, dstate) + expected_x_shape = (1, 6, nheads, dim) + expected_B_shape = (1, 6, ngroups, dstate) + cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32) + kernel_max_seqlen = max_seqlen + else: + x_shape = (batch, max_seqlen, nheads, dim) + B_shape = (batch, max_seqlen, ngroups, dstate) + expected_x_shape = x_shape + expected_B_shape = B_shape + cu_seqlens = None + kernel_max_seqlen = None + x = torch.empty(x_shape) + dt = torch.empty_like(x) + A = torch.empty(nheads, dim, dstate) + B = torch.empty(B_shape) + C = torch.empty_like(B) + out = torch.empty_like(x) + x_cache = torch.empty(2, nheads, 20, dim) + dt_cache = torch.empty(2, nheads, 20) + B_cache = torch.empty(2, ngroups, 20, dstate) + ring_start = torch.zeros(2, dtype=torch.int32) + prev_num_accepted = torch.zeros(2, dtype=torch.int32) + prev_query_len = torch.zeros(2, dtype=torch.int32) + selective_state_update_replayssm_flashinfer( + state, + x, + dt, + A, + B, + C, + out, + x_cache, + B_cache, + dt_cache, + ring_start, + prev_num_accepted, + prev_query_len, + logical_window=16, + state_batch_indices=torch.tensor([0, 1], dtype=torch.int32), + cu_seqlens=cu_seqlens, + max_seqlen=kernel_max_seqlen, + update_trackers=False, + ) + + args = kernel.call_args.args + assert args[6].shape == expected_x_shape + assert args[7].shape == expected_x_shape + assert args[9].shape == expected_B_shape + assert args[10].shape == expected_B_shape + assert args[11].shape == expected_x_shape + assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens + assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen + + @pytest.mark.parametrize( - ("backend", "expected_ring_len"), + ("query_start_loc", "expected_shape", "expected_max_seqlen"), [ - (MambaBackendEnum.TRITON, 16), - (MambaBackendEnum.FLASHINFER, 17), + pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"), + pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"), ], ) -def test_replayssm_physical_ring_shape(backend, expected_ring_len): +def test_replayssm_mixer_selects_mtp_layout( + monkeypatch, query_start_loc, expected_shape, expected_max_seqlen +): + import vllm.model_executor.layers.mamba.mamba_mixer2 as mod + + mixer = MambaMixer2.__new__(MambaMixer2) + torch.nn.Module.__init__(mixer) + mixer.prefix = "mixer" + mixer.tped_intermediate_size = 0 + mixer.tped_conv_size = 1 + mixer.tped_dt_size = 2 + mixer.num_heads = 2 + mixer.head_dim = 4 + mixer.n_groups = mixer.tp_size = 1 + mixer.ssm_state_size = 8 + mixer.num_spec = 3 + mixer.use_replayssm = True + mixer.replayssm_buffer_len = 16 + mixer._commits_replayssm_trackers = True + mixer._updates_replayssm_trackers = False + mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER) + mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none") + mixer.conv_weights = torch.empty(0) + mixer.conv1d = SimpleNamespace(bias=None) + mixer.activation = "silu" + mixer.A = torch.empty(2) + mixer.dt_bias = torch.empty(2) + mixer.D = torch.empty(2) + mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32) + mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32) + mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32) + mixer.kv_cache = ( + torch.empty(3, 1), + torch.empty(3, 2, 4, 8), + torch.empty(3, 2, 20, 4), + torch.empty(3, 2, 20), + torch.empty(3, 1, 20, 8), + ) + + num_decode_tokens = query_start_loc[-1] + query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32) + metadata = Mamba2AttentionMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decodes=2, + num_decode_tokens=num_decode_tokens, + num_reqs=2, + has_initial_states_p=None, + query_start_loc_p=None, + num_computed_tokens_p=None, + state_indices_tensor_p=None, + state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32), + query_start_loc_d=query_start_loc_d, + num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32), + block_idx_last_scheduled_token=None, + block_idx_first_scheduled_token_p=None, + block_idx_last_computed_token=None, + block_idx_last_scheduled_token_prev_step=None, + seq_lens=torch.tensor([104, 102], dtype=torch.int32), + replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)), + replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32), + ) + + def split_hidden_states_B_C(values): + tokens = values.size(0) + return ( + torch.empty(tokens, 8), + torch.empty(tokens, 8), + torch.empty(tokens, 8), + ) + + mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C + kernel = Mock() + monkeypatch.setattr( + mod, + "get_forward_context", + lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}), + ) + monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock()) + monkeypatch.setattr( + mod, "causal_conv1d_update", lambda values, *args, **kwargs: values + ) + monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel) + + mixer.conv_ssm_forward( + torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8) + ) + + assert kernel.call_args.args[1].shape == expected_shape + if expected_max_seqlen is None: + assert kernel.call_args.kwargs["cu_seqlens"] is None + else: + assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d + assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen + + +@pytest.mark.parametrize( + ("backend", "num_speculative_tokens", "expected_ring_len"), + [ + (MambaBackendEnum.TRITON, 0, 16), + (MambaBackendEnum.FLASHINFER, 0, 17), + (MambaBackendEnum.FLASHINFER, 3, 20), + ], +) +def test_replayssm_physical_ring_shape( + backend, num_speculative_tokens, expected_ring_len +): base_shapes = ((64, 3), (8, 4, 16)) shapes = MambaStateShapeCalculator.append_replayssm_ring( @@ -247,6 +634,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len): tp_world_size=2, logical_window=16, backend=backend, + num_speculative_tokens=num_speculative_tokens, ) assert shapes[2:] == ( diff --git a/tests/model_executor/test_replayssm_warmup.py b/tests/model_executor/test_replayssm_warmup.py index 272fa2cf85..5db0f9a50d 100644 --- a/tests/model_executor/test_replayssm_warmup.py +++ b/tests/model_executor/test_replayssm_warmup.py @@ -67,8 +67,13 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs): with patch.object( warmup, "flashinfer_replayssm_autotune_supported", return_value=True ): - result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs)) + result = warmup._replayssm_autotune_kwargs( + _autotune_runner(**runner_kwargs), PREFILL_KWARGS + ) + assert result is not None + max_num_reqs, decode_kwargs = result + assert max_num_reqs == expected_num_reqs expected_kwargs = { **PREFILL_KWARGS, "num_tokens": expected_num_reqs * query_len, @@ -82,7 +87,7 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs): force_attention=True, profile_seq_lens=query_len + 1, ) - assert result == (expected_num_reqs, expected_kwargs) + assert decode_kwargs == expected_kwargs @pytest.mark.parametrize( @@ -91,8 +96,14 @@ def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs): (dict(use_replayssm=False), True), (dict(backend=MambaBackendEnum.TRITON), True), ({}, False), + (dict(num_blocks=1), True), + ], + ids=[ + "replayssm_disabled", + "non_flashinfer_backend", + "kernel_unavailable", + "zero_non_padding_slots", ], - ids=["replayssm_disabled", "non_flashinfer_backend", "kernel_unavailable"], ) def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported): with patch.object( @@ -100,7 +111,9 @@ def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported): "flashinfer_replayssm_autotune_supported", return_value=flashinfer_supported, ): - result = warmup._replayssm_autotune_kwargs(_autotune_runner(**runner_kwargs)) + result = warmup._replayssm_autotune_kwargs( + _autotune_runner(**runner_kwargs), PREFILL_KWARGS + ) assert result is None @@ -116,10 +129,12 @@ def test_replayssm_autotune_slots_restore_state_and_trackers(): ) mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32) mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32) + mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32) tracked = ( *mixer.kv_cache, mixer._replayssm_ring_start, mixer._replayssm_prev_num_accepted, + mixer._replayssm_prev_query_len, ) block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1) diff --git a/tests/test_config.py b/tests/test_config.py index 93578ab918..712d01fd33 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -153,6 +153,7 @@ def test_kda_recoverssm_derivation_is_revalidated(): use_replayssm=True, use_kda_recoverssm=False, mamba_cache_mode="none", + replayssm_buffer_len=16, ), num_speculative_tokens=3, model_config=SimpleNamespace( @@ -184,8 +185,9 @@ def test_kda_recoverssm_derivation_is_revalidated(): config.cache_config.mamba_cache_mode = "none" config.model_config.architecture = "NemotronHForCausalLM" - with pytest.raises(ValueError, match="only supported for Kimi-K3 KDA"): - VllmConfig.validate_mamba_cached_kernel(config) + config.mamba_config.backend = MambaBackendEnum.FLASHINFER + VllmConfig.validate_mamba_cached_kernel(config) + assert not config.cache_config.use_kda_recoverssm config.model_config.architecture = "KimiLinearForCausalLM" config.parallel_config.pipeline_parallel_size = 2 diff --git a/tests/v1/attention/test_attention_backends_selection.py b/tests/v1/attention/test_attention_backends_selection.py index 8486d216a1..2695517e14 100644 --- a/tests/v1/attention/test_attention_backends_selection.py +++ b/tests/v1/attention/test_attention_backends_selection.py @@ -5,7 +5,9 @@ from types import SimpleNamespace import pytest +import torch +from vllm.model_executor.layers.mamba.abstract import MambaBase from vllm.model_executor.layers.mamba.linear.minimax_linear_attn import ( MiniMaxText01LinearAttention, ) @@ -19,6 +21,32 @@ from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend +@pytest.mark.parametrize(("use_replayssm", "expected_blocks"), [(False, 3), (True, 0)]) +def test_replayssm_does_not_reserve_speculative_state_blocks( + use_replayssm, expected_blocks +): + layer = SimpleNamespace( + get_state_shape=lambda: ((2,),), + get_state_dtype=lambda: (torch.float32,), + mamba_type=MambaAttentionBackendEnum.MAMBA2, + is_kv_cache_tp_replicated=False, + ) + vllm_config = SimpleNamespace( + cache_config=SimpleNamespace( + mamba_block_size=1, + mamba_page_size_padded=None, + mamba_cache_mode="none", + use_replayssm=use_replayssm, + ), + num_speculative_tokens=3, + ) + + spec = MambaBase.get_kv_cache_spec(layer, vllm_config) + + assert spec is not None + assert spec.num_speculative_blocks == expected_blocks + + @pytest.mark.parametrize( "layer_class, init_kwargs, expected_backend, expected_mamba_type", [ diff --git a/tests/v1/attention/test_replayssm_metadata_builder.py b/tests/v1/attention/test_replayssm_metadata_builder.py index 89cb0f5664..58679ecac9 100644 --- a/tests/v1/attention/test_replayssm_metadata_builder.py +++ b/tests/v1/attention/test_replayssm_metadata_builder.py @@ -16,6 +16,8 @@ from tests.v1.attention.utils import ( create_common_attn_metadata, create_vllm_config, ) +from vllm.config import SpeculativeConfig +from vllm.config.compilation import CUDAGraphMode from vllm.config.mamba import MambaBackendEnum from vllm.v1.kv_cache_interface import MambaSpec @@ -191,9 +193,12 @@ REPLAYSSM_BUILD_CASES = { def _make_mamba_spec( buffer_len: int, mamba_backend: MambaBackendEnum, + num_speculative_tokens: int = 0, ) -> MambaSpec: ring_buffer_len = buffer_len + ( - 1 if mamba_backend == MambaBackendEnum.FLASHINFER else 0 + 1 + num_speculative_tokens + if mamba_backend == MambaBackendEnum.FLASHINFER + else 0 ) shapes = ( (1, 1), @@ -214,6 +219,7 @@ def _create_replayssm_builder( mamba_cache_mode: str = "none", *, mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON, + num_speculative_tokens: int = 0, ) -> MockMambaBuilder: vllm_config = create_vllm_config( model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE @@ -224,21 +230,32 @@ def _create_replayssm_builder( vllm_config.cache_config.replayssm_buffer_len = buffer_len vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode vllm_config.mamba_config.backend = mamba_backend + if num_speculative_tokens > 0: + # Triton ReplaySSM requires exact synchronous CPU metadata. + vllm_config.scheduler_config.async_scheduling = False + vllm_config.speculative_config = SpeculativeConfig( + method="ngram", + num_speculative_tokens=num_speculative_tokens, + ) return MockMambaBuilder( - _make_mamba_spec(buffer_len, mamba_backend), + _make_mamba_spec(buffer_len, mamba_backend, num_speculative_tokens), ["layer0"], vllm_config, DEVICE, ) -def _build(builder: MockMambaBuilder, case: ReplaySSMBuildCase): +def _build( + builder: MockMambaBuilder, + case: ReplaySSMBuildCase, + num_accepted_tokens: torch.Tensor | None = None, +): batch = BatchSpec(seq_lens=case.seq_lens, query_lens=case.query_lens) common = create_common_attn_metadata(batch, BLOCK_SIZE, DEVICE).replace( is_prefilling=torch.tensor(case.is_prefilling, dtype=torch.bool), replayssm_decode_base_cpu=torch.tensor(case.decode_base, dtype=torch.int32), ) - return builder.build(0, common) + return builder.build(0, common, num_accepted_tokens=num_accepted_tokens) @pytest.mark.parametrize( @@ -270,21 +287,60 @@ def test_resumed_request_differs_from_fresh(): assert meta.is_flush_d.tolist()[:2] == [0, 0] -def test_flashinfer_replayssm_scratch_metadata_fresh_decode(): - checkpointing_ssu = pytest.importorskip("flashinfer.mamba.checkpointing_ssu") - if not hasattr(checkpointing_ssu, "allocate_checkpointing_ssu_scratch"): - pytest.skip("FlashInfer does not expose ReplaySSM scratch allocation") +def test_spec_decode_single_token_chunk_synthesizes_acceptance_metadata(): + builder = _create_replayssm_builder(16, num_speculative_tokens=3) + case = REPLAYSSM_BUILD_CASES["leftover_prompt_one_token_flush"] - builder = _create_replayssm_builder(16, mamba_backend=MambaBackendEnum.FLASHINFER) - case = REPLAYSSM_BUILD_CASES["fresh_decode"] meta = _build(builder, case) - assert meta.write_pos_d is None - assert meta.is_flush_d is None - assert meta.bc_pre_scratch is None - assert meta.replayssm_scratch is not None - assert [tensor.shape for tensor in meta.replayssm_scratch] == [ - (1, 1, 32, 8), - (1, 1, 16), - (1, 1, 32, 8), - ] + assert meta.query_start_loc_d is not None + assert meta.query_start_loc_d.tolist() == [0, 1] + assert meta.num_accepted_tokens is not None + assert meta.num_accepted_tokens.tolist() == [1] + + +def test_flashinfer_replayssm_state_indices_are_stable_for_full_cudagraph(): + checkpointing_ssu = pytest.importorskip("flashinfer.mamba.checkpointing_ssu") + if not hasattr(checkpointing_ssu, "allocate_checkpointing_ssu_scratch"): + pytest.skip("requires FlashInfer ReplaySSM autotuning support") + + builder = _create_replayssm_builder( + 16, + mamba_backend=MambaBackendEnum.FLASHINFER, + num_speculative_tokens=3, + ) + builder.compilation_config.cudagraph_mode = CUDAGraphMode.FULL + + first = _build( + builder, + ReplaySSMBuildCase( + seq_lens=[106, 106], + query_lens=[1, 1], + is_prefilling=[False, False], + decode_base=[100, 100], + buffer_len=16, + expected_write_pos=[], + expected_is_flush=[], + ), + ) + first_indices = first.replayssm_state_indices_d + assert first_indices is not None + assert first_indices.is_contiguous() + first_ptr = first_indices.data_ptr() + + second = _build( + builder, + ReplaySSMBuildCase( + seq_lens=[122, 122], + query_lens=[1, 1], + is_prefilling=[False, False], + decode_base=[116, 116], + buffer_len=16, + expected_write_pos=[], + expected_is_flush=[], + ), + ) + second_indices = second.replayssm_state_indices_d + assert second_indices is not None + assert second_indices.data_ptr() == first_ptr + assert torch.equal(second_indices, second.state_indices_tensor_d[:, 0]) diff --git a/tests/v1/e2e/test_replayssm_decode.py b/tests/v1/e2e/test_replayssm_decode.py index 2e66a46689..1a369c781f 100644 --- a/tests/v1/e2e/test_replayssm_decode.py +++ b/tests/v1/e2e/test_replayssm_decode.py @@ -10,11 +10,16 @@ from vllm.v1.metrics.reader import Counter from ...models.utils import check_logprobs_close from ...utils import large_gpu_mark, multi_gpu_test -# Mamba2 (Nemotron-3) hybrid. +# Mamba2 (Nemotron-3) hybrids. MAMBA2_MODEL = "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16" +# FlashInfer 0.7.0's standard SSU cannot compile Nano's DIM=80 configuration. +MAMBA2_FLASHINFER_MODEL = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" MODELS = [ pytest.param(MAMBA2_MODEL, marks=large_gpu_mark(min_gb=40)), ] +FLASHINFER_MODELS = [ + pytest.param(MAMBA2_FLASHINFER_MODEL, marks=large_gpu_mark(min_gb=40)), +] PROMPTS = [ "The capital of France is", @@ -22,6 +27,16 @@ PROMPTS = [ ] +@pytest.fixture(autouse=True) +def _use_v1_model_runner_by_default(monkeypatch): + # Triton ReplaySSM is V1-only. FlashInfer V2 tests override this locally. + with monkeypatch.context() as patch: + patch.setenv("VLLM_USE_V2_MODEL_RUNNER", "0") + envs.disable_envs_cache() + yield + envs.disable_envs_cache() + + def _check_replayssm_parity( vllm_runner, model_name, @@ -29,17 +44,11 @@ def _check_replayssm_parity( tensor_parallel_size=1, mamba_backend: str = "triton", name_1: str = "replayssm", - require_v2: bool = False, - monkeypatch: pytest.MonkeyPatch | None = None, + expected_v2: bool | None = None, ): # Compare logprobs, not greedy ids: ReplaySSM's fp arithmetic can flip a - # near-tie. Baseline and ReplaySSM run at the same TP, so TP numerics are - # common-mode and only ReplaySSM varies. - if require_v2: - assert monkeypatch is not None - monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") - envs.disable_envs_cache() - + # near-tie. Baseline and ReplaySSM use the same backend so only ReplaySSM + # behavior varies. common = dict( max_model_len=1024, trust_remote_code=True, @@ -49,14 +58,17 @@ def _check_replayssm_parity( mamba_backend=mamba_backend, ) with vllm_runner(model_name, **common) as llm: - if require_v2: - assert llm.llm.llm_engine.vllm_config.use_v2_model_runner + if expected_v2 is not None: + assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2 baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5) with vllm_runner( - model_name, use_replayssm=True, replayssm_buffer_len=16, **common + model_name, + use_replayssm=True, + replayssm_buffer_len=16, + **common, ) as llm: - if require_v2: - assert llm.llm.llm_engine.vllm_config.use_v2_model_runner + if expected_v2 is not None: + assert llm.llm.llm_engine.vllm_config.use_v2_model_runner is expected_v2 replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5) check_logprobs_close( @@ -80,19 +92,101 @@ def test_replayssm_decode_matches_baseline_tp2(vllm_runner, model_name): _check_replayssm_parity(vllm_runner, model_name, tensor_parallel_size=2) -@pytest.mark.parametrize("model_name", MODELS) -def test_replayssm_flashinfer_decode_matches_baseline_v2( - vllm_runner, model_name, monkeypatch +@pytest.mark.parametrize("model_name", FLASHINFER_MODELS) +@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"]) +def test_replayssm_flashinfer_decode_matches_baseline( + vllm_runner, model_name, monkeypatch, use_v2_model_runner ): - pytest.importorskip("flashinfer.mamba.checkpointing_ssu") - _check_replayssm_parity( - vllm_runner, - model_name, - mamba_backend="flashinfer", - name_1="replayssm_flashinfer_v2", - require_v2=True, - monkeypatch=monkeypatch, + try: + with monkeypatch.context() as patch: + patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner))) + envs.disable_envs_cache() + _check_replayssm_parity( + vllm_runner, + model_name, + mamba_backend="flashinfer", + name_1="replayssm_flashinfer", + expected_v2=use_v2_model_runner, + ) + finally: + # The context restores the environment before the final cache reset. + envs.disable_envs_cache() + + +@pytest.mark.parametrize("model_name", FLASHINFER_MODELS) +def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name): + common = dict( + max_model_len=1024, + trust_remote_code=True, + enable_prefix_caching=False, + mamba_cache_mode="none", + speculative_config={ + "method": "ngram", + "num_speculative_tokens": 3, + "prompt_lookup_max": 3, + }, ) + with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm: + baseline = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5) + with vllm_runner( + model_name, + use_replayssm=True, + replayssm_buffer_len=16, + mamba_backend="flashinfer", + **common, + ) as llm: + replay = llm.generate_greedy_logprobs(PROMPTS, max_tokens=32, num_logprobs=5) + + check_logprobs_close( + outputs_0_lst=baseline, + outputs_1_lst=replay, + name_0="baseline_spec", + name_1="replayssm_flashinfer_spec", + ) + + +@multi_gpu_test(num_gpus=2) +@large_gpu_mark(min_gb=40) +@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"]) +def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner): + common = dict( + max_model_len=1024, + trust_remote_code=True, + enable_prefix_caching=False, + mamba_cache_mode="none", + mamba_backend="flashinfer", + tensor_parallel_size=2, + disable_log_stats=False, + speculative_config={"method": "mtp", "num_speculative_tokens": 3}, + ) + try: + with monkeypatch.context() as patch: + patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner))) + envs.disable_envs_cache() + with vllm_runner( + MAMBA2_FLASHINFER_MODEL, + use_replayssm=True, + replayssm_buffer_len=16, + **common, + ) as llm: + assert ( + llm.llm.llm_engine.vllm_config.use_v2_model_runner + is use_v2_model_runner + ) + outputs = llm.generate_greedy(PROMPTS, max_tokens=32) + draft_count = sum( + metric.value + for metric in llm.llm.get_metrics() + if isinstance(metric, Counter) + and metric.name == "vllm:spec_decode_num_drafts" + ) + finally: + envs.disable_envs_cache() + + # At least one request must run past the 16-token replay window; another + # may legitimately stop early on EOS. + assert any(len(token_ids) > 16 for token_ids, _ in outputs) + assert draft_count > 0 # Prefix spans several mamba blocks; prefix caching only reuses full blocks. diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 1833bca8ab..43ac710f35 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -1260,6 +1260,8 @@ class _TestReplaySSMMixer(MambaMixer2): self.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER) self._replayssm_ring_start = torch.empty(0, dtype=torch.int32) self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32) + self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32) + self._commits_replayssm_trackers = True self._updates_replayssm_trackers = True def get_state_shape(self) -> tuple[tuple[int, ...], ...]: @@ -1294,19 +1296,19 @@ def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group(layers_only): else: bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups) - assert ( - mixers[0]._replayssm_ring_start.data_ptr() - == mixers[2]._replayssm_ring_start.data_ptr() + tracker_names = ( + "_replayssm_ring_start", + "_replayssm_prev_num_accepted", + "_replayssm_prev_query_len", ) - assert ( - mixers[0]._replayssm_prev_num_accepted.data_ptr() - == mixers[2]._replayssm_prev_num_accepted.data_ptr() - ) - assert ( - mixers[1]._replayssm_ring_start.data_ptr() - != mixers[0]._replayssm_ring_start.data_ptr() - ) - # Group {0, 2} shares trackers; layer 2 (not 0) updates after both run. + for tracker_name in tracker_names: + group_tracker = getattr(mixers[0], tracker_name) + assert group_tracker.data_ptr() == getattr(mixers[2], tracker_name).data_ptr() + assert group_tracker.data_ptr() != getattr(mixers[1], tracker_name).data_ptr() + assert group_tracker.shape == (4,) + assert torch.count_nonzero(group_tracker) == 0 + + assert [m._commits_replayssm_trackers for m in mixers] == [True, True, False] assert [m._updates_replayssm_trackers for m in mixers] == [False, True, True] diff --git a/tools/pre_commit/test_tethering_allowlist.txt b/tools/pre_commit/test_tethering_allowlist.txt index cc5294cf26..d7f19ec79a 100644 --- a/tools/pre_commit/test_tethering_allowlist.txt +++ b/tools/pre_commit/test_tethering_allowlist.txt @@ -60,7 +60,6 @@ tests/models/test_language_model_cache_is_weak.py # no models_*.yaml job collec tests/models/transformers/test_layer_registry.py # no models_*.yaml job collects this path # tests/v1/ - the v1 subtree is otherwise covered by many `pytest v1/...` jobs. -tests/v1/e2e/test_replayssm_decode.py # ReplaySSM decode test (#52506) wired into no job tests/v1/ec_connector/integration/test_epd_correctness.py # run_epd_correctness_test.sh exists but no Buildkite job invokes it tests/v1/ec_connector/integration/test_nixl_failure.py # 2-GPU + NIXL E/PD integration test (#55290); no Buildkite job collects ec_connector/integration/ tests/v1/kv_connector/nixl_integration/test_disagg_accuracy.py # run by intel_jobs/misc_intel.yaml's run_xpu_disagg_accuracy_test.sh, which names it via $EXP_ROOT (the script's own dir) - not statically resolvable diff --git a/vllm/config/cache.py b/vllm/config/cache.py index eeff06933e..13665f3cff 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -204,15 +204,16 @@ class CacheConfig: Mamba block size.""" replayssm_buffer_len: int = Field(default=16, gt=0) """ReplaySSM logical history length B for Mamba2. Triton uses B physical - rows and FlashInfer uses B+1. Kimi-K3 speculative decode does not use B. - Default 16.""" + rows and FlashInfer uses B+T, where T is the target verification length. + Kimi-K3 speculative decode does not use B. Default 16.""" use_replayssm: bool = False """Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip the per-step full-state store, writing the checkpoint back only on flush. Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton - or FlashInfer mamba backend; standard (non-speculative) decode only. In align - mode flushes are most efficient when mamba_block_size is a multiple of - replayssm_buffer_len, but this is not required.""" + or FlashInfer mamba backend. Mamba2 speculative decode requires FlashInfer + and mamba_cache_mode 'none'. In align mode flushes are most efficient when + mamba_block_size is a multiple of replayssm_buffer_len, but this is not + required.""" use_kda_recoverssm: bool = field(default=False, init=False) """Whether Kimi-K3 KDA uses RecoverSSM speculative decode.""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 2da17707e5..0fd536ff87 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -3458,19 +3458,35 @@ class VllmConfig: if not self.cache_config.use_replayssm: self.cache_config.use_kda_recoverssm = False return self - self.cache_config.use_kda_recoverssm = self.num_speculative_tokens > 0 + + kda_architectures = ( + "KimiLinearForCausalLM", + "KimiK3ForConditionalGeneration", + ) + is_kda_model = ( + self.model_config is not None + and self.model_config.architecture in kda_architectures + ) + self.cache_config.use_kda_recoverssm = ( + self.num_speculative_tokens > 0 and is_kda_model + ) + use_mamba_replayssm_spec = ( + self.num_speculative_tokens > 0 and not self.cache_config.use_kda_recoverssm + ) if self.model_config is not None and not self.model_config.supports_replayssm: raise ValueError( "--use-replayssm is not supported for architecture " f"{self.model_config.architecture!r}" ) + if ( + self.mamba_config.backend == MambaBackendEnum.FLASHINFER + and self.cache_config.replayssm_buffer_len > 16 + ): + raise ValueError( + "FlashInfer ReplaySSM requires --replayssm-buffer-len <= 16" + ) if self.cache_config.use_kda_recoverssm: - if self.model_config is not None and self.model_config.architecture not in ( - "KimiLinearForCausalLM", - "KimiK3ForConditionalGeneration", - ): - raise ValueError("RecoverSSM is only supported for Kimi-K3 KDA") if self.mamba_config.enable_stochastic_rounding: raise ValueError( "RecoverSSM supports bfloat16/float32 " @@ -3494,6 +3510,25 @@ class VllmConfig: ) if self.mamba_config.backend != MambaBackendEnum.TRITON: raise ValueError("RecoverSSM requires --mamba-backend triton") + elif use_mamba_replayssm_spec: + if self.cache_config.mamba_cache_mode != "none": + raise ValueError( + "FlashInfer ReplaySSM speculative decoding requires " + "--mamba-cache-mode none" + ) + query_len = 1 + self.num_speculative_tokens + if self.cache_config.replayssm_buffer_len < query_len: + raise ValueError( + "FlashInfer ReplaySSM speculative decoding requires " + "--replayssm-buffer-len >= 1 + num_speculative_tokens " + f"({query_len}); got " + f"{self.cache_config.replayssm_buffer_len}" + ) + if self.mamba_config.backend != MambaBackendEnum.FLASHINFER: + raise ValueError( + "Mamba2 ReplaySSM speculative decoding requires " + "--mamba-backend flashinfer" + ) elif self.cache_config.mamba_cache_mode == "all": raise ValueError( "--use-replayssm supports prefix caching only in align mode; " diff --git a/vllm/model_executor/layers/mamba/abstract.py b/vllm/model_executor/layers/mamba/abstract.py index fa0eb25758..5322f899dd 100644 --- a/vllm/model_executor/layers/mamba/abstract.py +++ b/vllm/model_executor/layers/mamba/abstract.py @@ -74,11 +74,11 @@ class MambaBase(AttentionLayerBase): mamba_type=self.mamba_type, tp_replicated=self.is_kv_cache_tp_replicated, mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode, - # RecoverSSM verifies the whole window off one checkpoint, so it + # ReplaySSM verifies the whole window off one checkpoint, so it # never writes the baseline's per-draft-token state slots. num_speculative_blocks=( 0 - if vllm_config.cache_config.use_kda_recoverssm + if vllm_config.cache_config.use_replayssm else vllm_config.num_speculative_tokens ), ) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index 812cab0212..fc58aed91c 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -43,6 +43,7 @@ from vllm.model_executor.layers.mamba.ops.ssd_combined import ( mamba_chunk_scan_combined_varlen, ) from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( + commit_replayssm_ring_trackers, reset_replayssm_ring_trackers, selective_state_update, selective_state_update_replayssm_flashinfer, @@ -69,6 +70,13 @@ from vllm.v1.kv_cache_interface import KVCacheGroupSpec logger = init_logger(__name__) + +def _view_mtp_decode_tensor( + tensor: torch.Tensor, decode_batch: int, spec_query_len: int +) -> torch.Tensor: + return tensor.view(decode_batch, spec_query_len, *tensor.shape[1:]) + + # Added by the IBM Team, 2024 @@ -536,6 +544,8 @@ class MambaMixer2(MambaBase, PluggableLayer): self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state)) self._replayssm_ring_start = torch.empty(0, dtype=torch.int32) self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32) + self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32) + self._commits_replayssm_trackers = True self._updates_replayssm_trackers = True self.num_spec = vllm_config.num_speculative_tokens @@ -724,7 +734,8 @@ class MambaMixer2(MambaBase, PluggableLayer): assert self.cache_config is not None mamba_block_size = self.cache_config.mamba_block_size is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" - ring_start = prev_num_accepted = None + use_spec_decode = self.num_spec > 0 + ring_start = prev_num_accepted = prev_query_len = None attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: @@ -746,6 +757,7 @@ class MambaMixer2(MambaBase, PluggableLayer): if self.mamba_config.backend == MambaBackendEnum.FLASHINFER: ring_start = self._replayssm_ring_start prev_num_accepted = self._replayssm_prev_num_accepted + prev_query_len = self._replayssm_prev_query_len else: x_cache = dt_cache = B_cache = None has_initial_states_p = attn_metadata.has_initial_states_p @@ -757,6 +769,7 @@ class MambaMixer2(MambaBase, PluggableLayer): last_chunk_indices_p = attn_metadata.last_chunk_indices_p state_indices_tensor_p = attn_metadata.state_indices_tensor_p state_indices_tensor_d = attn_metadata.state_indices_tensor_d + replayssm_state_indices_d = attn_metadata.replayssm_state_indices_d num_accepted_tokens = attn_metadata.num_accepted_tokens query_start_loc_d = attn_metadata.query_start_loc_d num_decodes = attn_metadata.num_decodes @@ -996,6 +1009,7 @@ class MambaMixer2(MambaBase, PluggableLayer): reset_replayssm_ring_trackers( ring_start, prev_num_accepted, + prev_query_len, state_indices_tensor_p, ) @@ -1003,7 +1017,7 @@ class MambaMixer2(MambaBase, PluggableLayer): if has_decode: assert state_indices_tensor_d is not None if is_mamba_cache_all: - if self.num_spec > 0: + if use_spec_decode: assert block_idx_last_scheduled_token_prev_step_d is not None input_indices = ( block_idx_last_scheduled_token_prev_step_d.unsqueeze(1) @@ -1031,6 +1045,34 @@ class MambaMixer2(MambaBase, PluggableLayer): state_indices_tensor_d_input = state_indices_tensor_d state_indices_tensor_d_output = state_indices_tensor_d + if ( + self.use_replayssm + and self.mamba_config.backend == MambaBackendEnum.FLASHINFER + and use_spec_decode + and self._commits_replayssm_trackers + ): + assert ring_start is not None + assert prev_num_accepted is not None + assert prev_query_len is not None + assert replayssm_state_indices_d is not None + assert num_accepted_tokens is not None + assert query_start_loc_d is not None + assert x_cache is not None + assert self.replayssm_buffer_len is not None + # The previous MTP query's accepted prefix is known only now. + # Commit it once, before this cache group's layers evaluate the + # current speculative query with the shared ring positions. + commit_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + replayssm_state_indices_d, + num_accepted_tokens, + query_start_loc_d, + logical_window=self.replayssm_buffer_len, + ring_buffer_len=x_cache.size(2), + ) + # 2. Convolution sequence transformation hidden_states_B_C_d = causal_conv1d_update( hidden_states_B_C_d, @@ -1043,7 +1085,13 @@ class MambaMixer2(MambaBase, PluggableLayer): initial_state_idx=block_idx_last_computed_token_d, num_accepted_tokens=num_accepted_tokens, query_start_loc=query_start_loc_d, - max_query_len=state_indices_tensor_d.size(-1), + # ReplaySSM keeps one physical state block while a speculative + # decode call still processes the full target + draft window. + max_query_len=( + 1 + self.num_spec + if self.use_replayssm and use_spec_decode + else state_indices_tensor_d.size(-1) + ), ) hidden_states_d, B_d, C_d = self.split_hidden_states_B_C_fn( @@ -1079,7 +1127,35 @@ class MambaMixer2(MambaBase, PluggableLayer): if self.mamba_config.backend == MambaBackendEnum.FLASHINFER: assert ring_start is not None assert prev_num_accepted is not None + assert prev_query_len is not None assert attn_metadata.replayssm_scratch is not None + fi_cu_seqlens = query_start_loc_d + fi_max_seqlen = None + if use_spec_decode: + spec_query_len = 1 + self.num_spec + fi_max_seqlen = spec_query_len + assert replayssm_state_indices_d is not None + decode_batch = replayssm_state_indices_d.size(0) + if num_decode_tokens == decode_batch * spec_query_len: + hidden_states_d = _view_mtp_decode_tensor( + hidden_states_d, decode_batch, spec_query_len + ) + dt_d = _view_mtp_decode_tensor( + dt_d, decode_batch, spec_query_len + ) + B_d = _view_mtp_decode_tensor( + B_d, decode_batch, spec_query_len + ) + C_d = _view_mtp_decode_tensor( + C_d, decode_batch, spec_query_len + ) + preallocated_ssm_out_d = _view_mtp_decode_tensor( + preallocated_ssm_out_d, + decode_batch, + spec_query_len, + ) + fi_cu_seqlens = None + fi_max_seqlen = None selective_state_update_replayssm_flashinfer( ssm_state, hidden_states_d, @@ -1093,19 +1169,25 @@ class MambaMixer2(MambaBase, PluggableLayer): dt_cache, ring_start, prev_num_accepted, + prev_query_len, logical_window=self.replayssm_buffer_len, D=D_d, dt_bias=dt_bias, dt_softplus=True, - state_batch_indices=state_indices_tensor_d_input, + state_batch_indices=replayssm_state_indices_d, scratch=attn_metadata.replayssm_scratch, - update_trackers=self._updates_replayssm_trackers, + update_trackers=( + self._updates_replayssm_trackers and not use_spec_decode + ), enable_stochastic_rounding=( self.mamba_config.enable_stochastic_rounding ), stochastic_rounding_philox_rounds=( self.mamba_config.stochastic_rounding_philox_rounds ), + cu_seqlens=fi_cu_seqlens, + max_seqlen=fi_max_seqlen, + enable_pdl=False, ) else: selective_state_update_replayssm_output_only( @@ -1187,6 +1269,7 @@ class MambaMixer2(MambaBase, PluggableLayer): tp_world_size=tp_world_size, logical_window=self.replayssm_buffer_len, backend=self.mamba_config.backend, + num_speculative_tokens=self.num_spec, ) return base_shape @@ -1204,9 +1287,10 @@ def share_replayssm_ring_trackers( Layers backed by one KV-cache group use the same physical block indices and can therefore share cursors. Different KV-cache groups may assign different - block indices to the same request and must keep separate cursor tensors. - The final local layer in each group advances its cursors after every layer - in that group has consumed the previous values. + block indices to the same request and must keep separate cursor tensors. For + speculative decode, the first local layer commits the preceding acceptance; + for standard decode, the final local layer advances after all layers consume + the previous values. """ replayssm_mixers: dict[str, MambaMixer2] = {} for layer_name in ordered_layer_names: @@ -1232,6 +1316,7 @@ def share_replayssm_ring_trackers( groups_by_namespace.setdefault(namespace, []).append(layer_name) for group_layer_names in groups_by_namespace.values(): + first_layer_name = group_layer_names[0] last_layer_name = group_layer_names[-1] first_mixer = replayssm_mixers[group_layer_names[0]] @@ -1246,11 +1331,17 @@ def share_replayssm_ring_trackers( ring_start = torch.zeros(num_blocks, dtype=torch.int32, device=device) prev_num_accepted = torch.zeros_like(ring_start) + prev_query_len = torch.zeros_like(ring_start) for layer_name in group_layer_names: mixer = replayssm_mixers[layer_name] mixer._replayssm_ring_start = ring_start mixer._replayssm_prev_num_accepted = prev_num_accepted - mixer._updates_replayssm_trackers = layer_name == last_layer_name + mixer._replayssm_prev_query_len = prev_query_len + mixer._commits_replayssm_trackers = False + mixer._updates_replayssm_trackers = False + + replayssm_mixers[first_layer_name]._commits_replayssm_trackers = True + replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True def mamba_mixer2( diff --git a/vllm/model_executor/layers/mamba/mamba_utils.py b/vllm/model_executor/layers/mamba/mamba_utils.py index 8c89488d2d..31e89ffe55 100644 --- a/vllm/model_executor/layers/mamba/mamba_utils.py +++ b/vllm/model_executor/layers/mamba/mamba_utils.py @@ -227,6 +227,7 @@ class MambaStateShapeCalculator: tp_world_size: int, logical_window: int, backend: MambaBackendEnum, + num_speculative_tokens: int = 0, ) -> tuple[ tuple[int, int], tuple[int, int, int], @@ -241,8 +242,8 @@ class MambaStateShapeCalculator: """ ring_buffer_len = logical_window if backend == MambaBackendEnum.FLASHINFER: - # FlashInfer keeps the live window and appended token together. - ring_buffer_len += 1 + # FlashInfer keeps the live window and current verify window together. + ring_buffer_len += 1 + num_speculative_tokens local_nheads, head_dim, state_size = base_shapes[1] local_ngroups = divide(n_groups, tp_world_size) return ( diff --git a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py index b622924327..b6e9108a94 100644 --- a/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py +++ b/vllm/model_executor/layers/mamba/ops/ssu_dispatch.py @@ -31,6 +31,7 @@ logger = init_logger(__name__) def _update_replayssm_ring_trackers_kernel( ring_start, prev_num_accepted, + prev_query_len, state_batch_indices, state_batch_indices_stride, n_slots, @@ -52,6 +53,9 @@ def _update_replayssm_ring_trackers_kernel( if RESET: tl.store(ring_start + slots, 0, mask=valid) tl.store(prev_num_accepted + slots, 0, mask=valid) + # MTP alone reads prev_query_len. Standard decode deliberately leaves + # it unchanged; reset clears the pending MTP window after prefill. + tl.store(prev_query_len + slots, 0, mask=valid) else: prev = tl.load(prev_num_accepted + slots, mask=valid, other=0) start = tl.load(ring_start + slots, mask=valid, other=0) @@ -66,9 +70,70 @@ def _update_replayssm_ring_trackers_kernel( tl.store(prev_num_accepted + slots, next_prev, mask=valid) +@triton.jit( + do_not_specialize=["n_slots", "state_batch_indices_stride"], + do_not_specialize_on_alignment=["state_batch_indices"], +) +def _commit_replayssm_ring_trackers_kernel( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + num_accepted_tokens, + query_start_loc, + state_batch_indices_stride, + n_slots, + num_states, + logical_window: tl.constexpr, + ring_buffer_len: tl.constexpr, + pad_slot_id: tl.constexpr, + BLOCK: tl.constexpr, +) -> None: + offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = offsets < n_slots + slots = tl.load( + state_batch_indices + offsets * state_batch_indices_stride, + mask=mask, + other=pad_slot_id, + ) + valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states) + accepted_since_checkpoint = tl.load(prev_num_accepted + slots, mask=valid, other=0) + current_ring_start = tl.load(ring_start + slots, mask=valid, other=0) + previous_speculative_query_len = tl.load( + prev_query_len + slots, mask=valid, other=0 + ) + accepted_from_previous_query = tl.load( + num_accepted_tokens + offsets, mask=mask, other=0 + ) + must_checkpoint = (previous_speculative_query_len > 0) & ( + accepted_since_checkpoint + previous_speculative_query_len > logical_window + ) + next_start = tl.where( + must_checkpoint, + (current_ring_start + accepted_since_checkpoint) % ring_buffer_len, + current_ring_start, + ) + next_prev = tl.where( + previous_speculative_query_len == 0, + 0, + tl.where( + must_checkpoint, + accepted_from_previous_query, + accepted_since_checkpoint + accepted_from_previous_query, + ), + ) + current_speculative_query_len = tl.load( + query_start_loc + offsets + 1, mask=mask, other=0 + ) - tl.load(query_start_loc + offsets, mask=mask, other=0) + tl.store(ring_start + slots, next_start, mask=valid) + tl.store(prev_num_accepted + slots, next_prev, mask=valid) + tl.store(prev_query_len + slots, current_speculative_query_len, mask=valid) + + def update_replayssm_ring_trackers( ring_start: torch.Tensor, prev_num_accepted: torch.Tensor, + prev_query_len: torch.Tensor, state_batch_indices: torch.Tensor, logical_window: int | None = None, ring_buffer_len: int | None = None, @@ -90,10 +155,15 @@ def update_replayssm_ring_trackers( _update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)]( ring_start, prev_num_accepted, + prev_query_len, state_batch_indices, state_batch_indices.stride(0), n_slots, - min(ring_start.numel(), prev_num_accepted.numel()), + min( + ring_start.numel(), + prev_num_accepted.numel(), + prev_query_len.numel(), + ), logical_window, ring_buffer_len, pad_slot_id, @@ -105,6 +175,7 @@ def update_replayssm_ring_trackers( def reset_replayssm_ring_trackers( ring_start: torch.Tensor, prev_num_accepted: torch.Tensor, + prev_query_len: torch.Tensor, state_batch_indices: torch.Tensor, pad_slot_id: int = NULL_BLOCK_ID, ) -> None: @@ -112,11 +183,62 @@ def reset_replayssm_ring_trackers( update_replayssm_ring_trackers( ring_start, prev_num_accepted, + prev_query_len, state_batch_indices, pad_slot_id=pad_slot_id, ) +def commit_replayssm_ring_trackers( + ring_start: torch.Tensor, + prev_num_accepted: torch.Tensor, + prev_query_len: torch.Tensor, + state_batch_indices: torch.Tensor, + num_accepted_tokens: torch.Tensor, + query_start_loc: torch.Tensor, + logical_window: int, + ring_buffer_len: int, + pad_slot_id: int = NULL_BLOCK_ID, +) -> None: + """Commit the preceding speculative window and record the current one. + + MTP evaluates a target token and its draft tokens together, but the number + accepted from that query is available only on the next forward pass. This + function then advances the per-request ring by the accepted prefix and + records the current query length for the following pass. A zero + ``prev_query_len`` means that reset/prefill left no prior MTP query to + commit; slot validity is handled independently by the kernel mask. + + Standard single-token decode needs no delayed commit: its ReplaySSM kernel + advances the shared ring trackers directly after every token. + """ + if state_batch_indices.dim() > 1: + state_batch_indices = state_batch_indices[:, 0] + n_slots = state_batch_indices.numel() + if n_slots == 0: + return + block = 128 + _commit_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)]( + ring_start, + prev_num_accepted, + prev_query_len, + state_batch_indices, + num_accepted_tokens, + query_start_loc, + state_batch_indices.stride(0), + n_slots, + min( + ring_start.numel(), + prev_num_accepted.numel(), + prev_query_len.numel(), + ), + logical_window, + ring_buffer_len, + pad_slot_id, + BLOCK=block, + ) + + class MambaSSUBackend(ABC): """Abstract base class for Mamba SSU backends.""" @@ -385,6 +507,7 @@ def selective_state_update_replayssm_flashinfer( dt_cache: torch.Tensor, ring_start: torch.Tensor, prev_num_accepted_tokens: torch.Tensor, + prev_query_len: torch.Tensor, logical_window: int, D: torch.Tensor | None = None, dt_bias: torch.Tensor | None = None, @@ -395,6 +518,9 @@ def selective_state_update_replayssm_flashinfer( update_trackers: bool = True, enable_stochastic_rounding: bool = False, stochastic_rounding_philox_rounds: int = 0, + cu_seqlens: torch.Tensor | None = None, + max_seqlen: int | None = None, + enable_pdl: bool = False, ) -> torch.Tensor: """Run FlashInfer checkpointing SSU and optionally advance shared trackers.""" if _flashinfer_replayssm_kernel is None: @@ -404,11 +530,12 @@ def selective_state_update_replayssm_flashinfer( ) if x.dim() == 3: - x = x.unsqueeze(1) - dt = dt.unsqueeze(1) - B = B.unsqueeze(1) - C = C.unsqueeze(1) - out = out.unsqueeze(1) + dim = 0 if cu_seqlens is not None else 1 + x = x.unsqueeze(dim) + dt = dt.unsqueeze(dim) + B = B.unsqueeze(dim) + C = C.unsqueeze(dim) + out = out.unsqueeze(dim) indices = state_batch_indices if indices is not None and indices.dim() > 1: @@ -443,6 +570,9 @@ def selective_state_update_replayssm_flashinfer( pad_slot_id=null_block_id, rand_seed=rand_seed, philox_rounds=stochastic_rounding_philox_rounds or 10, + cu_seqlens=cu_seqlens, + max_seqlen=max_seqlen, + enable_pdl=enable_pdl, cb_scaled=cb_scaled, cumAdt_vec=cumAdt_vec, cb_old=cb_old, @@ -451,6 +581,7 @@ def selective_state_update_replayssm_flashinfer( update_replayssm_ring_trackers( ring_start, prev_num_accepted_tokens, + prev_query_len, indices, logical_window=logical_window, ring_buffer_len=x_cache.size(2), diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index 3f97deebe4..bef3da5cc5 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -815,6 +815,7 @@ class NemotronHForCausalLM( tp_world_size=parallel_config.tensor_parallel_size, logical_window=cache_config.replayssm_buffer_len, backend=vllm_config.mamba_config.backend, + num_speculative_tokens=vllm_config.num_speculative_tokens, ) return base_shape diff --git a/vllm/model_executor/warmup/replayssm_warmup.py b/vllm/model_executor/warmup/replayssm_warmup.py index 60b3eced20..28c45804ca 100644 --- a/vllm/model_executor/warmup/replayssm_warmup.py +++ b/vllm/model_executor/warmup/replayssm_warmup.py @@ -21,6 +21,7 @@ logger = init_logger(__name__) def _replayssm_autotune_kwargs( runner: "GPUModelRunner", + max_token_prefill_kwargs: dict[str, Any], ) -> tuple[int, dict[str, Any]] | None: config = runner.vllm_config if not ( @@ -46,11 +47,16 @@ def _replayssm_autotune_kwargs( runner.max_num_tokens // query_len, runner.kv_cache_config.num_blocks - 1, ) + if max_num_reqs <= 0: + logger.warning_once( + "Skipping FlashInfer ReplaySSM autotuning because no non-padding " + "state slot is available." + ) + return None + decode_kwargs = { + **max_token_prefill_kwargs, "num_tokens": max_num_reqs * query_len, - "skip_eplb": True, - "is_profile": True, - "randomize_inputs": True, "uniform_decode": True, } if config.use_v2_model_runner: @@ -75,18 +81,22 @@ def _temporary_replayssm_autotune_state( ) reset_tensors: dict[int, torch.Tensor] = {} - tracker_specs: dict[int, tuple[torch.Tensor, torch.Tensor, int, int]] = {} + tracker_specs: dict[ + int, tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int] + ] = {} for module in runner.get_model().modules(): if not isinstance(module, MambaMixer2) or not module.use_replayssm: continue assert module.replayssm_buffer_len is not None ring_start = module._replayssm_ring_start prev_num_accepted = module._replayssm_prev_num_accepted + prev_query_len = module._replayssm_prev_query_len tracker_specs.setdefault( ring_start.data_ptr(), ( ring_start, prev_num_accepted, + prev_query_len, module.replayssm_buffer_len, module.kv_cache[2].size(2), ), @@ -95,6 +105,7 @@ def _temporary_replayssm_autotune_state( *module.kv_cache, ring_start, prev_num_accepted, + prev_query_len, ) for tensor in tensors: if tensor.numel(): @@ -121,20 +132,26 @@ def _temporary_replayssm_autotune_state( for ( ring_start, prev_num_accepted, + prev_query_len, logical_window, ring_buffer_len, ) in tracker_specs.values(): # Compile reset (prefill) and advance (decode) before inference. # The final reset leaves the decode tuning run in a clean state. - reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots) + reset_replayssm_ring_trackers( + ring_start, prev_num_accepted, prev_query_len, state_slots + ) update_replayssm_ring_trackers( ring_start, prev_num_accepted, + prev_query_len, state_slots, logical_window, ring_buffer_len, ) - reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots) + reset_replayssm_ring_trackers( + ring_start, prev_num_accepted, prev_query_len, state_slots + ) try: yield @@ -151,7 +168,13 @@ def _temporary_replayssm_autotune_state( def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None: - autotune = _replayssm_autotune_kwargs(runner) + max_token_prefill_kwargs = { + "num_tokens": runner.scheduler_config.max_num_batched_tokens, + "skip_eplb": True, + "is_profile": True, + "randomize_inputs": True, + } + autotune = _replayssm_autotune_kwargs(runner, max_token_prefill_kwargs) if autotune is None: return max_num_reqs, decode_kwargs = autotune diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index c831266109..7c543570aa 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -87,6 +87,8 @@ class BaseMambaAttentionMetadata: bc_pre_scratch: torch.Tensor | None = None # ReplaySSM — FlashInfer checkpointing_ssu two-kernel scratch. replayssm_scratch: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None + # Contiguous cache-slot indices shared by all FlashInfer ReplaySSM layers. + replayssm_state_indices_d: torch.Tensor | None = None class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): @@ -182,6 +184,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): self.decode_replayssm_scratch: ( tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None ) = None + self.decode_replayssm_state_indices_d: torch.Tensor | None = None # ReplaySSM CUDA-graph buffers for the selected backend. if self.use_replayssm and not self.use_flashinfer_replayssm: self.decode_write_pos_d: torch.Tensor = torch.empty( @@ -218,11 +221,17 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): self.decode_replayssm_scratch = allocate_checkpointing_ssu_scratch( batch_size=scheduler_config.max_num_seqs, num_heads=nheads, - num_predicted_tokens=1, + num_predicted_tokens=1 + self.num_spec_tokens, max_window=self.replayssm_buffer_len, dtype=vllm_config.model_config.dtype, device=device, ) + # Full CUDA graphs retain capture-time tensor addresses. Keep the + # contiguous first-column view used by FlashInfer in a persistent + # buffer and refresh its contents before each replay. + self.decode_replayssm_state_indices_d = torch.empty( + (self.decode_cudagraph_max_bs,), dtype=torch.int32, device=device + ) self._init_reorder_batch_threshold(1, self.use_spec_decode) if self.use_spec_decode: @@ -592,13 +601,16 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): ] state_indices_tensor_p = state_indices_tensor_p[:, 0] - # Sometimes even with specdec enabled we get single-token prefill chunks that - # should be treated as decodes but don't have num_accepted_tokens set. - # These should be fine to process as non-spec decodes since there's only - # one token, so no risk of placing accepted tokens in the wrong slot. - if num_decodes > 0 and self.use_spec_decode and num_accepted_tokens is not None: + if num_decodes > 0 and self.use_spec_decode: query_start_loc_d = common_attn_metadata.query_start_loc[: num_decodes + 1] - num_accepted_tokens = num_accepted_tokens[:num_decodes] + if num_accepted_tokens is None: + # Single-token prefill chunks can be reclassified as decodes before + # speculative decoding has produced acceptance counts. Treat each + # token as accepted so recurrent state and ReplaySSM trackers follow + # the normal speculative-decode path. + num_accepted_tokens = torch.diff(query_start_loc_d) + else: + num_accepted_tokens = num_accepted_tokens[:num_decodes] if num_prefills > 0: if num_computed_tokens is None: @@ -788,6 +800,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): is_flush_d = metadata.is_flush_d bc_pre_scratch = metadata.bc_pre_scratch replayssm_scratch = metadata.replayssm_scratch + replayssm_state_indices_d = None if ( metadata.num_prefills == 0 and metadata.num_decodes <= self.decode_cudagraph_max_bs @@ -876,6 +889,20 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): cumAdt_vec[:padded_bs], cb_old[:padded_bs], ) + assert self.decode_replayssm_state_indices_d is not None + self.decode_replayssm_state_indices_d[:padded_bs].copy_( + state_indices_tensor_d[:, 0], non_blocking=True + ) + replayssm_state_indices_d = self.decode_replayssm_state_indices_d[ + :padded_bs + ] + + if ( + self.use_flashinfer_replayssm + and state_indices_tensor_d is not None + and replayssm_state_indices_d is None + ): + replayssm_state_indices_d = state_indices_tensor_d[:, 0].contiguous() return replace( metadata, @@ -886,6 +913,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): is_flush_d=is_flush_d, bc_pre_scratch=bc_pre_scratch, replayssm_scratch=replayssm_scratch, + replayssm_state_indices_d=replayssm_state_indices_d, block_idx_last_scheduled_token=block_idx_last_scheduled_token, block_idx_last_computed_token=block_idx_last_computed_token, block_idx_last_scheduled_token_prev_step=( From 8fb16ea5a66f0ef362e728f0ff025f2845927fd5 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Wed, 30 Sep 2026 17:47:28 -0400 Subject: [PATCH 154/248] [Docs] Add PR checklist skill for coding agents (#57084) Signed-off-by: Benjamin Chislett Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .agents/skills/pr-checklist/SKILL.md | 190 +++++++++++++++++++++++++++ .github/PULL_REQUEST_TEMPLATE.md | 33 +++-- AGENTS.md | 1 + docs/contributing/model/basic.md | 2 + 4 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 .agents/skills/pr-checklist/SKILL.md diff --git a/.agents/skills/pr-checklist/SKILL.md b/.agents/skills/pr-checklist/SKILL.md new file mode 100644 index 0000000000..886e6211a7 --- /dev/null +++ b/.agents/skills/pr-checklist/SKILL.md @@ -0,0 +1,190 @@ +--- +name: pr-checklist +description: Prepare a vLLM change for human PR review or re-review, or analyze an open pull request. Use to check design fit, behavioral coverage, performance evidence, diff quality, and closure of previous feedback before requesting maintainer attention. +--- + +# PR Checklist + +This SKILL is designed to help systematically enforce vLLM's code quality and contribution standards during the pull request process, +and to reduce the burden of manual code review on maintainers. + +## Usage + +The following checklist can be used to polish a change in preparation for a vLLM pull request, analyze an open pull request (either as the author or as a reviewer), or ensure that all contribution standards are met before requesting maintainer attention. To use this checklist effectively, go through each item and verify that it has been addressed in the diff / pull request. + +This is intended to be used with a human in the loop: when running programmatically, prepare notes on every section to ensure a thorough report, with a clear header summarizing the findings and areas needing attention. When running interactively, present this report to the user with a clear list of actionable follow-up tasks and design decisions to make, then iterate. + +### Re-Review + +To update a PR following a review, read all comments and threads on the PR. For each concrete concern, work with your human operator to decide on a plan of action to address each either by a direct reply or by updating the change and acknowledging with a comment. Ensure the PR description remains fresh and all relevant documentation (including comments) in the change is up-to-date. + +`gh pr view --comments` does not include inline review comments. Set `pr` to the PR number, then fetch all pages of conversation comments, review submissions, and inline comments (including replies): + +```bash +gh api --paginate "repos/vllm-project/vllm/issues/$pr/comments" +gh api --paginate "repos/vllm-project/vllm/pulls/$pr/reviews" +gh api --paginate "repos/vllm-project/vllm/pulls/$pr/comments" +``` + +Use `in_reply_to_id` to group inline replies with their parent comments. Use GraphQL `reviewThreads` when resolved or outdated thread status is needed. + +## Sections + +### 1. Design Fit + +Does the change align with the overall design and architecture of vLLM? Does it introduce any design inconsistencies or anti-patterns? Are there any potential long-term maintenance concerns related to this change? + +#### 1.1: Impact on Core Components + +vLLM's core components are shared across many different deployments. Changes to these files have high risk and changes should be structured in order to be easily maintainable and minimally invasive. + +Such components include: + +- The Model Runner (`vllm/v1/worker/*/model_runner.py`) +- The Scheduler (`vllm/v1/core/sched/*`) +- Core infrastructure (`vllm/v1/core/*`, including KV cache managers and utilities) + +etc. + +Changes should be structured to minimize complexity added to these files. Consider refactoring or isolating new functionality to reduce the impact on core components, both in terms of code maintainability and potential for introducing bugs. + +Nontrivial logic associated with non-standard/non-default functionality should not be added to core/common code paths and files. + +#### 1.2: Replication + +Some changes may duplicate code that already exists elsewhere in the codebase. Does the change introduce unnecessary replication, or could existing functionality be reused instead? Consider refactoring to reduce duplication and improve maintainability. + +Consult the affected area's agent instructions and contribution guides for its reuse conventions. See the [model implementation guide](../../../docs/contributing/model/basic.md) for model code and the [MRV2 development guidance](../../../docs/design/model_runner_v2.md#development-philosophy). Justify any duplication needed to isolate functionality and limit its impact on existing workflows. + +#### 1.3: Complexity + +Changes which increase the complexity of the codebase should be carefully scrutinized. Specifically those which add new abstractions, dependencies, or branching cases to existing components should be evaluated for their necessity and potential impact on maintainability. + +Consider whether the added complexity could be mitigated through refactoring, modularization, or other design improvements. Aim to keep the codebase as simple and understandable as possible while still achieving the desired functionality. + +As a rule of thumb, the complexity of a change can be estimated by the number of new lines of code added to the main codebase: + +- Low complexity: < 50 lines. Generally easy to review and unlikely to introduce significant complexity. +- Medium complexity: 50-250 lines. Requires careful review and consideration of potential impact on maintainability. +- High complexity: > 250 lines. May introduce significant complexity and requires thorough review and justification. + +You may exclude test cases and documentation updates from this line count, as they typically do not contribute significantly to the complexity of the main change. Note that line count is only a coarse indicator, meant to guide and flag possible overly-complex contributions. Use judgement when assessing the true semantic complexity and implied review burden. + +Bug fixes should aim to remain below the "High complexity" bar. Those which exceed this threshold often warrant additional scrutiny and justification. Redesigning core components, modifying core interfaces, or otherwise introducing significant complexity in order to patch a bug is very often discouraged and should be carefully justified. + +A PR that adds significant complexity or a substantial amount of new code must be well justified. A new feature should be relevant to, or requested by, enough users to warrant its size and complexity. A performance optimization should deliver end-to-end improvements on workloads that matter, and those gains should be large enough to outweigh the complexity it adds. + +#### 1.4: Correctness and Compatibility + +Ensure that all changes to vLLM are sound and do not violate existing contracts. Specifically, verify that all affected deployments which share an execution pathway with the changed code will remain correct. Trace related implementations, alternative backends sharing the same flow, and affected call sites of modified functions. + +When adding features or changing contracts, ensure that relevant feature compatibility is assessed. Identify any potential incompatibilities or breaking changes and justify them explicitly in the PR notes. + +When adding functionality, consider parallels with existing designs and identify opportunities to simplify and unify them rather than adding a separate implementation alongside them. + +#### 1.5: Tradeoffs + +When making a design change, adding a feature, or contributing a performance improvement, assess the tradeoffs and potential downsides. When changing a default backend to improve performance as measured on one configuration, could this cause a decrease in performance for another configuration? Could a new architecture change accelerate one workload but cause a feature incompatibility or performance regression in another? Ensure tradeoffs are clearly noted and validation is sufficient to catch potential lurking regressions or drawbacks. + +Check that the change does not introduce CPU/GPU synchronization on the main model execution thread during steady-state inference. Inspect explicit synchronization, host reads of GPU tensors, boolean tensors as indices, and transfers that can block the host. Run the affected configuration with `VLLM_GPU_SYNC_CHECK=error` to confirm a concern; passing CI is insufficient when CI does not exercise that configuration. See the [MRV2 async execution design](../../../docs/design/model_runner_v2.md#2-async-first) for examples. + +#### 1.6: Model Runner V1 Deprecation + +Account for Model Runner V1 (MRV1) deprecation when reviewing model-runner changes. Target new GPU model-runner features at Model Runner V2 (MRV2) and follow its [design and development guidance](../../../docs/design/model_runner_v2.md#development-philosophy). For fixes needed by deployments still using MRV1, explain why the legacy path needs the change and check whether MRV2 is also affected. + +### 2. Testing and Validation + +All serving-engine changes contributed to vLLM must be _validated_: bug fixes must be tested on a reproducer, and new features or performance optimizations must be run locally and/or benchmarked to ensure correctness and effectiveness. It is unacceptable, for example, to contribute a bugfix patch without running vLLM end-to-end and verifying that it resolves the issue as intended. + +Beyond _validation_, many (but not all) changes to vLLM should be _tested_: this includes writing new tests, updating existing tests, and ensuring that the test suite continues to provide reliable coverage of the affected functionality. + +#### 2.1: Test Coverage and Effectiveness + +When adding or modifying tests, consider both the coverage and the effectiveness of the test suite. Coverage refers to the extent to which the codebase is exercised by tests, while effectiveness refers to the ability of the tests to catch regressions and ensure correctness. + +Assess whether the included test cases are meaningful, concise, and maintainable. Testing should not be exhaustive: local validation is typically sufficient to ensure a baseline correctness of the change. Contributed tests should focus exclusively on areas that are deemed prone to errors, regressions, or complex interactions within the codebase. Recommend deletion of redundant, trivial, or otherwise unnecessary tests: this is one of the most common complaints raised during code reviews. + +Verify that testing actually exercises the new path, rather than passing through an unchanged path or fallback. Ensure added speculative decoding correctness tests assert nonzero acceptance rate, selection tests identify the desired backend, etc. + +Aim to write tests that are meaningful, targeted, and maintainable. Avoid redundant or trivial tests that do not contribute to the overall reliability of the test suite. Avoid using mock objects, especially when highly coupled with internal logic. + +#### 2.2: Test Reliability and Robustness + +Some tests are considered "flaky": they may pass or fail intermittently without any changes to the codebase. Flaky tests can undermine confidence in the test suite and block successful CI runs. Analyze the added tests for potential flakiness and ensure they are reliable and robust. + +Frequent sources of flaky tests include: + +- Assertions requiring bit-exact output correctness when minor floating-point variation is expected. For example, a change in default kernel selection priority may cause exact-floating-point-correctness assertions to fail. +- Tests that depend on the order of execution or the state of shared resources, which can lead to intermittent failures when tests are run in parallel or in different environments. + +#### 2.3: Test Automation and Integration + +vLLM's CI runs in Buildkite, with the launch configuration under `.buildkite/`. Ensure new tests run in an appropriate CI workflow when the required resources are available; add or update a workflow where needed. If CI cannot exercise the changed path because the required hardware or checkpoint is unavailable, document the missing coverage, the resource constraint, and validation performed outside CI. A CI coverage gap does not waive validation: the human submitter must confirm the required validation before publishing the PR. + +### 3. Code Quality and Style + +Code quality is another of the primary areas that maintainer effort is spent to address. Carefully audit the changes for adherence to coding standards, readability, maintainability, and overall design principles. + +#### 3.1: Comments + +Keep comments concise and use them sparingly. Consider whether the same information can be conveyed in fewer lines, and prefer clear code with meaningful, concise variable names, and fewer comments overall. Use comments to explain the reasoning behind complex logic, non-obvious decisions, or important context that is not immediately apparent from the code itself. Context regarding specific user decisions made during the design phase are often not needed and can instead be included as notes in the PR description instead. + +When writing new functions, consider when a docstring may be useful. Most short helper functions do not need a docstring, but more complex functions or those forming part of the public API should include one. In particular, docstrings are useful when adding custom kernels or other performance-optimized implementations of otherwise-straightforward logic, as they make clear the input and output shapes/dtypes/assumptions, and can shed light on hard-to-follow implementations. + +For bug fixes, there is no need to include details of the original bug and explanation of why the pre-changed version of the code would be incorrect. Those details should be in the PR description instead. + +#### 3.2: Documentation and Examples + +Ensure that any new features, changes, or public APIs are properly documented. This includes updating relevant documentation files under `docs/` and providing usage examples where applicable. Refer to `supported_models.md` for new models and `examples/` for usage examples. + +#### 3.3: Helper Functions + +Identify and inline trivially-simple helper functions which have only one call site. + +#### 3.4: Formatting + +Avoid formatting or whitespace changes that are unrelated to the PR, as long as pre-commit still passes. + +### 4. Pull Request Contents + +This section pertains to the contents of the pull request itself, rather than the changes. + +When acting as the author, prepare or update these contents. When acting as a reviewer, assess them and report missing context or evidence. + +Start from `.github/PULL_REQUEST_TEMPLATE.md`, retaining its sections and checklist. When creating a PR interactively, use `gh pr create --template .github/PULL_REQUEST_TEMPLATE.md`. For noninteractive creation or editing, fill out a copy of that template and pass the completed file with `--body-file`. + +#### 4.1: Description and Context + +Ensure the PR description has a short 1-2 line summary of the changes to facilitate quick understanding by reviewers. This section should link relevant issues, predecessor/blocking PRs, and provide any necessary context for the changes. + +Keep Claims to a few concrete bullets and explain motivation briefly. Keep explanations in Validation and Details concise too, while retaining the evidence, root cause, tradeoffs, and limitations needed to assess the change. Link full logs or use collapsible HTML details for lengthy supporting material rather than polluting the main narrative. + +#### 4.2: Claims and Supporting Evidence + +Ensure the PR makes clear, concise claims about the accomplishment: + +- For bug fixes, what specific issue is being resolved and how can it be verified? +- For new features, what functionality is being added and how can it be utilized? +- For refactoring, what was the motivation and what improvements are expected? +- For performance optimizations, what specific model/hardware/serving configurations & workloads are being targeted and how can the performance gains be measured? + +For the stated claims, ensure the PR provides sufficient evidence, examples, or references to support them, making it easy for reviewers to verify the correctness and impact of the changes. + +Summarize model evaluations and benchmarks in compact tables with the configuration, baseline, changed result, and reproduction commands. For repeated benchmarks, report the run count and variability. For kernel performance changes, include before/after profile visualizations/timelines when they help explain the measured gains. Link relevant CI failures or regressing PRs when they motivate the change. + +- For bug fixes, ensure the reproducer is clearly provided and was validated to confirm the issue and the resolution. +- For new features, ensure that usage examples are included and validated to confirm that the feature works as intended. +- For refactoring, ensure that changes are sufficiently validated at appropriate test levels (unit is sufficient for minor refactors, E2E serving is required for significant refactoring). +- For performance optimizations, ensure that benchmarks or tests are included to validate the claimed performance improvements. Ideally, an end-to-end evaluation should demonstrate speedup on a concrete serving example. Microbenchmark-only validation is often insufficient: make a note if end-to-end evaluation is not included (e.g. due to lack of access to sufficient hardware to fully validate). + +#### 4.3: Root-Cause Analysis + +When fixing bugs or issues, provide a clear root-cause analysis. Explain what caused the problem, how it was identified, and why the chosen solution effectively addresses it. A patch which resolves the issue without a proper root-cause analysis may lead to recurring problems and increased maintenance overhead. + +#### 4.4: Implementation Details + +Provide a detailed explanation of the implementation, including design decisions, trade-offs, and any relevant technical details. Highlight any non-obvious choices, tradeoffs, potential limitations, and areas where maintainer attention is required. + +#### 4.5: Contributing Guide + +Ensure that the PR follows the [contributing guide](https://docs.vllm.ai/en/latest/contributing), including adhering to coding standards, commit message conventions, and any other project-specific guidelines. This helps maintain consistency and quality across the codebase. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8a3934670e..3f3c00d981 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,20 +1,33 @@ -PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED. -## Purpose +## Overview -## Test Plan + -## Test Result +## Claims + + + +## Validation + + + +## Details + + --- -
- Essential Elements of an Effective PR Description Checklist -- [ ] The purpose of the PR, such as "Fix some issue (link existing issues this PR will resolve)". -- [ ] The test plan, such as providing test command. -- [ ] The test results, such as pasting the results comparison before and after, or e2e results -- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model. +
+ Pull Request Checklist + +- [ ] I used vLLM's `/pr-checklist` skill. (Mandatory for agents, optional for humans). +- [ ] AI assistance was used during the creation of this PR. + +- [ ] **Design Fit:** Minimizes impact on core components, reuses existing functionality, and justifies added complexity. +- [ ] **Testing and Validation:** Validates the change and ensures any added tests are meaningful and reliable, with CI coverage or documented CI resource constraints and validation performed outside CI. +- [ ] **Code Quality and Style:** Keeps code and comments clear and concise, and updates relevant documentation and examples. +- [ ] **Pull Request Contents:** Includes a brief summary and relevant links, supports claims with evidence, explains root causes and implementation trade-offs, and follows the contributing guide.
**BEFORE SUBMITTING, PLEASE READ ** (anything written below this line will be removed by GitHub Actions) diff --git a/AGENTS.md b/AGENTS.md index a1060975b0..b3ce260383 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,7 @@ Do not open one-off PRs for tiny edits (single typo, isolated style change, one - Test commands run and results. - Model evaluation results when the change affects output, accuracy, or serving. - Clear statement that AI assistance was used. +- Use vLLM's [`/pr-checklist`](.agents/skills/pr-checklist/SKILL.md) skill before submitting a PR to ensure quality and adherence to the contribution guidelines. ### Fail-closed behavior diff --git a/docs/contributing/model/basic.md b/docs/contributing/model/basic.md index 0cc24baae9..1db0981cf1 100644 --- a/docs/contributing/model/basic.md +++ b/docs/contributing/model/basic.md @@ -85,6 +85,8 @@ def forward( For reference, check out our [Llama implementation](../../../vllm/model_executor/models/llama.py). vLLM already supports a large number of models. It is recommended to find a model similar to yours and adapt it to your model's architecture. Check out [vllm/model_executor/models](../../../vllm/model_executor/models) for more examples. +Adapting model-specific structure or boilerplate can keep architectures independent. Reuse shared layer implementations, such as Attention and MoE, where their contracts fit the model. + ## 3. (Optional) Implement tensor parallelism and quantization support If your model is too large to fit into a single GPU, you can use tensor parallelism to manage it. From 26ce58b48cb7335abba9d0e2a8a522c58795402a Mon Sep 17 00:00:00 2001 From: Aarushi Jain <142941703+aarushjain29@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:58:07 -0500 Subject: [PATCH 155/248] [ROCm][CI] Sync two AMD test groups between test-amd.yaml and test_areas (#59499) Signed-off-by: aarushjain29 Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 1 + .buildkite/test_areas/engine.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 2b20e60d61..402d5d2ea9 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1859,6 +1859,7 @@ steps: - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_gptoss_tp.py; fi - if [ "$$BUILDKITE_PARALLEL_JOB" = "1" ]; then pytest -v -s -x lora/test_qwen35_densemodel_lora.py; fi - if [ "$$BUILDKITE_PARALLEL_JOB" = "2" ]; then pytest -v -s -x lora/test_gemma4_tp.py; fi + - if [ "$$BUILDKITE_PARALLEL_JOB" = "0" ]; then pytest -v -s -x lora/test_sequence_classification.py::test_batched_loras_tp; fi #------------------------------------------------------ mi300 · model_executor -------------------------------------------------------# diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index fa71c9f737..c3f0ced718 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -59,6 +59,11 @@ steps: - image-build-amd source_file_dependencies: - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/engine/test_preprocess_error_handling.py + - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py + - pytest -v -s v1/test_tensor_ipc_queue.py + - pytest -v -s config/test_mp_reducer.py - label: ":nvidia: (H200 MIG 18GB) E2E Scheduling" key: e2e-scheduling-1-gpu From 21c08ac4589493291afa9c1471d08fb8a07a901a Mon Sep 17 00:00:00 2001 From: jackLei <42642542+jackLei0901@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:10:03 -0700 Subject: [PATCH 156/248] [CI] Deflake shutdown wait-timeout test by synchronizing on request admission (#58125) Signed-off-by: jackLei0901 <42642542+jackLei0901@users.noreply.github.com> --- tests/entrypoints/launchers/test_shutdown.py | 48 ++++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/tests/entrypoints/launchers/test_shutdown.py b/tests/entrypoints/launchers/test_shutdown.py index 6d18d825eb..f03afbff09 100644 --- a/tests/entrypoints/launchers/test_shutdown.py +++ b/tests/entrypoints/launchers/test_shutdown.py @@ -14,6 +14,7 @@ import httpx import openai import psutil import pytest +from prometheus_client.parser import text_string_to_metric_families from tests.utils import RemoteOpenAIServer from vllm.utils.network_utils import get_open_port @@ -127,6 +128,40 @@ async def _concurrent_request_loop( t.cancel() +def _num_running_requests(metrics_text: str) -> float: + """Sum ``vllm:num_requests_running`` across the server's label sets.""" + total = 0.0 + for family in text_string_to_metric_families(metrics_text): + if family.name == "vllm:num_requests_running": + for sample in family.samples: + if sample.name == "vllm:num_requests_running": + total += sample.value + return total + + +async def _wait_for_running_request( + metrics_url: str, + timeout: float = _INFLIGHT_REQUEST_START_TIMEOUT, +) -> float: + """Wait until the server reports a running request, returning the delay. + + ``ShutdownState.inflight_requests`` is incremented before the HTTP call is + awaited, so it only shows that a client coroutine started. Draining can + only complete work the engine has already admitted, so synchronize on the + server's own scheduler gauge instead. + """ + start = time.monotonic() + deadline = start + timeout + async with httpx.AsyncClient() as metrics_client: + while time.monotonic() < deadline: + response = await metrics_client.get(metrics_url) + response.raise_for_status() + if _num_running_requests(response.text) > 0: + return time.monotonic() - start + await asyncio.sleep(_INFLIGHT_REQUEST_POLL_INTERVAL) + raise AssertionError(f"No request reached the running state within {timeout}s") + + @pytest.mark.asyncio async def test_shutdown_on_engine_failure(tmp_path: Path): """Verify that API returns connection error when server process is killed. @@ -240,11 +275,15 @@ async def test_wait_timeout_completes_requests(): _concurrent_request_loop(client, state, sigterm_sent, concurrency=10) ) - await asyncio.sleep(0.5) - proc.send_signal(signal.SIGTERM) - sigterm_sent.set() - try: + # Drain can only complete requests the engine already admitted, so + # wait for the server to report one instead of guessing with a sleep. + admission_delay = await _wait_for_running_request( + remote_server.url_for("metrics") + ) + proc.send_signal(signal.SIGTERM) + sigterm_sent.set() + await asyncio.wait_for(request_task, timeout=_SHUTDOWN_DETECTION_TIMEOUT) except asyncio.TimeoutError: pass @@ -257,6 +296,7 @@ async def test_wait_timeout_completes_requests(): # wait timeout should complete in-flight requests assert state.requests_after_sigterm > 0, ( f"Wait timeout should complete in-flight requests. " + f"admission_delay: {admission_delay:.3f}s, " f"503: {state.got_503}, 500: {state.got_500}, " f"conn_errors: {state.connection_errors}, errors: {state.errors}" ) From 8bc31bd21075c5ca8a070e6da84e96e2eaf816ad Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 30 Sep 2026 17:22:58 -0700 Subject: [PATCH 157/248] [CI] Deflake the multi-API-server metrics test and Mooncake PD ports (#59507) Signed-off-by: khluu Co-authored-by: Claude Opus 5.5 --- .../dev/rlhf/test_weight_operation_metrics.py | 14 +++++--- .../mooncake_integration/run_accuracy_test.sh | 32 +++++++++++++------ .../mooncake_integration/test_accuracy.py | 2 +- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py b/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py index 6c7cb60bbb..063b307767 100644 --- a/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py +++ b/tests/entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py @@ -21,8 +21,10 @@ from vllm.distributed.weight_transfer.ipc_engine import IPCTrainerInitInfo DURATION = "vllm:rl_weight_update_operation_duration_seconds" IN_FLIGHT = "vllm:rl_weight_update_operations_in_flight" -# Enough requests that the kernel spreads them over both API servers. -SYNCS = 8 +# The API servers accept from one shared listening socket, so the kernel may +# hand one process most connections. Sync until both have recorded, within a cap. +MIN_SYNCS = 8 +MAX_SYNCS = 64 def scrape(url: str) -> dict[tuple[str, str], float]: @@ -75,11 +77,15 @@ def test_weight_sync_metrics_aggregate_across_api_servers(tmp_path): client=HTTPVLLMWeightSyncClient(url), source=ModuleSource(trainer), ) - for _ in range(SYNCS): + syncs = 0 + while syncs < MIN_SYNCS or ( + len(pids_that_recorded(tmp_path)) < 2 and syncs < MAX_SYNCS + ): engine.send_weights() + syncs += 1 metrics = scrape(url) - calls = {"init": 1, "start": SYNCS, "update": SYNCS, "finish": SYNCS} + calls = {"init": 1, "start": syncs, "update": syncs, "finish": syncs} for operation, count in calls.items(): assert metrics[(f"{DURATION}_count", operation)] == count, operation assert metrics[(IN_FLIGHT, operation)] == 0, operation diff --git a/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh b/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh index 9b3d129804..6436ccdb13 100755 --- a/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh +++ b/tests/v1/kv_connector/mooncake_integration/run_accuracy_test.sh @@ -78,6 +78,20 @@ wait_for_server() { done" && return 0 || return 1 } +# b200-k8s pods use host networking, so a fixed port can already be taken by +# another job on the node. Bind all the ports at once so they are distinct. +free_ports() { + python3 - "$1" <<'EOF' +import socket +import sys + +socks = [socket.socket() for _ in range(int(sys.argv[1]))] +for s in socks: + s.bind(("", 0)) +print(" ".join(str(s.getsockname()[1]) for s in socks)) +EOF +} + # Function to clean up previous instances cleanup_instances() { echo "Cleaning up any running vLLM instances..." @@ -107,6 +121,9 @@ run_tests_for_model() { # Proxy args: repeated --prefill URL BOOTSTRAP_PORT and --decode URL. PROXY_ARGS=() + # Per prefiller: server and bootstrap. Per decoder: server. Then the proxy. + read -r -a PORTS <<< "$(free_ports $((2 * NUM_PREFILL_INSTANCES + NUM_DECODE_INSTANCES + 1)))" + PROXY_PORT=${PORTS[${#PORTS[@]} - 1]} # Start prefill instances for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do @@ -119,10 +136,8 @@ run_tests_for_model() { GPU_ID="${GPU_ID},${NEXT_GPU}" done - # Calculate port number (base port + instance number) - PORT=$((8100 + i)) - # Bootstrap server port for this prefiller. Avoid clash across instances. - BOOTSTRAP_PORT=$((8998 + i)) + PORT=${PORTS[$((2 * i))]} + BOOTSTRAP_PORT=${PORTS[$((2 * i + 1))]} echo "Starting prefill instance $i on GPU $GPU_ID, port $PORT, bootstrap $BOOTSTRAP_PORT" @@ -162,8 +177,7 @@ run_tests_for_model() { NEXT_GPU=$(((GPU_ID + j) % $(get_num_gpus))) GPU_ID="${GPU_ID},${NEXT_GPU}" done - # Calculate port number (base port + instance number) - PORT=$((8200 + i)) + PORT=${PORTS[$((2 * NUM_PREFILL_INSTANCES + i))]} echo "Starting decode instance $i on GPU $GPU_ID, port $PORT" @@ -205,16 +219,16 @@ run_tests_for_model() { done # Start the proxy server - echo "Starting proxy server on port 8192" + echo "Starting proxy server on port $PROXY_PORT" python3 "${GIT_ROOT}/examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py" \ - --port 8192 "${PROXY_ARGS[@]}" & + --port "$PROXY_PORT" "${PROXY_ARGS[@]}" & # Wait for the proxy to start sleep 5 # Run lm eval for this model echo "Running tests for $model_name" - TEST_MODEL=$model_name python3 -m pytest -s -x \ + TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \ "${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py" # Clean up before running next model diff --git a/tests/v1/kv_connector/mooncake_integration/test_accuracy.py b/tests/v1/kv_connector/mooncake_integration/test_accuracy.py index 25033399e7..aa4c72faf4 100644 --- a/tests/v1/kv_connector/mooncake_integration/test_accuracy.py +++ b/tests/v1/kv_connector/mooncake_integration/test_accuracy.py @@ -5,7 +5,7 @@ import os import lm_eval import openai -BASE_URL = "http://localhost:8192/v1" +BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1" NUM_CONCURRENT = 100 TASK = "gsm8k" FILTER = "exact_match,strict-match" From d7c55116acabbecc6c588eec2467005d5fe6eb30 Mon Sep 17 00:00:00 2001 From: Venky <23023424+venkywonka@users.noreply.github.com> Date: Wed, 30 Sep 2026 17:34:52 -0700 Subject: [PATCH 158/248] [CI] Fix MiniMax-M3 PP aux-state test mock after #58648 (#59508) Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) --- tests/v1/worker/test_eagle3_aux_hidden_states_pp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py b/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py index 309a81b36c..7c6acfafe3 100644 --- a/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py +++ b/tests/v1/worker/test_eagle3_aux_hidden_states_pp.py @@ -102,7 +102,10 @@ def minimax_pp_stage(monkeypatch): ) model = minimax_m3.MiniMaxM3Model( vllm_config=SimpleNamespace( - model_config=SimpleNamespace(hf_text_config=config), quant_config=None + model_config=SimpleNamespace(hf_text_config=config), + quant_config=None, + speculative_config=None, + use_v2_model_runner=False, ) ) model._set_aux_hidden_state_layers(taps) From c055c1e075ed461cff922aa710d387281f651940 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Wed, 30 Sep 2026 17:37:09 -0700 Subject: [PATCH 159/248] [Core] Include the LoRA path in prefix-cache block hashes (#59335) Signed-off-by: Nick Hill Co-authored-by: Claude Opus 5.5 --- tests/v1/core/test_kv_cache_utils.py | 26 +++++++++++++++++++++-- vllm/entrypoints/openai/models/serving.py | 10 +++++++++ vllm/v1/core/kv_cache_utils.py | 14 +++++++----- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 0c56e89d59..fdc9198fff 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1022,7 +1022,7 @@ def test_generate_block_hash_extra_keys_lora(): ) extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0) - assert extra_keys == (("lora", "test_lora_adapter"),) + assert extra_keys == (("lora", "test_lora_adapter", "/path/to/lora"),) request.lora_request = None extra_keys, _ = generate_block_hash_extra_keys(request, 0, 3, 0) @@ -1048,6 +1048,28 @@ def test_lora_name_and_cache_salt_block_hashes_do_not_collide(hash_fn): assert lora_req.block_hashes[0] != salted_req.block_hashes[0] +@pytest.mark.parametrize("hash_fn", [sha256, sha256_cbor]) +def test_lora_path_change_changes_block_hashes(hash_fn): + """Re-pointing a LoRA name at another adapter must not reuse its blocks.""" + + def make_lora_request(lora_path: str) -> Request: + return Request( + request_id=lora_path, + prompt_token_ids=[0, 1, 2], + sampling_params=SamplingParams(max_tokens=1), + pooling_params=None, + lora_request=LoRARequest( + lora_name="foo", lora_int_id=1, lora_path=lora_path + ), + block_hasher=get_request_block_hasher(3, hash_fn), + ) + + first = make_lora_request("/path/to/lora_a") + second = make_lora_request("/path/to/lora_b") + + assert first.block_hashes[0] != second.block_hashes[0] + + def test_to_event_extra_keys_keeps_untagged_event_format(): """KV events keep publishing the extra-key shapes consumers already parse.""" request = make_request( @@ -1064,7 +1086,7 @@ def test_to_event_extra_keys_keeps_untagged_event_format(): extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0) assert extra_keys == ( - ("lora", "adapter"), + ("lora", "adapter", "/path/to/lora"), ("mm", "hash1", 2), ("cache_salt", "salt"), ) diff --git a/vllm/entrypoints/openai/models/serving.py b/vllm/entrypoints/openai/models/serving.py index c139effa79..231fed5c4d 100644 --- a/vllm/entrypoints/openai/models/serving.py +++ b/vllm/entrypoints/openai/models/serving.py @@ -212,6 +212,16 @@ class OpenAIServingModels: status_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) + previous = self.lora_requests.get(lora_name) + if previous is not None and previous.lora_path == lora_path: + logger.warning( + "Reloaded LoRA adapter '%s' in place from the same path '%s'. " + "Prefix-cache blocks computed with its previous weights are " + "not invalidated; load changed weights from a new path to " + "avoid reusing them.", + lora_name, + lora_path, + ) self.lora_requests[lora_name] = lora_request logger.info( "Loaded new LoRA adapter: name '%s', path '%s'", lora_name, lora_path diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index a82d3e8113..4c8b807d16 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -565,20 +565,24 @@ def _gen_mm_extra_hash_keys( return extra_keys, curr_mm_idx -def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str]]: +def _gen_lora_extra_hash_keys(request: Request) -> list[tuple[str, str, str]]: """Generate extra keys related to LoRA for block hash computation. + The adapter path is included so that re-pointing a LoRA name at a different + adapter does not reuse KV computed with the previous one. + Args: request: The request object. Returns: - Return LoRA name of the request if it is a LoRA request. Return empty - list otherwise. + Return the LoRA name and path of the request if it is a LoRA request. + Return empty list otherwise. """ - if not request.lora_request: + lora_request = request.lora_request + if not lora_request: return [] - return [("lora", request.lora_request.lora_name)] + return [("lora", lora_request.lora_name, lora_request.lora_path)] def _gen_prompt_embeds_extra_hash_keys( From a47e4a5568b4791db0c9633b5dc7531128f98bae Mon Sep 17 00:00:00 2001 From: vx120 <57470515+vx120@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:57:00 +0800 Subject: [PATCH 160/248] [Doc] Score centering via top-k processed logprobs (#59361) Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Fable 5.1 --- docs/training/sampling_mask.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/training/sampling_mask.md b/docs/training/sampling_mask.md index 442906e488..67e0709388 100644 --- a/docs/training/sampling_mask.md +++ b/docs/training/sampling_mask.md @@ -104,6 +104,39 @@ log_prob = log_softmax(masked_logits)[sampled_token_id] Both sides normalize over the same token set, so the importance ratio is consistent. +## Score centering with top-k logprobs + +[Score centering](https://arxiv.org/abs/2609.20807) needs the sampler's top-k +token IDs and logprobs plus the sampled token's logprob; the paper uses +`k=128` and does not need the exact sampling support. vLLM already returns +this through the ordinary `logprobs` parameter: + +```bash +vllm serve --logprobs-mode processed_logprobs --max-logprobs 128 +``` + +```json +{"sampling_params": {"temperature": 1.0, "top_p": 0.9, "logprobs": 128}} +``` + +Each position then carries the sampled token's logprob and the top-128 +`(token_id, logprob)` pairs, normalized over the post-truncation support: + +```json +{"token": "token_id:11", "logprob": -0.757, + "top_logprobs": [{"token": "token_id:11", "logprob": -0.757}, + {"token": "token_id:13", "logprob": -0.632}, + {"token": "token_id:0", "logprob": -9999.0}, ...]} +``` + +- `--max-logprobs` defaults to 20; raise it to the requested `k`. +- Tokens outside the support are returned as `-9999.0` with arbitrary IDs; + drop them. The finite entries are the sampling mask when it has at most `k` + tokens. +- The sampled token is always the first entry and may repeat inside the top-k. +- `/inference/v1/generate` encodes IDs as `token_id:`; the OpenAI + endpoints need `return_tokens_as_token_ids`. + ## Limitations - **Engine-level flag:** `--return-sampling-mask` globally disables the From 37d6174096fd99a8d362c9f5b379f7ec59deedb8 Mon Sep 17 00:00:00 2001 From: Guy Stone Date: Wed, 30 Sep 2026 21:03:39 -0400 Subject: [PATCH 161/248] [Metrics] Add --custom-histogram-buckets to override histogram bucket families (#48867) The core engine histograms ship with default bucket boundaries tuned for typical serving workloads. The `--custom-histogram-buckets` option replaces the boundaries of one or more bucket families. Use it, for example, to track sub-300ms latency SLOs with the request-phase histograms, whose smallest default boundary is 0.3s. Signed-off-by: Guy Stone Co-authored-by: Claude Fable 5 --- docs/usage/metrics.md | 42 +++++++++++++++ tests/config/test_observability.py | 53 ++++++++++++++++++ tests/v1/engine/test_engine_args.py | 31 +++++++++++ tests/v1/metrics/test_histogram_buckets.py | 46 ++++++++++++++++ vllm/config/observability.py | 63 ++++++++++++++++++++++ vllm/engine/arg_utils.py | 8 +++ vllm/v1/metrics/buckets.py | 15 ++++-- vllm/v1/metrics/loggers.py | 31 ++++++++--- 8 files changed, 278 insertions(+), 11 deletions(-) create mode 100644 tests/config/test_observability.py diff --git a/docs/usage/metrics.md b/docs/usage/metrics.md index 8254a7f420..5cb5eebc55 100644 --- a/docs/usage/metrics.md +++ b/docs/usage/metrics.md @@ -75,6 +75,48 @@ These metrics are available via `--enable-mfu-metrics`: --8<-- "gen:metrics-mfu" +## Custom Histogram Buckets + +The core engine histograms ship with default bucket boundaries tuned for +typical serving workloads. The `--custom-histogram-buckets` option replaces +the boundaries of one or more *bucket families* — exactly the histograms +listed in the table below — with your own list; histograms owned by other +subsystems (for example, the NIXL connector metrics) are not affected. Use it, +for example, to track sub-300ms latency SLOs with the request-phase +histograms, whose smallest default boundary is 0.3s: + +```bash +vllm serve Qwen/Qwen3-0.6B \ + --custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 30.0]}' +``` + +Each family key overrides a group of related histograms: + +| Family key | Histograms | +| --- | --- | +| `request_latency` | `vllm:e2e_request_latency_seconds`, `vllm:request_queue_time_seconds`, `vllm:request_inference_time_seconds`, `vllm:request_prefill_time_seconds`, `vllm:request_decode_time_seconds` | +| `time_to_first_token` | `vllm:time_to_first_token_seconds` | +| `inter_token_latency` | `vllm:inter_token_latency_seconds`, `vllm:request_time_per_output_token_seconds` | +| `iteration_tokens` | `vllm:iteration_tokens_total` | +| `request_params_n` | `vllm:request_params_n` | +| `request_num_preemptions` | `vllm:request_num_preemptions` | +| `request_tokens` | `vllm:request_prompt_tokens`, `vllm:request_generation_tokens`, `vllm:request_max_num_generation_tokens`, `vllm:request_params_max_tokens`, `vllm:request_prefill_kv_computed_tokens` | +| `kv_cache_residency` | `vllm:kv_block_lifetime_seconds`, `vllm:kv_block_idle_before_evict_seconds`, `vllm:kv_block_reuse_gap_seconds` | + +Bucket values must be positive, finite, and strictly increasing; unknown +family keys are rejected at startup. Families you do not list keep their +default boundaries. The `request_tokens` defaults normally scale with +`--max-model-len`; an override replaces that computed list. The +`kv_cache_residency` family only takes effect when `--kv-cache-metrics` is +enabled. + +!!! warning "Bucket cardinality" + Every bucket boundary creates one extra time series per metric and per + label combination (model and engine index, multiplied under data-parallel + deployments). Long bucket lists inflate Prometheus storage, scrape sizes, + and query costs. Keep custom lists short, and only override the families + you actively monitor. + ## Deprecation Policy Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1` diff --git a/tests/config/test_observability.py b/tests/config/test_observability.py new file mode 100644 index 0000000000..c62f01cc05 --- /dev/null +++ b/tests/config/test_observability.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ObservabilityConfig validation.""" + +import pytest +from pydantic import ValidationError + +from vllm.config.observability import ObservabilityConfig + + +def test_custom_histogram_buckets_default_none(): + assert ObservabilityConfig().custom_histogram_buckets is None + + +def test_custom_histogram_buckets_valid(): + config = ObservabilityConfig( + custom_histogram_buckets={ + "request_latency": [0.01, 0.05, 0.1, 0.5], + "time_to_first_token": [1, 2, 5], + } + ) + assert config.custom_histogram_buckets == { + "request_latency": [0.01, 0.05, 0.1, 0.5], + "time_to_first_token": [1.0, 2.0, 5.0], + } + + +@pytest.mark.parametrize( + ("buckets", "match"), + [ + ({"bogus": [1.0, 2.0]}, "unknown bucket family 'bogus'"), + ({"request_latency": []}, "must not be empty"), + ({"request_latency": [0.0, 1.0]}, "must be finite and greater than 0"), + ({"request_latency": [-1.0, 1.0]}, "must be finite and greater than 0"), + ( + {"request_latency": [1.0, float("inf")]}, + "must be finite and greater than 0", + ), + ({"request_latency": [float("nan")]}, "must be finite and greater than 0"), + ({"request_latency": [1.0, 1.0]}, "must be strictly increasing"), + ({"request_latency": [2.0, 1.0]}, "must be strictly increasing"), + ({"request_latency": [True, 2.0]}, "must be a number, not a boolean"), + ({"request_latency": [False]}, "must be a number, not a boolean"), + ( + {"request_latency": [1.0, 2.0], "request_tokens": [16, 8]}, + r"request_tokens.*must be strictly increasing", + ), + ], +) +def test_custom_histogram_buckets_rejects_invalid(buckets, match): + """Each validation rule must name the field, the value, and the rule.""" + with pytest.raises(ValidationError, match=match): + ObservabilityConfig(custom_histogram_buckets=buckets) diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index 3f9f617bc7..5516ed840e 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -108,6 +108,37 @@ def test_mm_prefix_lm_raises_batched_tokens_floor(): assert vllm_config.scheduler_config.max_num_batched_tokens >= 2496 +def test_custom_histogram_buckets_from_cli(): + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + + # Unset: byte-identical default behavior. + args = parser.parse_args([]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.observability_config.custom_histogram_buckets is None + + # A JSON mapping round-trips into the config verbatim. + args = parser.parse_args( + [ + "--custom-histogram-buckets", + '{"request_latency": [0.01, 0.05, 0.1, 0.5]}', + ] + ) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.observability_config.custom_histogram_buckets == { + "request_latency": [0.01, 0.05, 0.1, 0.5] + } + + # Malformed JSON is rejected at argument-parsing time. + parser.exit_on_error = False + with pytest.raises(ArgumentError): + parser.parse_args(["--custom-histogram-buckets", "{not json"]) + + # An unknown family key parses as JSON but fails config validation. + args = parser.parse_args(["--custom-histogram-buckets", '{"bogus": [1.0, 2.0]}']) + with pytest.raises(ValueError, match="unknown bucket family"): + EngineArgs.from_cli_args(args=args).create_engine_config() + + def test_data_parallel_start_rank_zero_infers_hybrid_lb(): """An explicit --data-parallel-start-rank 0 must be treated the same as any other explicit start rank when inferring hybrid LB mode, not as diff --git a/tests/v1/metrics/test_histogram_buckets.py b/tests/v1/metrics/test_histogram_buckets.py index 54cc31da94..cf905b5d8b 100644 --- a/tests/v1/metrics/test_histogram_buckets.py +++ b/tests/v1/metrics/test_histogram_buckets.py @@ -271,3 +271,49 @@ def test_prometheus_logger_default_buckets(): assert found[metric_name] == [float(b) for b in expected], metric_name finally: unregister_vllm_metrics() + + +def test_histogram_buckets_override_precedence(): + """A present family key wins verbatim; absent keys keep defaults.""" + overrides = { + "request_latency": [0.5, 1.0], + "request_tokens": [16.0, 4096.0], + } + got = histogram_buckets("request_latency", overrides=overrides) + assert got == [0.5, 1.0] + assert got is not overrides["request_latency"] + # An override replaces the max_model_len-derived series entirely. + assert histogram_buckets("request_tokens", overrides=overrides) == [ + 16.0, + 4096.0, + ] + assert ( + histogram_buckets("time_to_first_token", overrides=overrides) + == DEFAULT_BUCKET_SNAPSHOTS["time_to_first_token"] + ) + + +def test_prometheus_logger_applies_overrides(): + """Every family must honour an override, with a distinct list each. + + Overriding all of them at once is what catches a histogram wired to the + defaults instead of the override, and a family wired to the wrong list. + """ + overridden: dict[str, list[float]] = { + family: [index + 1.0, index + 2.0] + for index, family in enumerate(sorted(BUCKET_FAMILY_KEYS)) + } + config = build_logger_config( + ObservabilityConfig( + kv_cache_metrics=True, + custom_histogram_buckets=overridden, + ) + ) + try: + PrometheusStatLogger(config) + found = collect_histogram_buckets() + assert set(found) == set(METRIC_FAMILIES) + for metric_name, family in METRIC_FAMILIES.items(): + assert found[metric_name] == overridden[family], metric_name + finally: + unregister_vllm_metrics() diff --git a/vllm/config/observability.py b/vllm/config/observability.py index a39962a075..28a9313603 100644 --- a/vllm/config/observability.py +++ b/vllm/config/observability.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import math from functools import cached_property +from itertools import pairwise from typing import Any, Literal from packaging.version import parse @@ -65,6 +67,18 @@ class ObservabilityConfig: kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1) """Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks.""" + custom_histogram_buckets: dict[str, list[float]] | None = None + """Custom Prometheus histogram bucket boundaries, as a JSON mapping from + bucket-family key to a list of strictly increasing, positive, finite + upper bounds. When a family key is present, its list replaces the default + buckets for every histogram in that family; families not listed keep + their defaults. Known families: `request_latency`, `time_to_first_token`, + `inter_token_latency`, `iteration_tokens`, `request_params_n`, + `request_num_preemptions`, `request_tokens`, `kv_cache_residency`. Example: + `--custom-histogram-buckets '{"request_latency": [0.01, 0.05, 0.1, 0.5]}'`. + Note that every extra bucket adds one time series per metric and label + set.""" + cudagraph_metrics: bool = False """Enable CUDA graph metrics (number of padded/unpadded tokens, runtime cudagraph dispatch modes, and their observed frequencies at every logging interval).""" @@ -150,6 +164,55 @@ class ObservabilityConfig: ) return value + @field_validator("custom_histogram_buckets", mode="before") + @classmethod + def _reject_bool_histogram_bounds(cls, value: object) -> object: + """Reject booleans before pydantic silently coerces them to floats.""" + if isinstance(value, dict): + for family, buckets in value.items(): + if not isinstance(buckets, list): + continue + for bound in buckets: + if isinstance(bound, bool): + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bound " + f"{bound!r} must be a number, not a boolean" + ) + return value + + @field_validator("custom_histogram_buckets") + @classmethod + def _validate_custom_histogram_buckets( + cls, value: dict[str, list[float]] | None + ) -> dict[str, list[float]] | None: + if value is None: + return value + from vllm.v1.metrics.buckets import BUCKET_FAMILY_KEYS + + for family, buckets in value.items(): + if family not in BUCKET_FAMILY_KEYS: + raise ValueError( + f"custom_histogram_buckets: unknown bucket family " + f"{family!r}; known families: {sorted(BUCKET_FAMILY_KEYS)}" + ) + if not buckets: + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bucket list " + "must not be empty" + ) + for bound in buckets: + if not math.isfinite(bound) or bound <= 0: + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bound " + f"{bound!r} must be finite and greater than 0" + ) + if any(a >= b for a, b in pairwise(buckets)): + raise ValueError( + f"custom_histogram_buckets[{family!r}]: bounds {buckets} " + "must be strictly increasing" + ) + return value + @model_validator(mode="after") def _validate_tracing_config(self): if self.collect_detailed_traces and not self.otlp_traces_endpoint: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index d641b1fbab..ce24ac28f2 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -723,6 +723,9 @@ class EngineArgs: kv_cache_metrics_sample: float = get_field( ObservabilityConfig, "kv_cache_metrics_sample" ) + custom_histogram_buckets: dict[str, list[float]] | None = ( + ObservabilityConfig.custom_histogram_buckets + ) cudagraph_metrics: bool = ObservabilityConfig.cudagraph_metrics enable_layerwise_nvtx_tracing: bool = ( ObservabilityConfig.enable_layerwise_nvtx_tracing @@ -1633,6 +1636,10 @@ class EngineArgs: "--kv-cache-metrics-sample", **observability_kwargs["kv_cache_metrics_sample"], ) + observability_group.add_argument( + "--custom-histogram-buckets", + **observability_kwargs["custom_histogram_buckets"], + ) observability_group.add_argument( "--cudagraph-metrics", **observability_kwargs["cudagraph_metrics"], @@ -2126,6 +2133,7 @@ class EngineArgs: per_request_spec_decode_metrics=self.per_request_spec_decode_metrics, kv_cache_metrics=self.kv_cache_metrics, kv_cache_metrics_sample=self.kv_cache_metrics_sample, + custom_histogram_buckets=self.custom_histogram_buckets, cudagraph_metrics=self.cudagraph_metrics, enable_layerwise_nvtx_tracing=self.enable_layerwise_nvtx_tracing, enable_mfu_metrics=self.enable_mfu_metrics, diff --git a/vllm/v1/metrics/buckets.py b/vllm/v1/metrics/buckets.py index 7b9efb8f6d..213caabdfb 100644 --- a/vllm/v1/metrics/buckets.py +++ b/vllm/v1/metrics/buckets.py @@ -9,6 +9,7 @@ subsystems, such as the KV connector and offloading metrics, keep their own boundaries. """ +from collections.abc import Mapping from typing import Literal, get_args BucketFamilyKey = Literal[ @@ -205,23 +206,31 @@ def build_1_2_5_buckets(max_value: int) -> list[float]: def histogram_buckets( family: BucketFamilyKey, max_model_len: int | None = None, + overrides: Mapping[str, list[float]] | None = None, ) -> list[float]: - """Return the default bucket boundaries for a histogram family. + """Return the bucket boundaries for a histogram family. Args: family: Canonical bucket family key. max_model_len: Cap for the token-count series of the `request_tokens` family; required for that family and ignored otherwise. + overrides: Mapping from family key to replacement bucket bounds. + A present key replaces the family's defaults verbatim. Values + must already be validated; the single validation point is + `ObservabilityConfig.custom_histogram_buckets`. Returns: A fresh list of bucket upper bounds; callers may mutate it freely. Raises: - ValueError: If `family` is `request_tokens` and `max_model_len` - is None. + ValueError: If `family` is `request_tokens`, no override is given, + and `max_model_len` is None. + """ + if overrides is not None and (custom := overrides.get(family)) is not None: + return list(custom) if family == "request_tokens": if max_model_len is None: raise ValueError( diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index 26a4142df9..ac93b26a8f 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -739,15 +739,28 @@ class PrometheusStatLogger(AggregateStatLoggerBase): for idx in engine_indexes } + custom_buckets = vllm_config.observability_config.custom_histogram_buckets request_tokens_buckets = histogram_buckets( - "request_tokens", max_model_len=max_model_len + "request_tokens", max_model_len=max_model_len, overrides=custom_buckets + ) + iteration_tokens_buckets = histogram_buckets( + "iteration_tokens", overrides=custom_buckets + ) + request_params_n_buckets = histogram_buckets( + "request_params_n", overrides=custom_buckets + ) + time_to_first_token_buckets = histogram_buckets( + "time_to_first_token", overrides=custom_buckets + ) + inter_token_latency_buckets = histogram_buckets( + "inter_token_latency", overrides=custom_buckets + ) + request_latency_buckets = histogram_buckets( + "request_latency", overrides=custom_buckets + ) + request_num_preemptions_buckets = histogram_buckets( + "request_num_preemptions", overrides=custom_buckets ) - iteration_tokens_buckets = histogram_buckets("iteration_tokens") - request_params_n_buckets = histogram_buckets("request_params_n") - time_to_first_token_buckets = histogram_buckets("time_to_first_token") - inter_token_latency_buckets = histogram_buckets("inter_token_latency") - request_latency_buckets = histogram_buckets("request_latency") - request_num_preemptions_buckets = histogram_buckets("request_num_preemptions") # # Histograms of counts @@ -925,7 +938,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase): # KV Cache residency metrics # if self.kv_cache_metrics_enabled: - kv_cache_residency_buckets = histogram_buckets("kv_cache_residency") + kv_cache_residency_buckets = histogram_buckets( + "kv_cache_residency", overrides=custom_buckets + ) histogram_kv_block_lifetime = self._histogram_cls( name="vllm:kv_block_lifetime_seconds", From df8fd42116f172b7a53bc10c8a680b05232edbed Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 30 Sep 2026 20:06:24 -0500 Subject: [PATCH 162/248] [CI][Spec Decode] Add async scheduling accuracy tests (#55840) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex Co-authored-by: OpenAI Codex --- .buildkite/test-amd.yaml | 30 +- .buildkite/test_areas/engine.yaml | 24 +- tests/v1/e2e/general/accuracy_utils.py | 491 ++++++++++++++ .../general/test_async_scheduling_accuracy.py | 624 ++++++++++++++++++ 4 files changed, 1166 insertions(+), 3 deletions(-) create mode 100644 tests/v1/e2e/general/accuracy_utils.py create mode 100644 tests/v1/e2e/general/test_async_scheduling_accuracy.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 402d5d2ea9..cc594d192e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -454,7 +454,7 @@ steps: - tests/v1/e2e/general/ - vllm/platforms/rocm.py commands: - - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py --ignore v1/e2e/general/test_kv_sharing_fast_prefill.py -k "not test_mamba_prefix_cache_mrv2" + - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py --ignore v1/e2e/general/test_async_scheduling_accuracy.py --ignore v1/e2e/general/test_kv_sharing_fast_prefill.py -k "not test_mamba_prefix_cache_mrv2" - label: ":amd: (MI250) E2E Scheduling" # TBD timeout_in_minutes: 55 @@ -470,6 +470,20 @@ steps: commands: - pytest -v -s v1/e2e/general/test_async_scheduling.py +- label: ":amd: (MI250) E2E Scheduling Accuracy" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false + agent_pool: mi250_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/ + - tests/v1/e2e/general/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/general/test_async_scheduling_accuracy.py + - label: ":amd: (MI250) V1 Engine" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] @@ -2654,6 +2668,20 @@ steps: commands: - pytest -v -s v1/e2e/general/test_async_scheduling.py +- label: ":amd: (MI300) E2E Scheduling Accuracy" # TBD + timeout_in_minutes: 55 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/v1/ + - tests/v1/e2e/general/ + - vllm/platforms/rocm.py + commands: + - pytest -v -s v1/e2e/general/test_async_scheduling_accuracy.py + - label: ":amd: (MI300) V1 Engine" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index c3f0ced718..53499d4a98 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -85,6 +85,26 @@ steps: source_file_dependencies: - vllm/platforms/rocm.py +- label: ":nvidia: (H200 MIG 18GB) E2E Scheduling Accuracy" + key: e2e-scheduling-accuracy-1-gpu + timeout_in_minutes: 53 + device: h200_18gb + source_file_dependencies: + - vllm/v1/ + - tests/v1/e2e/general/ + commands: + - pytest -v -s v1/e2e/general/test_async_scheduling_accuracy.py + mirror: + amd: + label: ":amd: (MI250) E2E Scheduling Accuracy" + dind: false + device: mi250_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/platforms/rocm.py + - label: ":nvidia: (H200 MIG 18GB) E2E Core" device: h200_18gb key: e2e-core-1-gpu @@ -93,13 +113,13 @@ steps: - vllm/v1/ - tests/v1/e2e/general/ commands: - - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py --ignore v1/e2e/general/test_kv_sharing_fast_prefill.py -k "not test_mamba_prefix_cache_mrv2" + - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py --ignore v1/e2e/general/test_async_scheduling_accuracy.py --ignore v1/e2e/general/test_kv_sharing_fast_prefill.py -k "not test_mamba_prefix_cache_mrv2" mirror: amd: label: ":amd: (MI250) E2E Core" dind: false commands: - - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py --ignore v1/e2e/general/test_kv_sharing_fast_prefill.py -k "not test_mamba_prefix_cache_mrv2" + - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py --ignore v1/e2e/general/test_async_scheduling_accuracy.py --ignore v1/e2e/general/test_kv_sharing_fast_prefill.py -k "not test_mamba_prefix_cache_mrv2" device: mi250_1 timeout_in_minutes: 50 depends_on: diff --git a/tests/v1/e2e/general/accuracy_utils.py b/tests/v1/e2e/general/accuracy_utils.py new file mode 100644 index 0000000000..c8fa770fc5 --- /dev/null +++ b/tests/v1/e2e/general/accuracy_utils.py @@ -0,0 +1,491 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Numerical checks against an independent, teacher-forced target model.""" + +import json +import math +from collections import Counter +from dataclasses import dataclass +from functools import lru_cache + +import torch +import xgrammar as xgr +from tokenizers.decoders import DecodeStream + +from tests.conftest import HfRunner +from vllm import SamplingParams +from vllm.logprobs import Logprob +from vllm.outputs import RequestOutput +from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens +from vllm.v1.engine.logprobs import LogprobsProcessor + + +@dataclass(frozen=True) +class AccuracyTolerance: + logprob_atol: float + greedy_atol: float + logprob_mean_atol: float = 0.1 + greedy_total_atol: float = 0.5 + + def logprob_error(self, reference: torch.Tensor) -> torch.Tensor: + return torch.full_like(reference, self.logprob_atol) + + +def check_first_divergence( + baseline: RequestOutput, + actual: RequestOutput, + params: SamplingParams, + tolerance: AccuracyTolerance, + context: str, +) -> None: + """Require the two engines' scores to justify their first different choice.""" + assert baseline.finished and actual.finished, context + assert len(baseline.outputs) == len(actual.outputs) == 1, context + assert baseline.prompt_token_ids is not None, context + assert baseline.prompt_token_ids == actual.prompt_token_ids, context + base, other = baseline.outputs[0], actual.outputs[0] + position = next( + (i for i, (a, b) in enumerate(zip(base.token_ids, other.token_ids)) if a != b), + None, + ) + if position is None: + assert len(base.token_ids) == len(other.token_ids), ( + f"{context}: one completion is a strict prefix of the other" + ) + assert (base.finish_reason, base.stop_reason) == ( + other.finish_reason, + other.stop_reason, + ), f"{context}: identical tokens have different stopping reasons" + return + + assert params.logprobs is not None and params.logprobs > 0, context + assert params.temperature == 0.0, context + assert params.presence_penalty == 0.0 and params.repetition_penalty == 1.0 + assert not params.logit_bias, context + a, b = base.token_ids[position], other.token_ids[position] + counts = Counter(base.token_ids[:position]) + context = ( + f"{context}, first divergence position={position}, " + f"baseline token={a}, actual token={b}" + ) + for side, completion, selected, alternative in ( + ("baseline", base, a, b), + ("actual", other, b, a), + ): + assert completion.logprobs is not None, context + assert len(completion.logprobs) == len(completion.token_ids), context + scores = completion.logprobs[position] + assert selected in scores, f"{context}, {side}: missing selected token score" + chosen = scores[selected].logprob + assert math.isfinite(chosen), f"{context}, {side}: non-finite selected score" + known = alternative in scores + if known: + competing = scores[alternative].logprob + assert math.isfinite(competing), ( + f"{context}, {side}: divergent token has a non-finite score" + ) + else: + # The sampled token may be returned in addition to the requested + # top-k. An omitted token's raw score is bounded by the kth score. + top_scores = [ + score.logprob + for token, score in scores.items() + if len(scores) == params.logprobs or token != selected + ] + assert len(top_scores) == params.logprobs, context + assert not any(math.isnan(score) for score in top_scores), context + competing = min(top_scores) + + # Both candidates were emitted with the same history and constraints; + # frequency penalties are the only supported transform changing their + # finite score difference. Prompt tokens do not contribute to counts. + chosen -= params.frequency_penalty * counts[selected] + competing -= params.frequency_penalty * counts[alternative] + gap = chosen - competing + detail = ( + f"{context}, {side}: selected={selected}, alternative={alternative}, " + f"{'gap' if known else 'gap lower bound'}={gap}" + ) + if known: + # Log-softmax and reconstructing penalties may round in FP32, but + # a near tie cannot excuse selecting the engine's own worse token. + assert gap >= -1e-5, f"{detail}: selected token is below its alternative" + assert gap <= tolerance.greedy_atol, ( + f"{detail}: divergence exceeds near-tie bound={tolerance.greedy_atol}" + ) + + +def check_logprobs( + scores: dict[int, Logprob], + reference: torch.Tensor, + token: int, + requested: int, + tolerance: AccuracyTolerance, + context: str, +) -> torch.Tensor: + """Check every score, reported rank, and the complete requested top-k.""" + assert all(0 <= token_id < reference.numel() for token_id in scores), context + assert token in scores, f"{context}: missing score for token {token}" + assert max(1, requested) <= len(scores) <= requested + 1, context + assert not torch.isnan(reference).any(), context + assert all(s.rank is not None and s.rank >= 1 for s in scores.values()), context + ids = torch.tensor(list(scores), device=reference.device) + actual = torch.tensor( + [score.logprob for score in scores.values()], device=reference.device + ) + expected = reference[ids] + finite = torch.isfinite(expected) + assert torch.equal(torch.isfinite(actual), finite), ( + f"{context}: finite/masked logprobs differ: {scores}" + ) + assert torch.equal(actual[~finite], expected[~finite]), context + assert (actual[finite] <= 0).all(), f"{context}: positive log probability" + errors = (actual[finite] - expected[finite]).abs() + bounds = tolerance.logprob_error(expected[finite]) + assert (errors <= bounds).all(), ( + f"{context}: token IDs={ids[finite].tolist()}, " + f"actual={actual[finite].tolist()}, reference={expected[finite].tolist()}, " + f"error={errors.tolist()}, bound={bounds.tolist()}" + ) + + for token_id, score in scores.items(): + value = reference[token_id] + if not torch.isfinite(value): + continue + delta = tolerance.logprob_error(value) + lower = 1 + int((reference > value + 2 * delta).sum()) + upper = max(1, int((reference >= value - 2 * delta).sum())) + assert score.rank is not None and lower <= score.rank <= upper, ( + f"{context}: token={token_id}, rank={score.rank}, " + f"reference rank interval=[{lower}, {upper}]" + ) + + if requested: + # The sampled token can be outside top-k even when it ties the kth + # score. Its strict-greater rank can therefore duplicate a top-k rank. + top_ids = ( + list(scores) + if len(scores) == requested + else [t for t in scores if t != token] + ) + assert len(top_ids) == requested, f"{context}: incomplete top-k" + assert {scores[t].rank for t in top_ids} == set(range(1, requested + 1)), ( + context + ) + ordered = sorted(top_ids, key=lambda t: scores[t].rank) + assert all( + scores[a].logprob >= scores[b].logprob for a, b in zip(ordered, ordered[1:]) + ), f"{context}: top-k scores disagree with their reported ranks" + boundary = reference.topk(requested).values[-1] + if torch.isfinite(boundary): + delta = tolerance.logprob_error(boundary) + required = (reference > boundary + 2 * delta).nonzero().flatten() + assert set(required.tolist()) <= scores.keys(), ( + f"{context}: missing an unambiguously top-{requested} token" + ) + assert (reference[top_ids] >= boundary - 2 * delta).all(), ( + f"{context}: returned token below the top-{requested} boundary" + ) + return errors + + +def check_greedy_token( + logits: torch.Tensor, + token: int, + tolerance: AccuracyTolerance, + context: str, +) -> float: + assert not torch.isnan(logits).any(), context + assert torch.isfinite(logits[token]), ( + f"{context}: selected token {token} is forbidden by the sampling parameters" + ) + best = int(logits.argmax()) + gap = float(logits[best] - logits[token]) + assert gap <= tolerance.greedy_atol, ( + f"{context}: selected token={token}, reference best={best}, " + f"logit gap={gap}, bound={tolerance.greedy_atol}" + ) + return gap + + +def check_accuracy_budget(errors, gaps, tolerance, context): + """Reject systematic sub-bound errors within each request and score stream.""" + if errors: + finite_errors = torch.cat(errors) + if finite_errors.numel(): + mean_error = float(finite_errors.mean()) + assert mean_error <= tolerance.logprob_mean_atol, ( + f"{context}: mean logprob error={mean_error}, " + f"bound={tolerance.logprob_mean_atol}" + ) + assert sum(gaps) <= tolerance.greedy_total_atol, ( + f"{context}: total greedy logit gap={sum(gaps)}, " + f"bound={tolerance.greedy_total_atol}" + ) + + +def _check_decoded_tokens(processor, scores, history, context): + ids = list(scores) + decoded = convert_ids_list_to_tokens(processor.tokenizer, ids) + decoded = processor._verify_tokens(decoded, ids, context_token_ids=history[-4:]) + for token, text in zip(ids, decoded, strict=True): + assert scores[token].decoded_token == text, ( + f"{context}: incorrect decoded text for token {token}" + ) + + +@torch.inference_mode() +def check_request_accuracy( + hf: HfRunner, + request: RequestOutput, + params: SamplingParams, + tolerance: AccuracyTolerance, + context: str, +) -> None: + """Score the actual continuation, including positions after any divergence.""" + assert request.finished and len(request.outputs) == 1, context + (completion,) = request.outputs + prompt = request.prompt_token_ids + assert prompt is not None and len(prompt) > 0, context + tokens = list(completion.token_ids) + assert params.max_tokens is not None and 0 < len(tokens) <= params.max_tokens, ( + context + ) + # Incremental decoding buffers incomplete UTF-8; decoding the final token + # list in one call can incorrectly append a replacement character. + stream = DecodeStream(skip_special_tokens=params.skip_special_tokens) + tokenizer = hf.tokenizer.backend_tokenizer + for token in prompt: + stream.step(tokenizer, token) + visible = tokens + if completion.finish_reason == "stop" and not params.include_stop_str_in_output: + visible = tokens[:-1] + expected_text = "".join(stream.step(tokenizer, token) or "" for token in visible) + assert completion.text == expected_text, ( + f"{context}: decoded text does not match generated token IDs" + ) + inputs = torch.tensor([prompt + tokens], device=hf.device) + logits = hf.model(input_ids=inputs, use_cache=False).logits[0].float() + # Use the existing byte-fallback handling with an independent request + # history; numerical scores always come from the HF model above. + decoder = LogprobsProcessor( + tokenizer=hf.tokenizer, + logprobs=None, + prompt_logprobs=None, + cumulative_logprob=None, + num_logprobs=None, + num_prompt_logprobs=None, + ) + # Prompt token i is predicted by the causal logits at position i - 1. + if params.prompt_logprobs is None: + assert request.prompt_logprobs is None, context + else: + assert request.prompt_logprobs is not None, context + assert len(request.prompt_logprobs) == len(prompt), context + assert request.prompt_logprobs[0] is None, context + errors = [] + for i, scores in enumerate(request.prompt_logprobs[1:], 1): + assert scores is not None, context + _check_decoded_tokens(decoder, scores, prompt[1:i], context) + errors.append( + check_logprobs( + scores, + logits[i - 1].log_softmax(-1), + prompt[i], + params.prompt_logprobs, + tolerance, + f"{context}, prompt position={i}", + ) + ) + check_accuracy_budget(errors, [], tolerance, f"{context}, prompt logprobs") + + if params.logprobs is None: + assert completion.logprobs is None, context + else: + assert completion.logprobs is not None, context + assert len(completion.logprobs) == len(tokens), context + + reference = SamplingReference( + hf.tokenizer, + logits.shape[-1], + hf.model.generation_config.to_dict(), + params, + ) + errors = [] + gaps = [] + for i, token in enumerate(tokens): + raw, processed = reference.distributions( + logits[len(prompt) + i - 1], tokens[:i] + ) + position = f"{context}, generated position={i}" + gaps.append(check_greedy_token(processed, token, tolerance, position)) + if completion.logprobs is not None: + _check_decoded_tokens(decoder, completion.logprobs[i], tokens[:i], position) + errors.append( + check_logprobs( + completion.logprobs[i], + raw.log_softmax(-1), + token, + params.logprobs, + tolerance, + position, + ) + ) + check_accuracy_budget(errors, gaps, tolerance, f"{context}, completion") + # Advance the grammar through the last token too; an incomplete JSON prefix + # is allowed at max_tokens, but every emitted token must remain valid. + reference.accept_history(tokens) + check_stopping( + tokens, + completion.finish_reason, + reference.stop_ids, + params, + context, + eos_token_id=hf.tokenizer.eos_token_id, + stop_reason=completion.stop_reason, + ) + + +def check_stopping( + tokens, + finish_reason, + stop_ids, + params, + context, + *, + eos_token_id: int | None, + stop_reason: int | str | None, +): + assert not params.ignore_eos and not params.stop, context + assert not stop_ids.intersection(tokens[:-1]), ( + f"{context}: generation continued after a stop token" + ) + if tokens[-1] in stop_ids: + assert finish_reason == "stop", ( + f"{context}: stop token did not terminate output" + ) + expected_stop_reason = None if tokens[-1] == eos_token_id else tokens[-1] + assert stop_reason == expected_stop_reason, ( + f"{context}: stop reason={stop_reason}, expected={expected_stop_reason}" + ) + else: + assert finish_reason == "length" and len(tokens) == params.max_tokens, context + assert stop_reason is None, f"{context}: length cap has a stop reason" + + +@lru_cache(maxsize=4) +def _compiler(tokenizer, vocab_size: int): + info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=vocab_size) + return xgr.GrammarCompiler(info, max_threads=1, cache_enabled=True) + + +class SamplingReference: + """One request's reference; histories must grow without changing their prefix.""" + + def __init__( + self, + tokenizer, + vocab_size: int, + generation_config: dict, + params: "SamplingParams", + ): + assert params.temperature == 0.0, "Only the original greedy matrix is supported" + assert params.presence_penalty == 0.0 and params.repetition_penalty == 1.0 + assert params.allowed_token_ids is None and not params.logit_bias + self.vocab_size = vocab_size + self.min_tokens = params.min_tokens + self.frequency_penalty = params.frequency_penalty + self.stop_ids = set(params.stop_token_ids or []) + for eos in (tokenizer.eos_token_id, generation_config.get("eos_token_id")): + if eos is not None: + self.stop_ids.update([eos] if isinstance(eos, int) else eos) + assert all(0 <= token < vocab_size for token in self.stop_ids) + + # Match public bad-word tokenization semantics without using sampler code. + self.bad_words: list[list[int]] = [] + for word in params.bad_words or []: + plain = tokenizer.encode(word.lstrip(), add_special_tokens=False) + spaced = tokenizer.encode(" " + word.lstrip(), add_special_tokens=False) + assert plain, f"Empty bad-word tokenization: {word!r}" + self.bad_words.append(plain) + if spaced and len(spaced) == len(plain) and spaced[0] != plain[0]: + self.bad_words.append(spaced) + + self.history: list[int] = [] + self.matcher = None + self.bitmask = None + if params.structured_outputs is not None: + schema = params.structured_outputs.json + assert schema is not None, "Only the original JSON schema is supported" + # Preserve dictionary order: xgrammar's default grammar orders fields. + schema = schema if isinstance(schema, str) else json.dumps(schema) + compiled = _compiler(tokenizer, vocab_size).compile_json_schema( + schema, any_whitespace=True + ) + self.matcher = xgr.GrammarMatcher( + compiled, override_stop_tokens=sorted(self.stop_ids) or None + ) + self.bitmask = xgr.allocate_token_bitmask(1, vocab_size) + + def accept_history(self, generated_ids: list[int]) -> bool: + """Validate newly generated grammar tokens, including the final EOS. + + Return whether the grammar has terminated. Use this after the final + output token; no next-token distribution exists after termination. + """ + assert generated_ids[: len(self.history)] == self.history, ( + "Each reference belongs to one monotonically growing request history" + ) + if self.matcher is not None: + for token in generated_ids[len(self.history) :]: + assert self.matcher.accept_token(token), ( + f"Grammar rejected token {token} at position {len(self.history)}" + ) + self.history.append(token) + return self.matcher.is_terminated() + else: + self.history = generated_ids.copy() + return False + + def distributions( + self, raw_logits: torch.Tensor, generated_ids: list[int] + ) -> tuple[torch.Tensor, torch.Tensor]: + """Return FP32 logits before and after non-grammar sampling transforms. + + ``raw_logits`` scores the next token after the prompt and + ``generated_ids``. Returned tensors do not alias the model's tensor. + Normalize the first result to check reported completion logprobs; + the second supplies the greedy decision reference. + """ + assert raw_logits.shape == (self.vocab_size,) + assert not self.accept_history(generated_ids), ( + "Cannot generate a distribution after grammar EOS" + ) + + grammar_logits = raw_logits.float().clone() + if self.matcher is not None: + assert self.bitmask is not None + self.matcher.fill_next_token_bitmask(self.bitmask) + words = self.bitmask[0, :, None] + allowed = ((words >> torch.arange(32)) & 1).flatten() + allowed = allowed[: self.vocab_size].bool().to(raw_logits.device) + grammar_logits.masked_fill_(~allowed, -torch.inf) + + processed = grammar_logits.clone() + for bad_word in self.bad_words: + prefix = bad_word[:-1] + if not prefix or generated_ids[-len(prefix) :] == prefix: + processed[bad_word[-1]] = -torch.inf + + if len(generated_ids) < self.min_tokens and self.stop_ids: + stop_ids = sorted(self.stop_ids) + stop_scores = processed[stop_ids].clone() + processed[stop_ids] = -torch.inf + # A completed grammar may have no continuation except a stop token. + if self.matcher is not None and torch.isneginf(processed).all(): + processed[stop_ids] = stop_scores + + if self.frequency_penalty: + for token, count in Counter(generated_ids).items(): + processed[token] -= self.frequency_penalty * count + return grammar_logits, processed diff --git a/tests/v1/e2e/general/test_async_scheduling_accuracy.py b/tests/v1/e2e/general/test_async_scheduling_accuracy.py new file mode 100644 index 0000000000..aa785b3270 --- /dev/null +++ b/tests/v1/e2e/general/test_async_scheduling_accuracy.py @@ -0,0 +1,624 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Accuracy with automatic attention selection and the model's normal dtype. + +Complements the strict Flex/FP32 invariance checks in test_async_scheduling.py. +""" + +import os +from dataclasses import replace +from typing import Any + +import pytest +import torch + +from tests.conftest import HfRunner, VllmRunner +from tests.utils import ( + large_gpu_mark, + single_gpu_only, +) +from tests.v1.e2e.general.accuracy_utils import ( + AccuracyTolerance, + check_accuracy_budget, + check_first_divergence, + check_greedy_token, + check_logprobs, + check_request_accuracy, + check_stopping, +) +from vllm import SamplingParams +from vllm.logprobs import Logprob +from vllm.outputs import CompletionOutput, RequestOutput +from vllm.platforms import current_platform +from vllm.sampling_params import StructuredOutputsParams +from vllm.utils.torch_utils import set_default_torch_num_threads +from vllm.v1.metrics.reader import Metric + +MODEL = "Qwen/Qwen3-0.6B" +EAGLE3_TARGET_MODEL = "meta-llama/Llama-3.2-1B-Instruct" + +# Need to enforce eager for MRV2 while we sort out cudagraph issues. +ENFORCE_EAGER = os.getenv("ENFORCE_EAGER", "0") == "1" + +first_prompt = ( + "The following numbers of the sequence " + + ", ".join(str(i) for i in range(10)) + + " are:" +) +example_prompts = [first_prompt, "In one word, the capital of France is "] + [ + f"Tell me about the number {i}: " for i in range(32) +] + +default_params = dict( + temperature=0.0, # greedy + max_tokens=30, + min_tokens=28, +) + + +@single_gpu_only +def test_without_spec_decoding( + vllm_runner: type[VllmRunner], + hf_runner: type[HfRunner], + sample_json_schema, +): + """Check target-model accuracy across scheduling and executor configurations.""" + struct_outputs = StructuredOutputsParams(json=sample_json_schema) + test_sampling_params: list[dict[str, Any]] = [ + dict(), + dict(frequency_penalty=-1.0), + dict(bad_words=["the", " the"]), + dict(bad_words=["the", " the"], logprobs=2), + dict(logprobs=2), + dict(logprobs=2, frequency_penalty=-1.0), + dict(prompt_logprobs=2), + dict(prompt_logprobs=2, logprobs=2), + dict(structured_outputs=struct_outputs), + dict( + structured_outputs=struct_outputs, + logprobs=2, + ), + dict( + structured_outputs=struct_outputs, + frequency_penalty=-1.0, + ), + dict( + structured_outputs=struct_outputs, + logprobs=2, + frequency_penalty=-1.0, + ), + ] + + # test_preemption, executor, async_scheduling, + # spec_config, test_prefill_chunking + test_configs = [ + (False, "mp", False, None, False), + (True, "mp", False, None, True), + (False, "mp", True, None, False), + (False, "uni", True, None, False), + (True, "mp", True, None, False), + (True, "uni", True, None, False), + (False, "mp", True, None, True), + (True, "mp", True, None, True), + (True, "uni", True, None, True), + ] + + run_tests(vllm_runner, hf_runner, MODEL, test_configs, test_sampling_params) + + +@single_gpu_only +@large_gpu_mark(min_gb=16) +def test_with_eagle3_spec_decoding( + vllm_runner: type[VllmRunner], + hf_runner: type[HfRunner], + sample_json_schema, +): + """Check accuracy and acceptance, including drafts that exceed model length.""" + spec_config = { + "method": "eagle3", + "num_speculative_tokens": 2, + "model": "nm-testing/Llama3_2_1B_speculator.eagle3", + } + # Set small draft model len to force doesn't-fit-in-drafter case. + spec_config_short = spec_config | {"max_model_len": 50} + + struct_outputs = StructuredOutputsParams(json=sample_json_schema) + + test_sampling_params: list[dict[str, Any]] = [ + dict(), + dict(frequency_penalty=-1.0), + dict(bad_words=["the", " the"]), + dict(bad_words=["the", " the"], logprobs=2), + dict(logprobs=2), + dict(logprobs=2, frequency_penalty=-1.0), + dict(prompt_logprobs=2), + dict(prompt_logprobs=2, logprobs=2), + dict(structured_outputs=struct_outputs), + dict( + structured_outputs=struct_outputs, + logprobs=2, + frequency_penalty=-1.0, + ), + ] + + # test_preemption, executor, async_scheduling, + # spec_config, test_prefill_chunking + test_configs = [ + (False, "mp", False, None, False), + (False, "mp", False, spec_config, False), + (True, "mp", False, spec_config, True), + (True, "uni", False, spec_config_short, True), + (False, "mp", True, spec_config, False), + (True, "mp", True, spec_config, False), + (False, "mp", True, spec_config_short, True), + (True, "uni", True, spec_config, False), + (True, "uni", True, spec_config_short, False), + (True, "mp", True, spec_config, True), + (True, "uni", True, spec_config_short, True), + ] + + run_tests( + vllm_runner, hf_runner, EAGLE3_TARGET_MODEL, test_configs, test_sampling_params + ) + + +@single_gpu_only +@pytest.mark.skipif( + current_platform.is_xpu(), + reason=("XPU matmul/attention kernels are not batch-invariant"), +) +def test_with_ngram_gpu_spec_decoding( + vllm_runner: type[VllmRunner], hf_runner: type[HfRunner] +): + """Check ngram accuracy and acceptance across scheduling configurations.""" + ngram_gpu_config = { + "method": "ngram_gpu", + "num_speculative_tokens": 3, + "prompt_lookup_max": 3, + "prompt_lookup_min": 2, + } + + # test_preemption, executor, async_scheduling, + # spec_config, test_prefill_chunking + test_configs = [ + (False, "mp", False, None, False), + (False, "mp", False, ngram_gpu_config, False), + (True, "mp", False, ngram_gpu_config, True), + (False, "mp", True, ngram_gpu_config, False), + (True, "mp", True, ngram_gpu_config, False), + (True, "uni", True, ngram_gpu_config, False), + (True, "mp", True, ngram_gpu_config, True), + ] + + run_tests(vllm_runner, hf_runner, MODEL, test_configs, [{}, dict(logprobs=2)]) + + +def run_tests( + vllm_runner: type[VllmRunner], + hf_runner: type[HfRunner], + model: str, + test_configs: list[tuple], + test_sampling_params: list[dict[str, Any]], +): + """Compare the shared prefix and independently validate every continuation.""" + assert test_configs[0] == (False, "mp", False, None, False), ( + "The baseline must use synchronous target-only decoding" + ) + outputs = [] + for n, config in enumerate(test_configs, 1): + outputs.append( + run_test( + vllm_runner, + model, + f"{n}/{len(test_configs)}", + test_sampling_params, + *config, + ) + ) + + baseline_config, baseline_batches, _ = outputs[0] + # Compare vLLM's own decisions at the first divergence, where histories + # still match. HF then scores each continuation, including after divergence. + # Neither engine forces an attention backend, dtype, or matmul precision. + # Qwen/Llama BF16 controls and Qwen holdouts measured max score error <0.38, + # per-stream request mean <0.073, max greedy gap 0.125, and request sum 0.25. + # Per-request budgets also reject systematic errors below the scalar bounds. + tolerance = AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25) + # These short CPU reductions cost more to distribute across a thread pool. + with set_default_torch_num_threads(1), hf_runner(model) as hf: + hf.model.eval() + for config, batches, _ in outputs: + assert len(batches) == len(test_sampling_params) + for baseline_batch, batch, overrides in zip( + baseline_batches, batches, test_sampling_params, strict=True + ): + assert len(batch) == len(example_prompts), config + params = SamplingParams(**default_params, **overrides) + for i, (baseline, request, prompt) in enumerate( + zip(baseline_batch, batch, example_prompts, strict=True) + ): + assert request.prompt_token_ids == hf.tokenizer.encode(prompt) + context = f"config=[{config}], params={overrides}, request={i}" + if params.logprobs is not None: + check_first_divergence( + baseline, + request, + params, + tolerance, + f"baseline=[{baseline_config}], {context}", + ) + check_request_accuracy( + hf, + request, + params, + tolerance, + context, + ) + print(f"ACCURACY PASSED: config=[{config}], params={overrides}") + + baseline_acceptances = next((o[2] for o in outputs if o[2] is not None), None) + if baseline_acceptances is not None: + for settings, (config, _, acceptances) in zip( + test_configs, outputs, strict=True + ): + if acceptances is None: + continue + test_preemption, _, _, spec_config, _ = settings + assert spec_config is not None + for baseline, actual, params in zip( + baseline_acceptances, acceptances, test_sampling_params, strict=True + ): + context = f"config=[{config}], params={params}" + if spec_config.get("max_model_len") is None: + # Keep the original acceptance-quality floor per batch. + relative_drop = ( + 0.10 if current_platform.is_rocm() and test_preemption else 0.05 + ) + assert actual >= baseline * (1 - relative_drop), ( + f"{context}: acceptance={actual}, baseline={baseline}" + ) + else: + assert actual > 0.1, f"{context}: acceptance={actual}" + + +def run_test( + vllm_runner: type[VllmRunner], + model: str, + test_str: str, + sampling_param_tests: list[dict[str, Any]], + test_preemption: bool, + executor: str, + async_scheduling: bool, + spec_config: dict[str, Any] | None, + test_prefill_chunking: bool, +): + spec_decoding = spec_config is not None + spec_method = (spec_config or {}).get("method", "none") + # Chunked ngram decoding admits fewer concurrent requests under the + # 48-token budget. Its original 33-block cache did not preempt on ROCm. + # A 17-block cache forces contention while every original request fits + # comfortably within 256 tokens (the longest prompt plus output is 67). + cache_blocks = 17 if test_prefill_chunking and spec_method == "ngram_gpu" else 33 + cache_arg: dict[str, Any] = ( + dict( + num_gpu_blocks_override=cache_blocks, + max_model_len=(cache_blocks - 1) * 16, + ) + if test_preemption + else dict(gpu_memory_utilization=0.9, max_model_len=4096) + ) + spec_mml = (spec_config or {}).get("max_model_len") + test_config = ( + f"executor={executor}, preemption={test_preemption}, " + f"async_sched={async_scheduling}, " + f"chunk_prefill={test_prefill_chunking}, " + f"spec_decoding={spec_decoding}, spec_method={spec_method}, spec_mml={spec_mml}" + ) + print("-" * 80) + print(f"---- TESTING {test_str}: {test_config}") + print("-" * 80) + + with vllm_runner( + model, + enable_chunked_prefill=test_prefill_chunking, + # Force prefill chunking + max_num_batched_tokens=48 if test_prefill_chunking else None, + enforce_eager=ENFORCE_EAGER, + async_scheduling=async_scheduling, + distributed_executor_backend=executor, + speculative_config=spec_config, + disable_log_stats=False, + enable_prefix_caching=False if current_platform.is_rocm() else None, + **cache_arg, + ) as vllm_model: + results = [] + acceptance_rates: list[float] | None = [] if spec_decoding else None + for override_params in sampling_param_tests: + metrics_before = vllm_model.llm.get_metrics() + print(f"----------- RUNNING PARAMS: {override_params}") + results.append( + vllm_model.llm.generate( + vllm_model.get_inputs(example_prompts), + sampling_params=SamplingParams(**default_params, **override_params), + ) + ) + metrics_after = vllm_model.llm.get_metrics() + if acceptance_rates is not None: + acceptance_rate = _get_acceptance_rate(metrics_before, metrics_after) + acceptance_rates.append(acceptance_rate) + print(f"ACCEPTANCE RATE {acceptance_rate}") + + if test_preemption: + preemptions = _get_count( + metrics_before, metrics_after, "vllm:num_preemptions" + ) + assert preemptions > 0, "preemption test had no preemptions" + + # Preserve the original parameter-effect checks across repeated batches. + if len(results) > 1: + baseline = _result_signature(results[0]) + for outputs, params in zip(results[1:], sampling_param_tests[1:], strict=True): + assert _result_signature(outputs) != baseline, ( + f"{test_config}: sampling parameters had no observable effect: {params}" + ) + # The old ROCm filter used schema-only as its first parameter set. + # Retain those comparisons after restoring the ordinary cases too. + schema_batches = [ + batch + for batch, params in zip(results, sampling_param_tests, strict=True) + if params.get("structured_outputs") is not None + ] + for batch in schema_batches[1:]: + assert _result_signature(batch) != _result_signature(schema_batches[0]), ( + f"{test_config}: structured sampling parameters had no effect" + ) + + return test_config, results, acceptance_rates + + +def _result_signature(requests): + return [ + ( + list(request.outputs[0].token_ids), + request.outputs[0].text, + request.prompt_logprobs, + request.outputs[0].logprobs, + ) + for request in requests + ] + + +def _get_acceptance_rate(before: list[Metric], after: list[Metric]) -> float: + draft = _get_count(before, after, "vllm:spec_decode_num_draft_tokens") + accept = _get_count(before, after, "vllm:spec_decode_num_accepted_tokens") + assert draft > 0, "speculative test did not draft any tokens" + assert 0 <= accept <= draft, f"invalid acceptance counters: {accept=}, {draft=}" + return accept / draft + + +def _get_count(before: list[Metric], after: list[Metric], name: str) -> int: + before_val = next(m.value for m in before if m.name == name) + after_val = next(m.value for m in after if m.name == name) + return after_val - before_val + + +@pytest.mark.parametrize( + "corruption", ["score", "rank", "top_k", "missing", "nan", "positive", "order"] +) +def test_accuracy_checks_reject_incorrect_logprobs(corruption): + second = 2.875 if corruption == "order" else 1.0 + reference = torch.tensor([3.0, second, -2.0, -5.0]).log_softmax(-1) + tolerance = AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25) + scores = { + 0: Logprob(logprob=float(reference[0]), rank=1, decoded_token="a"), + 1: Logprob(logprob=float(reference[1]), rank=2, decoded_token="b"), + } + check_logprobs(scores, reference, 0, 2, tolerance, "control") + if corruption == "score": + scores[0] = replace(scores[0], logprob=scores[0].logprob - 1.0) + elif corruption == "rank": + scores[1] = replace(scores[1], rank=1) + elif corruption == "top_k": + del scores[1] + scores[3] = Logprob(logprob=float(reference[3]), rank=2, decoded_token="d") + elif corruption == "missing": + del scores[0] + elif corruption == "positive": + scores[0] = replace(scores[0], logprob=0.1) + elif corruption == "order": + scores[0] = replace(scores[0], rank=2) + scores[1] = replace(scores[1], rank=1) + else: + scores[0] = replace(scores[0], logprob=float("nan")) + with pytest.raises(AssertionError): + check_logprobs(scores, reference, 0, 2, tolerance, corruption) + + +def test_accuracy_checks_distinguish_ties_from_wrong_tokens(): + tolerance = AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25) + logits = torch.tensor([3.0, 2.875, -2.0, -torch.inf]) + check_greedy_token(logits, 0, tolerance, "best") + check_greedy_token(logits, 1, tolerance, "near tie") + for token in (2, 3): + with pytest.raises(AssertionError): + check_greedy_token(logits, token, tolerance, "incorrect token") + + # top-k may omit a sampled token tied at its boundary. The sampled rank + # can duplicate a sequential top-k rank without corrupting the result. + reference = torch.zeros(4).log_softmax(-1) + scores = { + token: Logprob(logprob=float(reference[token]), rank=rank) + for token, rank in [(3, 1), (0, 1), (1, 2)] + } + check_logprobs(scores, reference, 3, 2, tolerance, "tied top-k") + + # A grammar with one allowed token still returns the requested top-k, + # including masked entries whose score is exactly negative infinity. + reference = torch.tensor([0.0, -torch.inf, -torch.inf]) + scores = {0: Logprob(0.0, 1), 1: Logprob(-float("inf"), 2)} + check_logprobs(scores, reference, 0, 2, tolerance, "masked top-k") + + +def test_accuracy_checks_reject_incorrect_stopping(): + params = SamplingParams(max_tokens=3) + stop_ids = {2, 3} + for tokens, reason, stop_reason in [ + ([0, 1, 2], "stop", None), + ([0, 1, 3], "stop", 3), + ([0, 1, 0], "length", None), + ]: + check_stopping( + tokens, + reason, + stop_ids, + params, + "valid stopping at length cap", + eos_token_id=2, + stop_reason=stop_reason, + ) + for tokens, reason, stop_reason in [ + ([0, 2, 1], "length", None), + ([0, 1, 2], "length", None), + ([0, 1, 2], "stop", 2), + ([0, 1, 3], "stop", None), + ([0, 1, 3], "stop", 2), + ([0, 1, 0], "length", 3), + ]: + with pytest.raises(AssertionError): + check_stopping( + tokens, + reason, + stop_ids, + params, + "incorrect stopping", + eos_token_id=2, + stop_reason=stop_reason, + ) + + +def test_accuracy_checks_reject_systematic_sub_bound_errors(): + tolerance = AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25) + reference = torch.tensor([3.0, 2.75, -2.0]).log_softmax(-1) + scores = {i: Logprob(float(reference[i]) - 0.4, i + 1) for i in (0, 1)} + # Each corrupted score fits the scalar bound, but the bias must not pass. + errors = check_logprobs(scores, reference, 0, 2, tolerance, "biased scores") + with pytest.raises(AssertionError, match="mean logprob error"): + check_accuracy_budget([errors], [], tolerance, "biased scores") + + gap = check_greedy_token(reference, 1, tolerance, "one ambiguous choice") + with pytest.raises(AssertionError, match="total greedy logit gap"): + check_accuracy_budget([], [gap] * 30, tolerance, "systematically worse choices") + check_accuracy_budget( + [torch.tensor([0.03, 0.07])], [0.125, 0.125], tolerance, "control variation" + ) + + +def _divergence_request( + tokens: list[int], scores: dict[int, float], position: int = 0 +) -> RequestOutput: + logprobs: list[dict[int, Logprob]] = [{} for _ in tokens] + logprobs[position] = { + token: Logprob(score, rank) + for rank, (token, score) in enumerate( + sorted(scores.items(), key=lambda item: item[1], reverse=True), 1 + ) + } + return RequestOutput( + request_id="test", + prompt=None, + prompt_token_ids=[0, 0, 0], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text="", + token_ids=tokens, + cumulative_logprob=None, + logprobs=logprobs, + finish_reason="length", + ) + ], + finished=True, + ) + + +@pytest.mark.parametrize( + "base_scores,actual_scores,error", + [ + ({0: -0.8, 1: -0.9}, {0: -0.9, 1: -0.8}, None), + ({0: -0.8, 1: -0.8}, {0: -0.8, 1: -0.8}, None), + ({0: -0.8, 1: -0.9}, {0: -0.8, 1: -0.9}, "actual.*below"), + ({0: -0.9, 1: -0.8}, {0: -0.9, 1: -0.8}, "baseline.*below"), + ({0: -0.8, 1: -1.1}, {0: -0.9, 1: -0.8}, "baseline.*near-tie"), + ({0: -0.8, 1: -0.9}, {0: -1.1, 1: -0.8}, "actual.*near-tie"), + ({0: -0.8, 2: -0.9}, {0: -0.9, 1: -0.8}, None), + ({0: -0.8, 2: -1.1}, {0: -0.9, 1: -0.8}, "baseline.*lower bound"), + ({0: -0.8, 1: -0.9}, {1: -0.8, 2: -1.1}, "actual.*lower bound"), + ], + ids=[ + "near-tie-rank-reversal", + "exact-tie", + "wrong-near-tie-actual-choice", + "wrong-near-tie-baseline-choice", + "decisive-baseline", + "decisive-actual", + "missing-candidate-inconclusive", + "missing-candidate-decisive-baseline", + "missing-candidate-decisive-actual", + ], +) +def test_first_divergence_checks_engine_preferences(base_scores, actual_scores, error): + baseline = _divergence_request([0], base_scores) + actual = _divergence_request([1], actual_scores) + args = ( + baseline, + actual, + SamplingParams(temperature=0, logprobs=2), + AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25), + "paired engines", + ) + if error is None: + check_first_divergence(*args) + else: + with pytest.raises(AssertionError, match=error): + check_first_divergence(*args) + + +@pytest.mark.parametrize("missing_alternative", [False, True]) +def test_first_divergence_uses_generated_penalty_history(missing_alternative): + # Raw ordering favors token 1; one generated 0 reverses the penalized order. + # The prompt contains three more 0s which must not contribute to penalties. + base_scores = {0: -1.5, 1: -0.6} + if missing_alternative: + # The selected 0 is outside top-2. Use the top-k boundary (-0.6), + # excluding the additional sampled token (-1.5), for the missing 1. + base_scores = {0: -1.5, 2: -0.5, 3: -0.6} + baseline = _divergence_request([0, 0, 2], base_scores, position=1) + actual = _divergence_request([0, 1, 3, 4], {0: -1.6, 1: -0.5}, position=1) + check_first_divergence( + baseline, + actual, + SamplingParams(temperature=0, logprobs=2, frequency_penalty=-1), + AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25), + "different later histories are validated separately", + ) + + +@pytest.mark.parametrize("mismatch", ["length", "finish_reason", "prompt"]) +def test_first_divergence_rejects_mismatches_without_a_token_change(mismatch): + baseline = _divergence_request([0, 1], {}) + actual = _divergence_request([0, 1], {}) + if mismatch == "length": + actual.outputs[0].token_ids = [0] + elif mismatch == "finish_reason": + actual.outputs[0].finish_reason = "stop" + else: + actual.prompt_token_ids = [1] + with pytest.raises(AssertionError): + check_first_divergence( + baseline, + actual, + SamplingParams(temperature=0, logprobs=2), + AccuracyTolerance(logprob_atol=0.5, greedy_atol=0.25), + mismatch, + ) From 31ec1c3aa18a4881889cf4bacd50a867a8248717 Mon Sep 17 00:00:00 2001 From: ZhenchengLin <130115215+ZhenchengLin@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:04:19 -0700 Subject: [PATCH 163/248] [Bugfix][Rust Frontend] Stop vllm-bench chat latency at the last token (#59251) Signed-off-by: zhenchenglin Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Roy Wang --- rust/src/bench/src/backends/openai_chat.rs | 79 ++++++++++++++++++++-- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/rust/src/bench/src/backends/openai_chat.rs b/rust/src/bench/src/backends/openai_chat.rs index 0842d23e6e..bf82b67878 100644 --- a/rust/src/bench/src/backends/openai_chat.rs +++ b/rust/src/bench/src/backends/openai_chat.rs @@ -96,10 +96,6 @@ impl OpenAIChatBackend { continue; } - // Python chat backend: timestamp is captured for ALL - // non-DONE messages, and most_recent_timestamp is updated - // unconditionally (outside `if choices:`). This differs from - // completions which only timestamps content chunks. let timestamp = Instant::now(); let data: ChatChunk = match serde_json::from_str(chunk) { @@ -126,6 +122,9 @@ impl OpenAIChatBackend { } generated_text.push_str(content); + // Only token chunks advance the request end; + // the trailing usage chunk carries no token. + most_recent_timestamp = timestamp; } // Separate `if` (not `else if`) — Dynamo may send // both choices and usage in the same chunk. @@ -134,8 +133,6 @@ impl OpenAIChatBackend { { output.output_tokens = ct as usize; } - - most_recent_timestamp = timestamp; } } @@ -361,6 +358,76 @@ mod tests { assert_eq!(a, b); } + /// Serve one streaming chat response over a raw socket: `num_tokens` + /// content chunks back-to-back, then `usage_delay` of silence, then the + /// choice-less usage chunk and `[DONE]`. Returns the endpoint URL. + fn spawn_sse_server(num_tokens: usize, usage_delay: std::time::Duration) -> String { + use std::io::{BufRead, BufReader, Read, Write}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut content_length = 0; + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") { + content_length = v.trim().parse().unwrap(); + } + if line == "\r\n" { + break; + } + } + reader.read_exact(&mut vec![0; content_length]).unwrap(); + + let mut send = |s: &str| { + stream.write_all(s.as_bytes()).unwrap(); + stream.flush().unwrap(); + }; + send("HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\nconnection: close\r\n\r\n"); + for _ in 0..num_tokens { + send("data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n"); + } + std::thread::sleep(usage_delay); + send(&format!( + "data: {{\"choices\":[],\"usage\":{{\"completion_tokens\":{num_tokens}}}}}\n\n" + )); + send("data: [DONE]\n\n"); + }); + format!("http://{addr}/v1/chat/completions") + } + + /// The trailing usage chunk carries no token, so it must not extend the + /// request's end: `latency - ttft` has to equal `sum(itl)`, otherwise + /// E2EL and TPOT absorb whatever gap precedes the usage chunk. Mirrors the + /// Python fix in vllm-project/vllm#55508. + #[tokio::test] + async fn test_usage_chunk_does_not_extend_latency() { + // The gap before the usage chunk is the quantity under test, so a + // real delay is needed here; there is no observable event to wait on. + let usage_delay = std::time::Duration::from_millis(150); + let input = RequestFuncInput { + api_url: spawn_sse_server(3, usage_delay), + model: "test-model".to_string(), + output_len: 3, + ..Default::default() + }; + + let output = OpenAIChatBackend.send_request(&input, &reqwest::Client::new()).await.unwrap(); + + assert!(output.success, "{}", output.error); + assert_eq!(output.output_tokens, 3); + assert_eq!(output.itl.len(), 2); + let decode_time = output.latency - output.ttft; + let itl_sum: f64 = output.itl.iter().sum(); + assert!( + (decode_time - itl_sum).abs() < 1e-6, + "latency - ttft ({decode_time}s) != sum(itl) ({itl_sum}s)" + ); + } + /// ignore_eos and extra_body must survive the raw-splice path. #[test] fn test_mm_payload_tail_fields() { From 34c458d27f512bb888d92acced70163b013ed1a5 Mon Sep 17 00:00:00 2001 From: Ausar <45146080+ArcheyChen@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:11:44 +0800 Subject: [PATCH 164/248] [Bugfix] Don't let a structured-output request sample from an unmasked row (#54442) Signed-off-by: ArcheyChen <45146080+ArcheyChen@users.noreply.github.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: GPT-5.6 Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- .../test_rejection_sampler_utils.py | 1 + .../v1/worker/test_grammar_invalid_drafts.py | 56 +++++++++++++++++++ vllm/v1/core/sched/output.py | 5 ++ vllm/v1/core/sched/scheduler.py | 10 +++- vllm/v1/worker/gpu/model_runner.py | 12 +++- vllm/v1/worker/gpu/sample/batch_shard.py | 7 +++ .../gpu/spec_decode/rejection_sampler.py | 17 +++++- vllm/v1/worker/gpu/structured_outputs.py | 37 ++++++++++++ 8 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 tests/v1/worker/test_grammar_invalid_drafts.py diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index e80f9e6d2c..f635a66a46 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -909,6 +909,7 @@ def test_verify_rejects_unproposed_drafts(): inputs["expanded_idx_mapping"], inputs["expanded_local_pos"], None, + draft_sampled, ) return sampled, num_sampled diff --git a/tests/v1/worker/test_grammar_invalid_drafts.py b/tests/v1/worker/test_grammar_invalid_drafts.py new file mode 100644 index 0000000000..77e0ef5242 --- /dev/null +++ b/tests/v1/worker/test_grammar_invalid_drafts.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Which speculative drafts a structured-output request may accept. + +`StructuredOutputManager.grammar_bitmask` fills row i before it inspects +`req_tokens[i]`, so with the first -1 placeholder at index j rows 0..j carry a +real mask and rows j+1.. carry the all-permissive `_full_mask`. Drafts j..K-1 +must therefore be rejected, or the request samples with no grammar constraint +at a position the model really does sample. +""" + +from types import SimpleNamespace + +import numpy as np +import torch + +from vllm.v1.worker.gpu.structured_outputs import grammar_invalid_drafts + + +def _input_batch(req_ids, num_drafts, num_admitted=None): + """`num_admitted` is what adaptive verification kept on device.""" + num_admitted = num_drafts if num_admitted is None else num_admitted + rows = [n + 1 for n in num_admitted] + return SimpleNamespace( + req_ids=list(req_ids), + num_reqs=len(req_ids), + num_draft_tokens=sum(num_drafts), + num_draft_tokens_per_req=np.array(num_drafts, dtype=np.int32), + cu_num_logits=torch.tensor([0, *np.cumsum(rows)], dtype=torch.int32), + expanded_local_pos=torch.cat( + [torch.arange(n, dtype=torch.int32) for n in rows] + ), + ) + + +def _invalid_rows(batch, grammar_req_ids, num_acceptable): + mask = grammar_invalid_drafts(batch, grammar_req_ids, num_acceptable) + return None if mask is None else mask.nonzero().flatten().tolist() + + +def test_reject_whole_window_when_nothing_was_backfilled(): + # Request "g" is at rows 0..3 (1 + 3 drafts), so its drafts are rows 1..3. + batch = _input_batch(["g", "p"], [3, 3]) + assert _invalid_rows(batch, ["g"], [0]) == [1, 2, 3] + + +def test_keep_the_drafts_the_bitmask_could_see(): + batch = _input_batch(["g", "p"], [3, 3]) + assert _invalid_rows(batch, ["g"], [2]) == [3] + + +def test_follow_the_device_layout_under_adaptive_verification(): + # Scheduled 3 drafts each, but adaptive verification admitted 2 for "a" and + # 3 for "b": the real rows are a=0..2, b=3..6, not the scheduled 0..3, 4..7. + batch = _input_batch(["a", "b"], [3, 3], num_admitted=[2, 3]) + assert _invalid_rows(batch, ["a", "b"], [1, 0]) == [2, 4, 5, 6] diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 0da64cf27e..9d16d2f710 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -335,3 +335,8 @@ class GrammarOutput: structured_output_request_ids: list[str] # Bitmask ordered as structured_output_request_ids. grammar_bitmask: "npt.NDArray[np.int32]" + # Per request, ordered as structured_output_request_ids: how many leading + # drafts the bitmask constrained. `grammar_bitmask` fills every row after + # the first -1 placeholder with the all-permissive `_full_mask`, so drafts + # from this index on must not be accepted. None invalidates every draft. + num_acceptable_drafts: list[int] | None = None diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 494dba5e46..fcb1421b10 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -71,6 +71,7 @@ from vllm.v1.request import Request, RequestStatus, StreamingUpdate from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup from vllm.v1.spec_decode.metrics import SpecDecodingStats from vllm.v1.structured_output import StructuredOutputManager +from vllm.v1.structured_output.utils import strip_speculative_padding from vllm.v1.utils import record_function_or_nullcontext logger = init_logger(__name__) @@ -1963,7 +1964,14 @@ class Scheduler(SchedulerInterface): structured_output_request_ids, scheduler_output.scheduled_spec_decode_tokens, ) - return GrammarOutput(structured_output_request_ids, bitmask) + spec_tokens = scheduler_output.scheduled_spec_decode_tokens + num_acceptable_drafts = [ + len(strip_speculative_padding(spec_tokens.get(req_id, []))) + for req_id in structured_output_request_ids + ] + return GrammarOutput( + structured_output_request_ids, bitmask, num_acceptable_drafts + ) def update_from_output( self, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index fed9e80d0c..ebc7a5058d 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -164,7 +164,10 @@ from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState -from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker +from vllm.v1.worker.gpu.structured_outputs import ( + StructuredOutputsWorker, + grammar_invalid_drafts, +) from vllm.v1.worker.gpu.ubatch_utils import ( UBatchRunner, UBatchState, @@ -1576,6 +1579,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): sample_hidden_states = hidden_states[input_batch.logits_indices] logits = self.model.compute_logits(sample_hidden_states) + invalid_drafts = None # A diffusion prefill has no logit rows even when a bitmask row # arrived for it. if grammar_output is not None and logits.shape[0] > 0: @@ -1587,6 +1591,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): grammar_output.structured_output_request_ids, grammar_output.grammar_bitmask, ) + invalid_drafts = grammar_invalid_drafts( + input_batch, + grammar_output.structured_output_request_ids, + grammar_output.num_acceptable_drafts, + ) sampler_output: SamplerOutput | None if input_batch.num_reqs == 0: @@ -1605,6 +1614,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): input_batch, # Draft logits are needed for probabilistic rejection sampling. self.speculator.draft_logits, + invalid_drafts, ) if shard_metadata is not None: diff --git a/vllm/v1/worker/gpu/sample/batch_shard.py b/vllm/v1/worker/gpu/sample/batch_shard.py index 3d41c49720..a7e369b188 100644 --- a/vllm/v1/worker/gpu/sample/batch_shard.py +++ b/vllm/v1/worker/gpu/sample/batch_shard.py @@ -339,9 +339,16 @@ def _shard_grammar_output( cursor += num_req_logits if not local_ids: return None + # num_acceptable_drafts is ordered with structured_output_request_ids. + num_acceptable = grammar_output.num_acceptable_drafts + if num_acceptable is not None: + owned = set(local_ids) + ids = grammar_output.structured_output_request_ids + num_acceptable = [n for i, n in zip(ids, num_acceptable) if i in owned] return GrammarOutput( structured_output_request_ids=local_ids, grammar_bitmask=grammar_output.grammar_bitmask[keep_indices], + num_acceptable_drafts=num_acceptable, ) diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index a9ad00e89f..1983126b77 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -253,6 +253,7 @@ class RejectionSampler: expanded_idx_mapping: torch.Tensor, expanded_local_pos: torch.Tensor, seq_lens_upper_bound_np: np.ndarray, + verify_draft_sampled: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: processed_logits = self.sampler.apply_sampling_params( logits, @@ -267,7 +268,7 @@ class RejectionSampler: sampled, num_sampled = rejection_sample( processed_logits, draft_logits, - draft_sampled, + verify_draft_sampled, cu_num_logits, pos, idx_mapping, @@ -294,7 +295,10 @@ class RejectionSampler: pos: torch.Tensor, max_chunk_logits: int, max_num_logprobs: int, + verify_draft_sampled: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]: + if verify_draft_sampled is None: + verify_draft_sampled = draft_sampled cu_num_logits_np = input_batch.cu_num_logits_np use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES num_reqs = input_batch.num_reqs @@ -330,6 +334,7 @@ class RejectionSampler: input_batch.expanded_idx_mapping[lo:hi], input_batch.expanded_local_pos[lo:hi], input_batch.seq_lens_cpu_upper_bound.numpy()[start:end], + verify_draft_sampled[lo:hi], ) chunk_logprobs = self._get_logprobs_tensors( sampled, @@ -368,6 +373,7 @@ class RejectionSampler: logits: torch.Tensor, input_batch: InputBatch, draft_logits: torch.Tensor | None = None, + invalid_drafts: torch.Tensor | None = None, ) -> SamplerOutput: # NOTE(woosuk): We intentionally compute num_nans before sampling to make clear # that num_nans is computed before applying penalties and temperature. @@ -382,6 +388,14 @@ class RejectionSampler: self.sampler.req_states.prefill_len.gpu, ) + # Marking a draft invalid (`is_valid_draft = draft_sampled >= 0`) pins the + # accepted length before it reaches a permissive bitmask row. Only + # verification sees the copy: `apply_sampling_params` and watermarking + # keep the real draft ids. + verify_draft_sampled = draft_sampled + if invalid_drafts is not None: + verify_draft_sampled = draft_sampled.masked_fill(invalid_drafts, -1) + max_num_logprobs = self.sampler.sampling_states.max_num_logprobs( input_batch.idx_mapping_np ) @@ -394,6 +408,7 @@ class RejectionSampler: pos, chunk_logit_limit, max_num_logprobs, + verify_draft_sampled, ) num_sampled, num_rejected = get_num_sampled_and_rejected( diff --git a/vllm/v1/worker/gpu/structured_outputs.py b/vllm/v1/worker/gpu/structured_outputs.py index 057d6f7154..336bc85861 100644 --- a/vllm/v1/worker/gpu/structured_outputs.py +++ b/vllm/v1/worker/gpu/structured_outputs.py @@ -35,6 +35,43 @@ def _build_grammar_mapping( return mapping +def grammar_invalid_drafts( + input_batch: InputBatch, + grammar_req_ids: list[str], + num_acceptable_drafts: list[int] | None, +) -> torch.Tensor | None: + """Mask over logit rows whose draft was verified against a permissive row. + + Draft i sits at local position i + 1, and drafts from `num_acceptable_drafts` + on met `_full_mask`. Local positions come from the device `cu_num_logits`, so + this stays right when adaptive verification trims drafts on device. + Returns None when there is nothing to invalidate. + """ + if not grammar_req_ids or input_batch.num_draft_tokens == 0: + return None + req_id_to_idx = {req_id: i for i, req_id in enumerate(input_batch.req_ids)} + limit = np.full(input_batch.num_reqs, np.iinfo(np.int32).max, dtype=np.int32) + for i, req_id in enumerate(grammar_req_ids): + req_idx = req_id_to_idx.get(req_id) + if req_idx is not None: + # None (an older scheduler, or warmup): invalidate the whole window. + limit[req_idx] = ( + num_acceptable_drafts[i] if num_acceptable_drafts is not None else 0 + ) + # Adaptive verification only trims drafts, so this bound holds either way. + num_drafts = input_batch.num_draft_tokens_per_req + if num_drafts is not None and (limit >= num_drafts).all(): + return None + local_pos = input_batch.expanded_local_pos + cu_num_logits = input_batch.cu_num_logits + row_limit = torch.repeat_interleave( + async_tensor_h2d(limit, device=local_pos.device), + cu_num_logits[1:] - cu_num_logits[:-1], + output_size=local_pos.shape[0], + ) + return local_pos > row_limit + + class StructuredOutputsWorker: def __init__( self, From 7314360c9eebb3b4b460ec49f6606ef0a08fcae2 Mon Sep 17 00:00:00 2001 From: LostFox11 <140548160+LostFox11@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:18:55 +0800 Subject: [PATCH 165/248] [Perf][PP] Skip sampled-token broadcasts whose requests leave the engine (#58542) Signed-off-by: LostFox11 Signed-off-by: Nick Hill Co-authored-by: LostFox11 Co-authored-by: Nick Hill --- tests/v1/worker/test_gpu_batch_ordering.py | 2 + tests/v1/worker/test_pp_utils.py | 87 +++++++++++++++++++++- vllm/v1/worker/gpu/input_batch.py | 4 + vllm/v1/worker/gpu/model_runner.py | 7 ++ vllm/v1/worker/gpu/pcp_manager.py | 1 + vllm/v1/worker/gpu/pp_utils.py | 15 ++-- vllm/v1/worker/gpu/ubatch_utils.py | 1 + 7 files changed, 111 insertions(+), 6 deletions(-) diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index db9e080e4e..498ff7a728 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -57,6 +57,7 @@ def _make_runner( runner.decode_query_len = decode_query_len runner.adaptive_verification = None runner.pcp_manager = None + runner.use_pp = False runner.req_states = SimpleNamespace( req_id_to_index={req_id: i for i, req_id in enumerate(req_states)}, # The runner keeps this as min(num_computed_tokens, prefill_len). @@ -246,6 +247,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): runner.decode_query_len = 8 runner.adaptive_verification = None runner.pcp_manager = None + runner.use_pp = False # State arrays in state-index order: a prefilling request, then two decodes. runner.req_states = SimpleNamespace( req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2}, diff --git a/tests/v1/worker/test_pp_utils.py b/tests/v1/worker/test_pp_utils.py index cfc92479f5..a6209abb97 100644 --- a/tests/v1/worker/test_pp_utils.py +++ b/tests/v1/worker/test_pp_utils.py @@ -9,12 +9,26 @@ import numpy as np from vllm.v1.worker.gpu import pp_utils -def _batch(num_computed, prefill_len, num_scheduled): +def _batch( + num_computed, + prefill_len, + num_scheduled, + is_prefilling=None, + max_seq_len=None, +): return Mock( num_reqs=len(num_computed), num_computed_tokens_np=np.array(num_computed, dtype=np.int32), prefill_len_np=np.array(prefill_len, dtype=np.int32), num_scheduled_tokens=np.array(num_scheduled, dtype=np.int32), + is_prefilling_np=( + np.array(is_prefilling, dtype=np.bool_) + if is_prefilling is not None + else None + ), + max_seq_len_np=( + np.array(max_seq_len, dtype=np.int32) if max_seq_len is not None else None + ), ) @@ -81,3 +95,74 @@ def test_decode_row_ahead_of_a_prefill_chunk(): assert mask is not None assert mask.tolist() == [True, False] + + +def test_excludes_final_prefill_chunk_reaching_its_length_cap(): + """A max_tokens=1 final chunk hands its token off and leaves the engine. + + Disaggregated prefill submits max_tokens=1, so the request finishes with + the final chunk's single sample and no follow-up step on this engine + reads the broadcast payload. + """ + batch = _batch( + num_computed=[1000], + prefill_len=[1004], + num_scheduled=[4], + is_prefilling=[True], + max_seq_len=[1005], + ) + + assert pp_utils.compute_need_sampled_mask(batch) is None + + +def test_keeps_final_prefill_chunk_with_generation_budget_left(): + """The same final chunk, but the request keeps decoding here.""" + batch = _batch( + num_computed=[1000], + prefill_len=[1004], + num_scheduled=[4], + is_prefilling=[True], + max_seq_len=[1004 + 256], + ) + + mask = pp_utils.compute_need_sampled_mask(batch) + + assert mask is not None + assert mask.tolist() == [True] + + +def test_length_cap_overrun_decode_row_is_kept(): + """Decoding rows are never excluded, even past their length cap. + + Speculative decoding can transiently overrun prompt_len + max_tokens + while the scheduler still runs the request; dropping the row would + desynchronize the PP stages. + """ + batch = _batch( + num_computed=[14176], + prefill_len=[12175], + num_scheduled=[8], + is_prefilling=[False], + max_seq_len=[12176], + ) + + mask = pp_utils.compute_need_sampled_mask(batch) + + assert mask is not None + assert mask.tolist() == [True] + + +def test_finishing_prefill_row_does_not_drop_a_sibling_row(): + """Only finishing rows are excluded; a decoding sibling keeps its row.""" + batch = _batch( + num_computed=[1000, 10], + prefill_len=[1004, 8], + num_scheduled=[4, 1], + is_prefilling=[True, False], + max_seq_len=[1005, 1024], + ) + + mask = pp_utils.compute_need_sampled_mask(batch) + + assert mask is not None + assert mask.tolist() == [False, True] diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 1fc3737837..e2f68ad598 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -103,6 +103,9 @@ class InputBatch: # [num_reqs] per-request prompt length, only populated for R-SWA. prompt_lens: torch.Tensor | None + # [num_reqs] only populated for non-dummy pipeline-parallel batches. + max_seq_len_np: np.ndarray | None = None + # Longest query the batch may contain. Set when a cudagraph descriptor promises # a query length this batch's own split does not reach, so attention metadata # stays valid for every replay the graph serves. @@ -203,6 +206,7 @@ class InputBatch: num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), has_prefill=False, + max_seq_len_np=None, decode_graph_eligible=True, input_ids=input_ids, positions=positions, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index ebc7a5058d..14ff11f448 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1274,6 +1274,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): ] is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np has_prefill = bool(is_prefilling_np.any()) + max_seq_len_np = None + if self.use_pp and has_prefill: + # consumed by the PP `compute_need_sampled_mask` for batches with prefill. + max_seq_len_np = self.req_states.max_seq_len[idx_mapping_np] num_draft_tokens_np = None if draft_tokens: @@ -1312,6 +1316,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): has_prefill=has_prefill, prefill_runs_as_decode_np=prefill_runs_as_decode_np, decode_graph_eligible=decode_graph_eligible, + max_seq_len_np=max_seq_len_np, ) return batch_state, get_uniform_decode_token_count( num_reqs, num_toks, max_query_len, decode_graph_eligible @@ -1499,6 +1504,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): has_prefill=batch_req_state.has_prefill, prefill_runs_as_decode_np=batch_req_state.prefill_runs_as_decode_np, decode_graph_eligible=batch_req_state.decode_graph_eligible, + max_seq_len_np=batch_req_state.max_seq_len_np, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], is_padding=is_padding, @@ -2392,6 +2398,7 @@ class BatchReqState(NamedTuple): prefill_len_np: np.ndarray # [num_reqs] num_computed_prefill_tokens_np: np.ndarray # [num_reqs] is_prefilling_np: np.ndarray # [num_reqs] + max_seq_len_np: np.ndarray | None # [num_reqs], None in non-pp case has_prefill: bool prefill_runs_as_decode_np: np.ndarray | None # [num_reqs] decode_graph_eligible: bool diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index f740e4caf3..89e6b1064b 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -649,6 +649,7 @@ class PCPManager: prefill_len_np=local_prefill_len_np, num_computed_prefill_tokens_np=local_num_computed_prefill_tokens_np, is_prefilling_np=local_is_prefilling_np, + max_seq_len_np=None, has_prefill=local_has_prefill, decode_graph_eligible=not local_has_prefill, prefill_runs_as_decode_np=None, diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index 0a246c98ce..818d80759a 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -37,14 +37,19 @@ class PendingRecv: def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None: """Return a bool array of shape `[input_batch.num_reqs]` marking requests - that produce a sampled token this step, and therefore must have that token - (and the draft block proposed from it) propagated to the earlier PP stages. - Returns None if no request in the batch produces a sample.""" + with outputs that might be needed in a subsequent (decode) step. + Returns None if no sampled outputs are needed in the requests' next step.""" old_computed = input_batch.num_computed_tokens_np prefill_len = input_batch.prefill_len_np # Exclude non-final prefill chunks (they don't produce a sample). - produces_sample = old_computed + input_batch.num_scheduled_tokens >= prefill_len - return produces_sample if produces_sample.any() else None + need_sampled_mask = old_computed + input_batch.num_scheduled_tokens >= prefill_len + if input_batch.max_seq_len_np is not None: + # Also exclude final prefill chunks whose single sampled token reaches + # the request's length cap. + finished_prefill = prefill_len + 1 >= input_batch.max_seq_len_np + finished_prefill &= input_batch.is_prefilling_np + need_sampled_mask &= ~finished_prefill + return need_sampled_mask if need_sampled_mask.any() else None class PPHandler: diff --git a/vllm/v1/worker/gpu/ubatch_utils.py b/vllm/v1/worker/gpu/ubatch_utils.py index 6889103ed1..3fa9783bf5 100644 --- a/vllm/v1/worker/gpu/ubatch_utils.py +++ b/vllm/v1/worker/gpu/ubatch_utils.py @@ -194,6 +194,7 @@ def _slice_input_batch( if input_batch.prompt_lens is None else input_batch.prompt_lens[req_start:req_stop] ), + max_seq_len_np=None, fast_prefill=None, ) From 76d3929916c6594c11924e9f91adbc69b90cbd35 Mon Sep 17 00:00:00 2001 From: Nils Matteson Date: Wed, 30 Sep 2026 19:20:17 -0700 Subject: [PATCH 166/248] [Docker] Compile Python bytecode at image build time (#55422) Signed-off-by: Nils Matteson Co-authored-by: Simon Mo Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docker/Dockerfile | 1 + docker/Dockerfile.cpu | 2 ++ docker/Dockerfile.ppc64le | 1 + docker/Dockerfile.rock | 1 + docker/Dockerfile.rocm | 1 + docker/Dockerfile.rocm_gfx1250 | 1 + docker/Dockerfile.s390x | 1 + docker/Dockerfile.xpu | 1 + 8 files changed, 9 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index ba046adae1..7dfcd61b9e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -906,6 +906,7 @@ ENV UV_INDEX_STRATEGY="unsafe-best-match" ENV UV_LINK_MODE=copy ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python ENV UV_CACHE_DIR=/opt/uv/cache +ENV UV_COMPILE_BYTECODE=1 RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" \ && chgrp -R 0 /opt/uv \ && chmod -R g+rwX,a+rX /opt/uv diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index ed5dfa5d24..f61752475d 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -138,6 +138,8 @@ ENV TARGETARCH=${TARGETARCH} ######################### COMMON BASE IMAGE + PYTHON DEPS ######################### FROM base-common AS base-common-pydeps +ENV UV_COMPILE_BYTECODE=1 + # PyTorch provides its own indexes for standard and nightly builds. # PYTORCH_NIGHTLY=1 builds the CPU image against torch nightly (parity with the # GPU torch-nightly lane); default 0 keeps the pinned release CPU build. diff --git a/docker/Dockerfile.ppc64le b/docker/Dockerfile.ppc64le index 1bb08591ca..68e6ab9563 100644 --- a/docker/Dockerfile.ppc64le +++ b/docker/Dockerfile.ppc64le @@ -22,6 +22,7 @@ ENV HOME=/root \ RUSTUP_HOME=/root/.rustup \ UV_CACHE_DIR=$HOME/.cache/uv \ PATH=/root/.cargo/bin:/root/.rustup/bin:${VIRTUAL_ENV}/bin:$PATH +ENV UV_COMPILE_BYTECODE=1 RUN echo "DEBUG: VLLM_VERSION=${VLLM_VERSION}" RUN --mount=type=cache,target=/var/cache/dnf \ diff --git a/docker/Dockerfile.rock b/docker/Dockerfile.rock index afeac31742..e478d8e5a9 100644 --- a/docker/Dockerfile.rock +++ b/docker/Dockerfile.rock @@ -898,6 +898,7 @@ COPY --from=test_smoke /vllm-smoke-ok / # ----------------------- # Final vLLM image FROM mori_base AS final_common +ENV UV_COMPILE_BYTECODE=1 RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 74b007e5e7..8ba942346b 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -920,6 +920,7 @@ COPY --from=test_smoke /vllm-smoke-ok / # ----------------------- # Final vLLM image FROM mori_base AS final_common +ENV UV_COMPILE_BYTECODE=1 RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rocm_gfx1250 b/docker/Dockerfile.rocm_gfx1250 index ac6dcdff0a..0609c73c04 100644 --- a/docker/Dockerfile.rocm_gfx1250 +++ b/docker/Dockerfile.rocm_gfx1250 @@ -655,6 +655,7 @@ RUN mkdir src && mv vllm src/vllm # ----------------------- # Final vLLM image FROM mori_base AS final +ENV UV_COMPILE_BYTECODE=1 RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.s390x b/docker/Dockerfile.s390x index 06be3ad523..e04ff55ce6 100644 --- a/docker/Dockerfile.s390x +++ b/docker/Dockerfile.s390x @@ -196,6 +196,7 @@ ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu" ENV LD_LIBRARY_PATH="/opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/torch/lib:/usr/local/lib:/opt/rh/gcc-toolset-14/root/usr/lib64:$LD_LIBRARY_PATH" ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH" ENV UV_LINK_MODE=copy +ENV UV_COMPILE_BYTECODE=1 ENV CARGO_HOME=/root/.cargo ENV RUSTUP_HOME=/root/.rustup ENV GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 diff --git a/docker/Dockerfile.xpu b/docker/Dockerfile.xpu index 9a33f6ba38..ada261dd8f 100644 --- a/docker/Dockerfile.xpu +++ b/docker/Dockerfile.xpu @@ -205,6 +205,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ /tmp/ucx_source /tmp/nixl_source FROM vllm-base AS vllm-openai +ENV UV_COMPILE_BYTECODE=1 # Must match the ucx-nixl-build stage above: ARG does not cross stages, so a stale # value here installs a meta package that disagrees with the wheel built there. From edaf3e7b613599449fbd2f2ac065869e7e4d358a Mon Sep 17 00:00:00 2001 From: Venky <23023424+venkywonka@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:20:33 -0700 Subject: [PATCH 167/248] [Attention][SM120] Occupancy-adaptive Triton split-K segment count (#58482) Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: vadiklyutiy <156319763+vadiklyutiy@users.noreply.github.com> --- .../v1/attention/test_triton_attn_segments.py | 73 +++++++++++++++++++ vllm/v1/attention/backends/triton_attn.py | 44 ++++++++++- .../attention/backends/triton_attn_diffkv.py | 2 +- 3 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 tests/v1/attention/test_triton_attn_segments.py diff --git a/tests/v1/attention/test_triton_attn_segments.py b/tests/v1/attention/test_triton_attn_segments.py new file mode 100644 index 0000000000..874c61d237 --- /dev/null +++ b/tests/v1/attention/test_triton_attn_segments.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata +from vllm.config import CUDAGraphMode +from vllm.platforms import current_platform +from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder +from vllm.v1.attention.backends.triton_attn_diffkv import ( + TritonAttentionDiffKVMetadataBuilder, +) +from vllm.v1.kv_cache_interface import FullAttentionSpec + + +# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and +# the 64-segment scratch holds every query token (e.g. multi-query verify). +@pytest.mark.parametrize( + "builder_cls", + [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder], +) +@pytest.mark.parametrize( + "capability,num_kv_heads,num_seqs,query_len,segments", + [ + ((12, 0), 1, 11, 1, 64), + ((12, 0), 1, 12, 1, 16), + ((12, 0), 8, 1, 1, 64), + ((12, 0), 8, 2, 1, 16), + ((9, 0), 1, 1, 1, 16), + ((12, 0), 1, 2, 4, 64), + ((12, 0), 1, 10, 4, 16), + ], +) +def test_split_k_segments_follow_sm_occupancy( + monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments +): + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + monkeypatch.setattr( + current_platform, "is_device_capability", lambda cap: cap == capability + ) + monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188) + config = SimpleNamespace( + model_config=SimpleNamespace( + get_num_attention_heads=lambda _: 8, + get_num_kv_heads=lambda _: num_kv_heads, + get_head_size=lambda: 128, + rswa_window=None, + ), + parallel_config=None, + scheduler_config=SimpleNamespace(max_num_seqs=16), + speculative_config=None, + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={} + ), + ) + spec = FullAttentionSpec( + block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16 + ) + builder = builder_cls(spec, ["layer.0"], config, "cpu") + batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs) + metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu")) + assert metadata.num_par_softmax_segments == segments + assert metadata.softmax_segm_output.shape[2] == segments + assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len + # DiffKV re-allocates the output; it must stay sized like the base buffers. + assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0] + assert ( + metadata.softmax_segm_output.data_ptr() + == builder.softmax_segm_output.data_ptr() + ) diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 8a2fab4cc7..d8577b61f1 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -18,7 +18,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability -from vllm.utils.math_utils import next_power_of_2 +from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.utils.torch_utils import get_dtype_size, is_quantized_kv_cache from vllm.v1.attention.backend import ( AttentionBackend, @@ -152,10 +152,29 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet ) self.num_par_softmax_segments = NUM_PAR_SOFTMAX_SEGMENTS + # On SM120, batches whose 16-segment grid under-fills the SMs use 64. + self.max_seqs_64_segments = 0 + if current_platform.is_cuda() and current_platform.is_device_capability( + (12, 0) + ): + self.max_seqs_64_segments = (current_platform.num_compute_units() - 1) // ( + NUM_PAR_SOFTMAX_SEGMENTS * self.num_heads_kv + ) + max_num_tokens_3d = self.seq_threshold_3D + if self.max_seqs_64_segments > 0: + self.num_par_softmax_segments = 64 + # build() reuses this scratch at 16 segments with proportionally more rows. + max_num_tokens_3d = max( + min(self.max_seqs_64_segments, max_num_tokens_3d), + cdiv( + max_num_tokens_3d, + self.num_par_softmax_segments // NUM_PAR_SOFTMAX_SEGMENTS, + ), + ) headdim_padded = next_power_of_2(self.headdim) self.softmax_segm_output = torch.empty( ( - self.seq_threshold_3D, + max_num_tokens_3d, self.num_heads_q, self.num_par_softmax_segments, headdim_padded, @@ -164,12 +183,12 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet device=device, ) self.softmax_segm_max = torch.empty( - (self.seq_threshold_3D, self.num_heads_q, self.num_par_softmax_segments), + (max_num_tokens_3d, self.num_heads_q, self.num_par_softmax_segments), dtype=torch.float32, device=device, ) self.softmax_segm_expsum = torch.empty( - (self.seq_threshold_3D, self.num_heads_q, self.num_par_softmax_segments), + (max_num_tokens_3d, self.num_heads_q, self.num_par_softmax_segments), dtype=torch.float32, device=device, ) @@ -246,6 +265,23 @@ class TritonAttentionMetadataBuilder(AttentionMetadataBuilder[TritonAttentionMet softmax_segm_max=self.softmax_segm_max, softmax_segm_expsum=self.softmax_segm_expsum, ) + if self.max_seqs_64_segments > 0 and ( + self.max_seqs_64_segments < seq_lens.shape[0] + or self.softmax_segm_max.shape[0] < num_actual_tokens + ): + # The 16-segment grid fills the SMs, or the 64-segment rows cannot + # hold every query token: same scratch, 16-segment layout. + segments = NUM_PAR_SOFTMAX_SEGMENTS + attn_metadata.num_par_softmax_segments = segments + attn_metadata.softmax_segm_output = self.softmax_segm_output.view( + -1, self.num_heads_q, segments, self.softmax_segm_output.shape[-1] + ) + attn_metadata.softmax_segm_max = self.softmax_segm_max.view( + -1, self.num_heads_q, segments + ) + attn_metadata.softmax_segm_expsum = self.softmax_segm_expsum.view( + -1, self.num_heads_q, segments + ) mm_ranges = common_attn_metadata.mm_req_doc_ranges if mm_ranges is not None: diff --git a/vllm/v1/attention/backends/triton_attn_diffkv.py b/vllm/v1/attention/backends/triton_attn_diffkv.py index ddbd066206..4b42838a15 100644 --- a/vllm/v1/attention/backends/triton_attn_diffkv.py +++ b/vllm/v1/attention/backends/triton_attn_diffkv.py @@ -56,7 +56,7 @@ class TritonAttentionDiffKVMetadataBuilder(TritonAttentionMetadataBuilder): head_size_v_padded = next_power_of_2(head_size_v) self.softmax_segm_output = torch.empty( ( - self.seq_threshold_3D, + self.softmax_segm_max.shape[0], self.num_heads_q, self.num_par_softmax_segments, head_size_v_padded, From 766b4ae54c491497306224ee4b316d9c2d57cd30 Mon Sep 17 00:00:00 2001 From: Duncan Moss Date: Wed, 30 Sep 2026 19:28:50 -0700 Subject: [PATCH 168/248] [Kimi-K3] Add FlashInfer speculative KDA backend (#54255) Signed-off-by: Duncan Moss Co-authored-by: Roger Wang Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Thien Tran --- tests/models/kimi_k3/test_kda.py | 241 ++++++++++++++++++++++++++++++ vllm/models/kimi_k3/nvidia/kda.py | 139 +++++++++++++++-- vllm/utils/flashinfer.py | 12 ++ 3 files changed, 383 insertions(+), 9 deletions(-) diff --git a/tests/models/kimi_k3/test_kda.py b/tests/models/kimi_k3/test_kda.py index eaf4b69b40..88acdb6197 100644 --- a/tests/models/kimi_k3/test_kda.py +++ b/tests/models/kimi_k3/test_kda.py @@ -30,13 +30,17 @@ from vllm.models.kimi_k3.amd.ops.third_party.kda import ( ) from vllm.models.kimi_k3.nvidia import kda as nvidia_kda from vllm.models.kimi_k3.nvidia.kda import ( + KimiK3DeltaAttention, _flashinfer_kda_prefill, _flashkda_prefill, is_flashinfer_fused_kda_decode_supported, + is_flashinfer_fused_kda_spec_decode_supported, is_flashinfer_recurrent_kda_prefill_supported, is_flashkda_supported, is_fused_kda_decode_supported, + resolve_kda_spec_decode_backend, ) +from vllm.models.kimi_k3.nvidia.kda_metadata import KimiK3KDAMetadata from vllm.models.kimi_k3.nvidia.model import KimiLinearForCausalLM from vllm.models.kimi_k3.nvidia.ops import recoverssm as recoverssm_ops from vllm.models.kimi_k3.nvidia.ops.recoverssm import ( @@ -90,6 +94,25 @@ def test_kda_warmup_skips_missing_metadata(monkeypatch): assert layer._forward(empty, empty, empty, empty, empty) is None +def test_resolve_kda_spec_decode_backend(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + "vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported", + lambda *args: True, + ) + args = (12, 128, 4, 6, torch.bfloat16, torch.bfloat16, torch.float32, -5.0) + assert resolve_kda_spec_decode_backend("auto", *args, False) == "flashinfer" + assert resolve_kda_spec_decode_backend("native", *args, False) == "native" + assert resolve_kda_spec_decode_backend("flashinfer", *args, False) == "flashinfer" + + monkeypatch.setattr( + "vllm.models.kimi_k3.nvidia.kda.is_flashinfer_fused_kda_spec_decode_supported", + lambda *args: False, + ) + assert resolve_kda_spec_decode_backend("auto", *args, False) == "native" + with pytest.raises(RuntimeError, match="packed_fused_kda_decode"): + resolve_kda_spec_decode_backend("flashinfer", *args, False) + + def test_kda_recoverssm_config_state_layout(): vllm_config = SimpleNamespace( model_config=SimpleNamespace( @@ -1182,6 +1205,224 @@ def test_fused_kda_decode_correctness( torch.testing.assert_close(state_actual, state_ref, atol=3e-2, rtol=3e-2) +@torch.inference_mode() +@pytest.mark.parametrize("state_dtype", [torch.float32, torch.bfloat16]) +def test_flashinfer_fused_kda_spec_decode_integration( + monkeypatch: pytest.MonkeyPatch, + state_dtype: torch.dtype, +): + if not is_flashinfer_fused_kda_spec_decode_supported( + num_heads=12, + head_dim=128, + conv_width=4, + num_spec=6, + input_dtype=torch.bfloat16, + conv_state_dtype=torch.bfloat16, + recurrent_state_dtype=state_dtype, + lower_bound=-5.0, + use_recoverssm=False, + ): + pytest.skip("FlashInfer packed fused KDA decode is not supported") + + generator = torch.Generator(device=DEVICE).manual_seed(20260820) + H, D, W, T = 12, 128, 4, 7 + hidden_size = H * D + num_rows = 10 + num_slots = 16 + + x = torch.randn( + num_rows, + 3 * hidden_size, + dtype=torch.bfloat16, + device=DEVICE, + generator=generator, + ) + conv_weight = 0.1 * torch.randn( + 3 * hidden_size, + W, + dtype=torch.float32, + device=DEVICE, + generator=generator, + ) + fused_weight = conv_weight.view(3, hidden_size, W).transpose(1, 2).contiguous() + raw_gate = torch.randn( + 1, num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator + ) + raw_beta = torch.randn( + 1, num_rows, H, dtype=torch.bfloat16, device=DEVICE, generator=generator + ) + output_gate = torch.randn( + num_rows, H, D, dtype=torch.bfloat16, device=DEVICE, generator=generator + ) + A_log = torch.log(torch.arange(1, H + 1, dtype=torch.float32, device=DEVICE)) + dt_bias = torch.empty(hidden_size, dtype=torch.float32, device=DEVICE).uniform_( + -7.0, -4.0, generator=generator + ) + norm_weight = torch.randn( + D, dtype=torch.float32, device=DEVICE, generator=generator + ) + norm_eps = 1e-5 + state_indices = torch.tensor( + [ + list(range(1, T + 1)), + list(range(T + 1, 2 * T + 1)), + [0] * T, + ], + dtype=torch.int32, + device=DEVICE, + ) + query_start_loc = torch.tensor([0, 7, 10, 10], dtype=torch.int32, device=DEVICE) + num_accepted_tokens = torch.tensor([1, 4, 1], dtype=torch.int32, device=DEVICE) + conv_seed = ( + 0.5 + * torch.randn( + num_slots, + T + W - 2, + 3 * hidden_size, + dtype=torch.bfloat16, + device=DEVICE, + generator=generator, + ) + ).transpose(1, 2) + state_seed = ( + 0.5 + * torch.randn( + num_slots, + H, + D, + D, + dtype=torch.float32, + device=DEVICE, + generator=generator, + ) + ).to(state_dtype) + metadata = KimiK3KDAMetadata( + num_prefills=0, + num_prefill_tokens=0, + num_decodes=0, + num_decode_tokens=0, + num_spec_decodes=2, + num_spec_decode_tokens=num_rows, + num_actual_tokens=num_rows, + spec_query_start_loc=query_start_loc, + spec_state_indices_tensor=state_indices, + num_accepted_tokens=num_accepted_tokens, + ) + monkeypatch.setattr( + "vllm.models.kimi_k3.nvidia.kda.get_forward_context", + lambda: SimpleNamespace(attn_metadata={"test.layer": metadata}), + ) + + def clone_strided(tensor: torch.Tensor) -> torch.Tensor: + clone = torch.empty_strided( + tensor.shape, tensor.stride(), dtype=tensor.dtype, device=tensor.device + ) + clone.copy_(tensor) + return clone + + def composed_reference( + conv_state: torch.Tensor, + recurrent_state: torch.Tensor, + ) -> torch.Tensor: + mixed_qkv = causal_conv1d_update( + x, + conv_state, + conv_weight, + bias=None, + activation="silu", + conv_state_indices=state_indices[:, 0], + num_accepted_tokens=num_accepted_tokens, + query_start_loc=query_start_loc, + max_query_len=T, + validate_data=False, + out=torch.empty_like(x), + ) + q, k, v = mixed_qkv.view(num_rows, 3, H, D).unbind(1) + recurrent_out, _ = fused_recurrent_kda_nvidia( + q=q.unsqueeze(0), + k=k.unsqueeze(0), + v=v.unsqueeze(0), + raw_g=raw_gate, + raw_beta=raw_beta, + A_log=A_log, + dt_bias=dt_bias, + lower_bound=-5.0, + initial_state=recurrent_state, + cu_seqlens=query_start_loc, + ssm_state_indices=state_indices, + num_accepted_tokens=num_accepted_tokens, + ) + recurrent_float = recurrent_out.float() + return ( + recurrent_float + * torch.rsqrt( + recurrent_float.square().mean(dim=-1, keepdim=True) + norm_eps + ) + * norm_weight + * output_gate.float().sigmoid().unsqueeze(0) + ).to(torch.bfloat16) + + def run_flashinfer( + conv_state: torch.Tensor, + recurrent_state: torch.Tensor, + output: torch.Tensor, + ) -> torch.Tensor: + layer = SimpleNamespace( + prefix="test.layer", + kv_cache=(conv_state.transpose(-1, -2), recurrent_state), + kda_spec_decode_backend="flashinfer", + decode_conv1d_weight=fused_weight, + decode_norm_weight=norm_weight, + gate_lower_bound=-5.0, + A_log=A_log, + dt_bias=dt_bias, + o_norm=SimpleNamespace(eps=norm_eps), + ) + KimiK3DeltaAttention._forward( + layer, + mixed_qkv=x, + g1=raw_gate, + g2=output_gate, + beta=raw_beta, + core_attn_out=output, + ) + return output + + expected_conv = clone_strided(conv_seed) + expected_state = state_seed.clone() + expected = composed_reference(expected_conv, expected_state) + actual_conv = clone_strided(conv_seed) + actual_state = state_seed.clone() + output = torch.empty_like(expected) + actual = run_flashinfer(actual_conv, actual_state, output) + + assert actual.data_ptr() == output.data_ptr() + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2) + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=3e-2) + assert torch.count_nonzero(actual[:, :7]) > 0 + torch.testing.assert_close(actual_state[0], state_seed[0], atol=0, rtol=0) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = run_flashinfer(actual_conv, actual_state, output) + + actual_conv.copy_(conv_seed) + actual_state.copy_(state_seed) + query_start_loc.copy_(torch.tensor([0, 6, 10, 10], device=DEVICE)) + num_accepted_tokens.copy_(torch.tensor([3, 2, 1], device=DEVICE)) + expected_conv.copy_(conv_seed) + expected_state.copy_(state_seed) + expected = composed_reference(expected_conv, expected_state) + graph.replay() + torch.accelerator.synchronize() + + assert captured.data_ptr() == output.data_ptr() + torch.testing.assert_close(output, expected, atol=2e-2, rtol=3e-2) + torch.testing.assert_close(actual_conv, expected_conv, atol=0, rtol=0) + torch.testing.assert_close(actual_state, expected_state, atol=2e-3, rtol=3e-2) + + def test_fused_kda_decode_rejects_speculative_conv_state(): assert not is_fused_kda_decode_supported( num_heads=12, diff --git a/vllm/models/kimi_k3/nvidia/kda.py b/vllm/models/kimi_k3/nvidia/kda.py index 97b4029084..26fe7cb5b4 100644 --- a/vllm/models/kimi_k3/nvidia/kda.py +++ b/vllm/models/kimi_k3/nvidia/kda.py @@ -55,8 +55,10 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig from vllm.utils.flashinfer import ( flashinfer_fused_kda_decode, + flashinfer_packed_fused_kda_decode, flashinfer_recurrent_kda, has_flashinfer_fused_kda_decode, + has_flashinfer_packed_fused_kda_decode, has_flashinfer_recurrent_kda, ) from vllm.v1.attention.backend import AttentionBackend @@ -214,6 +216,73 @@ def is_flashinfer_fused_kda_decode_supported( ) +def is_flashinfer_fused_kda_spec_decode_supported( + num_heads: int, + head_dim: int, + conv_width: int, + num_spec: int, + input_dtype: torch.dtype, + conv_state_dtype: torch.dtype, + recurrent_state_dtype: torch.dtype, + lower_bound: float | None, + use_recoverssm: bool, +) -> bool: + return ( + has_flashinfer_packed_fused_kda_decode() + and current_platform.is_device_capability_family(100) + and num_heads in (12, 24, 32, 48, 96) + and head_dim == 128 + and conv_width == 4 + and 1 <= num_spec <= 7 + and input_dtype == torch.bfloat16 + and conv_state_dtype == torch.bfloat16 + and recurrent_state_dtype in (torch.float32, torch.bfloat16) + and lower_bound is not None + and lower_bound < 0 + and not use_recoverssm + and not is_conv_state_dim_first() + ) + + +def resolve_kda_spec_decode_backend( + backend: str, + num_heads: int, + head_dim: int, + conv_width: int, + num_spec: int, + input_dtype: torch.dtype, + conv_state_dtype: torch.dtype, + recurrent_state_dtype: torch.dtype, + lower_bound: float | None, + use_recoverssm: bool, +) -> str: + if backend not in ("auto", "native", "flashinfer"): + raise ValueError(f"Unsupported KDA speculative decode backend: {backend}") + supported = is_flashinfer_fused_kda_spec_decode_supported( + num_heads, + head_dim, + conv_width, + num_spec, + input_dtype, + conv_state_dtype, + recurrent_state_dtype, + lower_bound, + use_recoverssm, + ) + if backend == "flashinfer" and not supported: + raise RuntimeError( + "FlashInfer packed fused KDA decode requires its " + "packed_fused_kda_decode API, CUDA SM10x, bfloat16 inputs and " + "convolution state, float32 or bfloat16 recurrent state, D=128, " + "W=4, a bounded gate, 1-7 speculative tokens, a supported head " + "count, the SD convolution-state layout, and RecoverSSM disabled." + ) + if supported and backend != "native": + logger.info_once("Using FlashInfer packed fused KDA speculative decode.") + return "flashinfer" + return "native" + + def resolve_kda_decode_backend( backend: str, num_heads: int, @@ -537,6 +606,13 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): self.projection_size = self.head_dim * self.num_heads self.local_projection_size = divide(self.projection_size, self.tp_size) self.conv_size = kda_config["short_conv_kernel_size"] + self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None) + if self.gate_lower_bound is not None: + assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, ( + "KDA gate lower bound must be in " + f"[{_KDA_GATE_LOGBOUND_MIN}, 0). " + f"Got {self.gate_lower_bound}." + ) assert kda_config.get("use_full_rank_gate", False), ( "KimiK3DeltaAttention requires a full-rank gate" ) @@ -618,8 +694,28 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): conv_state_dtype, recurrent_state_dtype, ) + spec_decode_backend = ( + additional_config.get("kda_spec_decode_backend", "auto") + if isinstance(additional_config, dict) + else "auto" + ) + self.kda_spec_decode_backend = resolve_kda_spec_decode_backend( + spec_decode_backend, + self.local_num_heads, + self.head_dim, + self.conv_size, + self.num_spec, + vllm_config.model_config.dtype, + conv_state_dtype, + recurrent_state_dtype, + self.gate_lower_bound, + self.use_recoverssm, + ) decode_conv1d_weight = None - if self.kda_decode_backend != "triton": + if ( + self.kda_decode_backend != "triton" + or self.kda_spec_decode_backend == "flashinfer" + ): decode_conv1d_weight = torch.empty( 3, self.conv_size, @@ -648,14 +744,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): ) set_weight_attrs(self.A_log, {"weight_loader": a_log_weight_loader(0)}) - self.gate_lower_bound: float | None = kda_config.get("gate_lower_bound", None) - if self.gate_lower_bound is not None: - assert _KDA_GATE_LOGBOUND_MIN <= self.gate_lower_bound < 0, ( - "KDA gate lower bound must be in " - f"[{_KDA_GATE_LOGBOUND_MIN}, 0). " - f"Got {self.gate_lower_bound}." - ) - backend = ( additional_config.get("kda_prefill_backend", "auto") if isinstance(additional_config, dict) @@ -860,6 +948,39 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): if not is_conv_state_dim_first(): conv_state = conv_state.transpose(-1, -2) + if ( + self.kda_spec_decode_backend == "flashinfer" + and has_spec_decode + and m.num_prefills == 0 + and m.num_decodes == 0 + ): + assert self.decode_conv1d_weight is not None + assert self.decode_norm_weight is not None + assert spec_state_indices_tensor is not None + assert spec_query_start_loc is not None + assert num_accepted_tokens is not None + assert self.gate_lower_bound is not None + # "Packed" is FlashInfer's name for the varlen T>1 speculative path. + flashinfer_packed_fused_kda_decode( + x=mixed_qkv, + weight=self.decode_conv1d_weight, + conv_state=conv_state, + raw_gate=g1, + raw_beta=beta, + A_log=self.A_log, + dt_bias=self.dt_bias, + state_indices=spec_state_indices_tensor, + query_start_loc=spec_query_start_loc, + num_accepted_tokens=num_accepted_tokens, + state=recurrent_state, + output_gate=g2[:num_actual_tokens], + norm_weight=self.decode_norm_weight, + lower_bound=self.gate_lower_bound, + norm_eps=self.o_norm.eps, + output=core_attn_out[:, :num_actual_tokens], + ) + return + if ( self.kda_decode_backend != "triton" and self.decode_conv1d_weight is not None diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index 44da20a3ed..d7fa807c33 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -313,6 +313,9 @@ flashinfer_xqa_batch_decode_with_kv_cache = _lazy_import_wrapper( "flashinfer.decode", "xqa_batch_decode_with_kv_cache", ) +flashinfer_packed_fused_kda_decode = _lazy_import_wrapper( + "flashinfer", "packed_fused_kda_decode" +) flashinfer_recurrent_kda = _lazy_import_wrapper( "flashinfer.kda", "recurrent_kda", @@ -337,6 +340,15 @@ def has_flashinfer_comm() -> bool: return has_flashinfer() and importlib.util.find_spec("flashinfer.comm") is not None +@functools.cache +def has_flashinfer_packed_fused_kda_decode() -> bool: + """Return whether FlashInfer's packed fused KDA decode API is available.""" + if not has_flashinfer(): + return False + module = _get_submodule("flashinfer") + return bool(module and callable(getattr(module, "packed_fused_kda_decode", None))) + + @functools.cache def has_flashinfer_nvlink_two_sided() -> bool: """Return `True` if FlashInfer mnnvl all2all is available.""" From bc505fc823b29739554e40c348d12b4f032f8369 Mon Sep 17 00:00:00 2001 From: Hanjie Qiu <50634613+hjjq@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:30:06 -0400 Subject: [PATCH 169/248] [Bugfix] Support CuTe DSL 4.8.0 block-scale API (#59480) Signed-off-by: hjjq <50634613+hjjq@users.noreply.github.com> Co-authored-by: Codex Co-authored-by: Shang Wang --- vllm/cute_utils/_tcgen05.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vllm/cute_utils/_tcgen05.py b/vllm/cute_utils/_tcgen05.py index 4ab75cc0cc..028ce3155c 100644 --- a/vllm/cute_utils/_tcgen05.py +++ b/vllm/cute_utils/_tcgen05.py @@ -148,6 +148,11 @@ def mma_mxfp8( loc=None, ip=None, ) -> None: + # CuTe DSL 4.8 renamed the block-scale enum and keyword. + if hasattr(nvvm, "Tcgen05MMABlockScale"): + scale_kwargs = {"block_scale": nvvm.Tcgen05MMABlockScale.DEFAULT} + else: + scale_kwargs = {"scale_vec_size": nvvm.Tcgen05MMAScaleVecSize.X1} with cute.arch.elect_one(): nvvm.tcgen05_mma_block_scale( nvvm.Tcgen05MMAKind.MXF8F6F4, @@ -159,9 +164,9 @@ def mma_mxfp8( Boolean(enable_input_d).ir_value(loc=loc, ip=ip), _make_tmem_llvm_ptr(sfa_tmem, loc=loc, ip=ip), _make_tmem_llvm_ptr(sfb_tmem, loc=loc, ip=ip), - scale_vec_size=nvvm.Tcgen05MMAScaleVecSize.X1, loc=loc, ip=ip, + **scale_kwargs, ) From 9e6550bd058560bec5690b6c06474adf048a9b85 Mon Sep 17 00:00:00 2001 From: Nils Matteson Date: Wed, 30 Sep 2026 19:34:28 -0700 Subject: [PATCH 170/248] [Bugfix] Register IR providers before hashing configuration (#58756) Signed-off-by: Nils Matteson --- tests/config/test_config_utils.py | 22 ++++++++++++++++++++++ vllm/config/kernel.py | 3 +++ 2 files changed, 25 insertions(+) diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index 1d0f4d915b..b08c1987c4 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -168,6 +168,28 @@ def test_classes_are_types(): assert endswith_fqname(LocalDummy, ".LocalDummy") +def test_ir_config_hash_initializes_provider_and_tracks_implementation(monkeypatch): + from types import SimpleNamespace + + from vllm.config.kernel import IrOpPriorityConfig + from vllm.ir.op import IrOp + from vllm.platforms import current_platform + + op = IrOp.registry["rms_norm"] + implementation = SimpleNamespace(uuid=lambda: "first-implementation") + + def import_ir_kernels(): + monkeypatch.setitem(op.impls, "test_lazy_provider", implementation) + + monkeypatch.setattr(current_platform, "import_ir_kernels", import_ir_kernels) + config = IrOpPriorityConfig(rms_norm=["test_lazy_provider"]) + first_hash = config.compute_hash() + assert config.compute_hash() == first_hash + + implementation.uuid = lambda: "changed-implementation" + assert config.compute_hash() != first_hash + + def test_envs_compile_factors_stable(): """Test that envs.compile_factors() hash is stable across fresh initializations. diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index ea0b7d60d1..a42ad62047 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -48,6 +48,9 @@ class IrOpPriorityConfig: # Implementations are hidden from Dynamo, # so they don't show up in the traced files list. from vllm.ir.op import IrOp + from vllm.platforms import current_platform + + current_platform.import_ir_kernels() assert "_impls" not in factors factors["_impls"] = { From bfbcde26d5c86475d478b6b1abc9a1285d3753d0 Mon Sep 17 00:00:00 2001 From: Ning Xie Date: Thu, 1 Oct 2026 10:54:33 +0800 Subject: [PATCH 171/248] [Observability] add model initializing duration log (#52141) Signed-off-by: Andy Xie --- vllm/model_executor/model_loader/base_loader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index 7712efbfd1..2b7cb72fff 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time from abc import ABC, abstractmethod import torch @@ -70,11 +71,17 @@ class BaseModelLoader(ABC): target_device = torch.device(load_device) with set_default_torch_dtype(model_config.dtype): with target_device: + time_before_load = time.perf_counter() model = self.create_model( vllm_config=vllm_config, model_config=model_config, prefix=prefix, ) + time_after_load = time.perf_counter() + logger.info_once( + "Initializing model took %.6f seconds", + time_after_load - time_before_load, + ) logger.debug("Loading weights on %s ...", load_device) self.load_weights(model, model_config) From a37c402f32a7ef035c4b9d3ddcb629f8b8abedb5 Mon Sep 17 00:00:00 2001 From: Ning Xie Date: Thu, 1 Oct 2026 11:07:31 +0800 Subject: [PATCH 172/248] [weight loader] add dtype equality validation between parameter and weight (#51792) Signed-off-by: Andy Xie --- vllm/model_executor/models/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index c6498319cd..bbb7a54b55 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -321,6 +321,14 @@ class AutoWeightsLoader: f"into a single parameter {base_prefix!r}" ) + if param.dtype != weight_data.dtype: + logger.warning( + "Attempted to load weight %s with dtype %s into " + "parameter with dtype %s", + weight_qualname, + weight_data.dtype, + param.dtype, + ) weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader(param, weight_data) From 2eaa3bc5ac03aa5c0b3282782fd89eb92e074bfb Mon Sep 17 00:00:00 2001 From: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:13:13 -0400 Subject: [PATCH 173/248] [Frontend] Enrich chat_parsing streaming events (#58603) Give serving adapters the delimiter spans and opener captures they need, and keep streaming after a region that fails to parse. Signed-off-by: Yoni Gozlan Co-authored-by: Cursor Fix mypy index types in the event-stream test. --- tests/parser/test_chat_parsing.py | 105 ++++++++++++++++++++ vllm/parser/chat_parsing/response_parser.py | 70 +++++++++---- 2 files changed, 157 insertions(+), 18 deletions(-) diff --git a/tests/parser/test_chat_parsing.py b/tests/parser/test_chat_parsing.py index 26336a540a..679d97f15b 100644 --- a/tests/parser/test_chat_parsing.py +++ b/tests/parser/test_chat_parsing.py @@ -1160,6 +1160,111 @@ class ResponseEventStreamTest(unittest.TestCase): self.assertEqual(result, {"role": "assistant"}) self.assertEqual(final_events, []) + def test_region_events_expose_delimiter_offsets(self): + spec = { + "start_anchor": "", + "fields": { + "content": {"content_args": {"strip": False}}, + "tag": {"open_pattern": r"", "close": ""}, + }, + } + parser = ResponseParser(spec, prefix="") + events = parser.feed("plainbodytail") + events += parser.finalize()[1] + + boundaries = [ + (event["type"], event["field"], parser.input_text[event["start"] : event["end"]]) + for event in events + if event["type"] != "region_chunk" + ] + self.assertEqual( + boundaries, + [ + ("region_open", "content", ""), + ("region_close", "content", ""), + ("region_open", "tag", ""), + ("region_close", "tag", ""), + ("region_open", "content", ""), + ("region_close", "content", ""), + ], + ) + + def test_open_captures_and_prefix_boundary(self): + spec = { + "start_anchor": "[BEGIN]", + "fields": { + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "transform": {"name": "{name}", "arguments": "{content}"}, + } + }, + } + parser = ResponseParser(spec, prefix="history[BEGIN]{"city":"Paris"}') + opening = events[0] + + self.assertEqual(prefix_end, len(""))) + self.assertEqual(opening["captures"], {"name": "get_weather"}) + + def test_held_prefix_bytes_are_not_rechunked(self): + spec = {"start_anchor": "[BEGIN]", "fields": {"content": {}, "tag": {"open": "", "close": ""}}} + parser = ResponseParser(spec, prefix="[BEGIN]Existing <") + chunks = [event["text"] for event in parser.feed("b> tail") if event["type"] == "region_chunk"] + message, _ = parser.finalize() + + self.assertEqual(chunks, ["b> tail"]) + self.assertEqual(message["content"], "Existing tail") + + def test_malformed_region_events_recover_and_continue(self): + spec = { + "start_anchor": "[BEGIN]", + "fields": { + "tool_calls": { + "open_pattern": r"\w+)>", + "close": "", + "content": "json", + "repeats": True, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + } + }, + } + parser = ResponseParser(spec, prefix="") + events = parser.feed('{"x":{"x":1}') + message, _ = parser.finalize() + opening, malformed = events[0], events[2] + + self.assertEqual(malformed["type"], "region_malformed") + self.assertIsInstance(malformed["error"], ValueError) + self.assertEqual(parser.input_text[opening["end"] : malformed["start"]], '{"x":') + self.assertEqual(parser.input_text[malformed["start"] : malformed["end"]], "") + self.assertEqual( + message["tool_calls"], + [{"type": "function", "function": {"name": "good", "arguments": {"x": 1}}}], + ) + + parser = ResponseParser(spec, prefix="") + parser.feed('{"x":') + _, events = parser.finalize() + self.assertEqual(events[-1]["type"], "region_malformed") + self.assertEqual(events[-1]["start"], len(parser.input_text)) + + def test_parse_response_raises_for_malformed_region(self): + spec = { + "start_anchor": "<|assistant|>", + "fields": {"x": {"open": "", "close": "", "content": "json"}}, + } + with self.assertRaisesRegex(ValueError, "could not parse region as JSON"): + parse_response('{"name":', spec, prefix="") + with self.assertRaisesRegex(ValueError, "could not parse region as JSON"): + parse_response("", spec, prefix='<|assistant|>{"name":') + class PrefixAndTruncationTest(unittest.TestCase): def test_prefix_lands_inside_explicit_region(self): diff --git a/vllm/parser/chat_parsing/response_parser.py b/vllm/parser/chat_parsing/response_parser.py index b739572759..556df803a7 100644 --- a/vllm/parser/chat_parsing/response_parser.py +++ b/vllm/parser/chat_parsing/response_parser.py @@ -85,8 +85,11 @@ def parse_response( """ response_template = load_response_template(response_template) stream = ResponseParser(response_template, prefix=prefix, tools=tools) - stream.feed(text) - message, _ = stream.finalize() + events = stream.feed(text) + message, final_events = stream.finalize() + for event in stream.initial_events + events + final_events: + if event["type"] == "region_malformed": + raise event["error"] return message @@ -110,7 +113,12 @@ class ResponseParser: Pass OpenAI-style `tools=` dictionaries to cast tool-call arguments using the calling tool's JSON schema as each region closes. - Events can be either "region_open", "region_chunk", or "region_close". + Events can be "region_open", "region_chunk", "region_close", or "region_malformed". + Open, close and malformed events carry `start` and `end`, the span of `input_text` the + event consumed: its opening or closing delimiter, empty for implicit boundaries. + Explicit opens also carry the opener's named `captures`. A region whose value fails to + parse ends with "region_malformed" instead of "region_close", carrying the `error`, and + parsing continues after it. ResponseParser requires the chat `prefix` (i.e. the chat history, the prefill before the current generation). This is because chat templates or assistant prefills can sometimes write part of the message, and if we @@ -155,10 +163,18 @@ class ResponseParser: self._body: str = "" self._opened: bool = False self._finalized: bool = False + self._prefix_len: int = 0 + self._malformed_fields: set[str] = set() self.initial_events: list[dict] = [] if prefix: self._consume_prefix(prefix) + @property + def input_text(self) -> str: + """Raw parser input after start-anchor truncation. Before the first `feed()`, this is + exactly the truncated prefix, so its length marks where generated text begins.""" + return self._buffer + def _consume_prefix(self, prefix: str) -> None: """Loads the prefix (the chat prefill sent to the model), right-truncates it to the start of the assistant message (as determined by start_anchor) and then runs the remainder through the parser. @@ -172,6 +188,7 @@ class ResponseParser: return self._buffer = truncated self._process(self.initial_events, eos=False) + self._prefix_len = len(truncated) def feed(self, text: str) -> list[dict]: """Feeds more text/tokens from the model output into the tokenizer, and returns any events that result @@ -197,7 +214,11 @@ class ResponseParser: raise RuntimeError("ResponseParser already finalized") events: list[dict] = [] self._process(events, eos=True) - missing = [n for n, f in self._spec.fields.items() if not f.optional and n not in self._output] + missing = [ + n + for n, f in self._spec.fields.items() + if not f.optional and n not in self._output and n not in self._malformed_fields + ] if missing: raise ValueError(f"Required response_template fields missing from parsed output: {missing}") defaults = self._spec.defaults @@ -216,12 +237,12 @@ class ResponseParser: self._accumulate(events, self._buffer[self._pos : m.start()]) self._pos = m.end() if kind == "open": - self._close_current(events) + self._close_current(events, m.start(), m.start()) self._open_explicit(events, field, m) else: # "close" (always the implicit region's close here, # since explicit regions only expose their own close) had_content = self._opened - self._close_current(events) + self._close_current(events, m.start(), m.end()) # Zero-width close on an already-empty region would just # re-fire next iteration -- bail out to make progress. if not had_content and m.start() == m.end(): @@ -233,7 +254,7 @@ class ResponseParser: if self._pos < len(self._buffer): self._accumulate(events, self._buffer[self._pos :]) self._pos = len(self._buffer) - self._close_current(events) + self._close_current(events, self._pos, self._pos) break # Stream everything up to the earliest still-pending delimiter. When # nothing is pending `hold_start == len(self._buffer)`, so this flushes @@ -341,20 +362,25 @@ class ResponseParser: return field = self._spec.fields[self._current] if not self._opened: - events.append({"type": "region_open", "field": self._current}) + events.append({"type": "region_open", "field": self._current, "start": self._pos, "end": self._pos}) self._opened = True self._body += text - dirty = field.content not in STREAMABLE_PARSERS - events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty}) + # Prefix bytes held back as a possible delimiter belong to the prompt, not the chunk. + text = text[max(0, self._prefix_len - self._pos) :] + if text: + dirty = field.content not in STREAMABLE_PARSERS + events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty}) def _open_explicit(self, events: list[dict], field: ResponseTemplateField, m: Any) -> None: self._current = field.name self._captures = {k: v for k, v in m.groupdict().items() if v is not None} self._body = "" self._opened = True - events.append({"type": "region_open", "field": field.name}) + events.append( + {"type": "region_open", "field": field.name, "start": m.start(), "end": m.end(), "captures": self._captures} + ) - def _close_current(self, events: list[dict]) -> None: + def _close_current(self, events: list[dict], close_start: int, close_end: int) -> None: """Close the current region and reset to the implicit/null region. Skipped (aside from the reset) when the current region never opened -- avoids vacuous open/close pairs at every explicit boundary.""" @@ -362,22 +388,30 @@ class ResponseParser: self._reset_to_implicit() return field = self._spec.fields[self._current] - value = process_field(self._body, field, self._captures) - if self._tool_params: - value = self._coerce_tool_calls(value) - if field.join is not None: - if not isinstance(value, str): + try: + value = process_field(self._body, field, self._captures) + if self._tool_params: + value = self._coerce_tool_calls(value) + if field.join is not None and not isinstance(value, str): raise ValueError( f"Field '{field.name}': 'join' requires each match to parse to a string, " f"got {type(value).__name__}." ) + except (KeyError, TypeError, ValueError) as error: + self._malformed_fields.add(self._current) + events.append( + {"type": "region_malformed", "field": self._current, "start": close_start, "end": close_end, "error": error} + ) + self._reset_to_implicit() + return + if field.join is not None: previous = self._output.get(self._current) self._output[self._current] = value if previous is None else previous + field.join + value elif field.repeats: self._output.setdefault(self._current, []).append(value) else: self._output[self._current] = value - events.append({"type": "region_close", "field": self._current, "value": value}) + events.append({"type": "region_close", "field": self._current, "start": close_start, "end": close_end, "value": value}) self._reset_to_implicit() def _coerce_tool_calls(self, value: Any) -> Any: From 4ed438b79ceaff5fe5d6d4a98b09bfb8718cceaa Mon Sep 17 00:00:00 2001 From: namgyu-youn Date: Thu, 1 Oct 2026 12:40:55 +0900 Subject: [PATCH 174/248] [Perf][Qwen4Exp] Add SM100 low-latency decode GEMM plans (#59214) Signed-off-by: namgyu-youn Co-authored-by: Thien Tran --- tests/kernels/test_bf16_skinny_gemm.py | 30 ++++++++ .../qwen4_exp/nvidia/low_latency_gemm.py | 69 +++++++++++++++++++ 2 files changed, 99 insertions(+) diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index d452354e5f..412e772a2b 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -85,6 +85,12 @@ QWEN4_EXP_SM90_CASES = [ for num_tokens, config in plans.items() ] +QWEN4_EXP_SM100_CASES = [ + (n, k, num_tokens, config) + for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS.items() + for num_tokens, config in plans.items() +] + EXPECTED_CUTE_CONFIGS = { (3072, 7168, 1): (224, 3, 4, 8), (3072, 7168, 2): (128, 3, 2, 8), @@ -717,6 +723,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None: "capability,expected_plans", [ ((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS), + ((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS), ((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS), ((8, 0), {}), ], @@ -909,6 +916,29 @@ def test_qwen4_exp_sm90_selected_shapes( assert cosine > 0.999 +@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM100_CASES) +def test_qwen4_exp_sm100_selected_shapes( + n: int, + k: int, + num_tokens: int, + config: SkinnyGemmConfig, +) -> None: + _require_capability_and_cute((10, 0)) + torch.manual_seed(42 + num_tokens) + x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda") + weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") + + selected = qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS[(n, k)][num_tokens] + assert selected == config + output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight) + + reference = torch.nn.functional.linear(x, weight) + cosine = torch.nn.functional.cosine_similarity( + output.float().flatten(), reference.float().flatten(), dim=0 + ).item() + assert cosine > 0.999 + + def test_glm52_q_b_nonpacked_single_row_falls_back() -> None: _require_sm103_and_cute() spec = glm52_gemm.GLM52_Q_B_PROJECTION diff --git a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py index c177f07745..f34f0b6335 100644 --- a/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py +++ b/vllm/models/qwen4_exp/nvidia/low_latency_gemm.py @@ -144,6 +144,73 @@ QWEN4_EXP_SM90_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = } +# B200 plans; the winning configs differ from the B300 table. +QWEN4_EXP_SM100_GEMM_PLANS: dict[tuple[int, int], dict[int, SkinnyGemmConfig]] = { + # GDN fused B/A projection, TP=4. + (24, 2560): { + 1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 2, k_unroll=4, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 128, 1, k_unroll=2, vector_width=4, static_k=2560), + }, + # Shared-expert fused gate/up projection, TP=4. + (320, 2560): { + 1: SkinnyGemmConfig(1, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 160, 1, k_unroll=2, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 16: SkinnyGemmConfig(16, 128, 1, vector_width=4, static_k=2560), + }, + # Final HC down projection, replicated in a TP=4 deployment. + (320, 10240): { + 1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240), + 2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240), + 4: SkinnyGemmConfig(4, 256, 1, k_unroll=2), + 8: SkinnyGemmConfig(8, 256, 1), + }, + # HC merged down/injection projection, replicated in a TP=4 deployment. + (336, 10240): { + 1: SkinnyGemmConfig(1, 256, 1, k_unroll=4, static_k=10240), + 2: SkinnyGemmConfig(2, 256, 1, k_unroll=4, static_k=10240), + 8: SkinnyGemmConfig(8, 256, 3, static_k=10240), + }, + # QSA indexer Q/K projection, replicated in a TP=4 deployment. + (640, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + 8: SkinnyGemmConfig(8, 128, 1, k_unroll=4, vector_width=4, static_k=2560), + }, + # GDN and QSA output projections, TP=4. + (2560, 1536): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=2, vector_width=4, static_k=1536), + 2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=1536), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=2, static_k=1536), + }, + # QSA fused QKV/gate projection, TP=4. + (3584, 2560): { + 1: SkinnyGemmConfig(1, 128, 4, k_unroll=2, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 128, 4, k_unroll=2, vector_width=4, static_k=2560), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560), + }, + # GDN fused QKVZ projection, TP=4. + (4096, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, vector_width=4, static_k=2560), + 2: SkinnyGemmConfig(2, 64, 2, k_unroll=2, static_k=2560), + 4: SkinnyGemmConfig(4, 64, 2, k_unroll=4, vector_width=4, static_k=2560), + }, + # LM head, TP=4. + (62080, 2560): { + 1: SkinnyGemmConfig(1, 128, 2, k_unroll=4, vector_width=4), + }, +} + + +def _is_sm100() -> bool: + return current_platform.is_device_capability((10, 0)) + + def _is_sm103() -> bool: return current_platform.is_device_capability((10, 3)) @@ -155,6 +222,8 @@ def _is_sm90() -> bool: def _gemm_plans() -> dict[tuple[int, int], dict[int, SkinnyGemmConfig]]: if _is_sm103(): return QWEN4_EXP_GEMM_PLANS + if _is_sm100(): + return QWEN4_EXP_SM100_GEMM_PLANS if _is_sm90(): return QWEN4_EXP_SM90_GEMM_PLANS return {} From 4c2d277643e217344056e1d2c42115d5f005912f Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Wed, 30 Sep 2026 20:54:44 -0700 Subject: [PATCH 175/248] [Perf][DSv4.1] Faster Engram host lookups: sorted rows, inline big lookups (#59327) Signed-off-by: Juntian Liu Co-authored-by: Claude Opus 5.5 (1M context) --- tests/distributed/test_engram_dp_shard.py | 29 +- tests/kernels/test_engram.py | 50 +-- vllm/models/deepseek_v41/amd/model.py | 6 +- vllm/models/deepseek_v41/common/engram.py | 512 +++++++++++++++++++++- vllm/models/deepseek_v41/nvidia/engram.py | 497 --------------------- vllm/models/deepseek_v41/nvidia/model.py | 9 +- 6 files changed, 547 insertions(+), 556 deletions(-) delete mode 100644 vllm/models/deepseek_v41/nvidia/engram.py diff --git a/tests/distributed/test_engram_dp_shard.py b/tests/distributed/test_engram_dp_shard.py index 84dd62851e..0158b342f3 100644 --- a/tests/distributed/test_engram_dp_shard.py +++ b/tests/distributed/test_engram_dp_shard.py @@ -39,10 +39,10 @@ from vllm.distributed.parallel_state import ( initialize_model_parallel, ) from vllm.forward_context import set_forward_context -from vllm.models.deepseek_v41.common.engram import EngramLayout -from vllm.models.deepseek_v41.nvidia import engram as engram_ops -from vllm.models.deepseek_v41.nvidia.engram import ( +from vllm.models.deepseek_v41.common import engram as engram_ops +from vllm.models.deepseek_v41.common.engram import ( Engram, + EngramLayout, ParallelEngramEmbedding, engram_head_shard_rank, gather_engram_hashes, @@ -686,11 +686,30 @@ def test_engram_dp_shared_memory_runtime_requirements( ) -def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch): - """A /dev/shm smaller than the tables must fall back instead of failing startup.""" +@pytest.mark.parametrize("query_fails", [False, True]) +def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails): + """A /dev/shm that is too small or cannot be queried must fall back, and the + leader must still broadcast that decision so its peers do not hang.""" + broadcasts = [] + + def broadcast(obj): + broadcasts.append(obj) + return obj + + def query(*args, **kwargs): + raise OSError("statvfs failed") + monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2) + monkeypatch.setattr( + engram_ops, + "get_engram_dp_group", + lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast), + ) + if query_fails: + monkeypatch.setattr(engram_ops, "check_shm_free_space", query) layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM) assert not engram_ops.can_share_engram_tables(layout) + assert len(broadcasts) == 1 and broadcasts[0] is not None @pytest.mark.parametrize( diff --git a/tests/kernels/test_engram.py b/tests/kernels/test_engram.py index 9f62b0d438..15bdc9162c 100644 --- a/tests/kernels/test_engram.py +++ b/tests/kernels/test_engram.py @@ -9,13 +9,10 @@ import torch from vllm.models.deepseek_v41.common import engram as engram_ops from vllm.models.deepseek_v41.common.engram import ( - Engram as CommonEngram, -) -from vllm.models.deepseek_v41.common.engram import ( + Engram, NgramHashState, + ParallelEngramEmbedding, ) -from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops -from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding from vllm.platforms import current_platform @@ -111,7 +108,7 @@ def test_fused_engram_post_wkv_matches_reference( ) monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: tp_rank) - module = CommonEngram.__new__(CommonEngram) + module = Engram.__new__(Engram) torch.nn.Module.__init__(module) module.dim = dim module.hc_mult = hc_mult @@ -121,6 +118,7 @@ def test_fused_engram_post_wkv_matches_reference( module.embed_tokens = torch.nn.Identity() # `forward` reads rows staged by `prepare_embeddings`, so inject kv there. module.embed_tokens.tp_size = module.embed_tokens.n_hash_cols = 1 + module.embed_tokens.dp_size = 1 module.staged_rows = kv.unsqueeze(1) if use_sequence_parallel: padded = torch.nn.functional.pad(kv, (0, 0, 0, (-num_kv_tokens) % tp_size)) @@ -592,9 +590,8 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch monkeypatch.setattr( engram_ops, "get_tensor_model_parallel_world_size", lambda: tp_size ) - monkeypatch.setattr(nvidia_engram_ops, "get_engram_dp_size", lambda: dp_size) + monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size) monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0) - monkeypatch.setattr(nvidia_engram_ops, "get_tensor_model_parallel_rank", lambda: 0) with pytest.raises(AssertionError, match="ranks without hash heads"): ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads) @@ -602,10 +599,19 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch @pytest.mark.skipif( not current_platform.is_cuda_alike(), reason="CUDA or ROCm required" ) -@pytest.mark.parametrize("cpu_offload", [False, True]) +@pytest.mark.parametrize( + "cpu_offload,sort_rows", [(False, False), (True, False), (True, True)] +) @pytest.mark.parametrize("tp_size", [1, 2, 4, 8]) -def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch): - """Keep complete buckets and reconstruct head order, including TP padding.""" +def test_engram_head_shards_reconstruct_checkpoint( + cpu_offload, sort_rows, tp_size, monkeypatch +): + """Keep complete buckets and reconstruct head order, including TP padding, + whether or not the host lookup sorts its rows by table offset.""" + if sort_rows: + monkeypatch.setattr( + engram_ops, "_engram_lookup_thresholds", lambda _: (0, None) + ) head_sizes = (17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73) num_rows, dim = sum(head_sizes), 64 torch.manual_seed(0) @@ -796,10 +802,8 @@ def test_engram_shared_prefetch_stream_waits_per_layer(): @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required") -@pytest.mark.parametrize( - "backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)] -) -def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload): +@pytest.mark.parametrize("cpu_offload", [None, False]) +def test_engram_constructor_honors_offload(monkeypatch, cpu_offload): """Default offload uses pinned storage and the given stream; False uses HBM.""" from vllm.config import EngramConfig @@ -817,23 +821,18 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload): else EngramConfig(cpu_offload=cpu_offload), scheduler_config=SimpleNamespace(max_num_batched_tokens=8), ) - offloaded = backend == "nvidia" and vllm_config.engram_config.cpu_offload - if backend == "common": - vllm_config.engram_config = None + offloaded = vllm_config.engram_config.cpu_offload monkeypatch.setattr(engram_ops, "get_current_vllm_config", lambda: vllm_config) - monkeypatch.setattr( - nvidia_engram_ops, "get_current_vllm_config", lambda: vllm_config - ) monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1) monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0) monkeypatch.setattr( engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity() ) stream = torch.cuda.Stream() - kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {} with torch.device("cuda"): - cls = Engram if backend == "nvidia" else CommonEngram - module = cls(config, None, layout, 0, False, "engram", **kwargs) + module = Engram( + config, None, layout, 0, False, "engram", prefetch_stream=stream + ) layer = module.embed_tokens if offloaded: assert layer.weight.device.type == "cpu" and layer.weight.is_pinned() @@ -897,7 +896,8 @@ def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_depende "The allocator reuse negative control requires the native allocator" ) - def start(self, hash_ids, rows, stream): + def start(self, hash_ids, rows): + stream = self._prefetch_stream if missing_dependency != "producer_wait": stream.wait_stream(torch.cuda.current_stream()) if missing_dependency != "record_stream": diff --git a/vllm/models/deepseek_v41/amd/model.py b/vllm/models/deepseek_v41/amd/model.py index 61c8d5de30..cff72f12dc 100644 --- a/vllm/models/deepseek_v41/amd/model.py +++ b/vllm/models/deepseek_v41/amd/model.py @@ -65,13 +65,9 @@ from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.ubatching import dbo_current_ubatch_id -from ..common.engram import EngramLayout, NgramHashState +from ..common.engram import Engram, EngramLayout, NgramHashState from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask -# Engram host offload and its prefetch stream are neither ROCm- nor -# NVIDIA-specific, so they are imported rather than duplicated. -from ..nvidia.engram import Engram - if typing.TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata diff --git a/vllm/models/deepseek_v41/common/engram.py b/vllm/models/deepseek_v41/common/engram.py index 8cd93547e6..8452c1ad8a 100644 --- a/vllm/models/deepseek_v41/common/engram.py +++ b/vllm/models/deepseek_v41/common/engram.py @@ -34,23 +34,41 @@ chunk-by-chunk while an n-gram at position ``p`` needs the token ids at cache for the rest. """ +import ctypes +import mmap +import os +import tempfile import weakref +from contextlib import ExitStack import numpy as np import torch from torch import nn +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig, get_current_vllm_config from vllm.distributed import ( + get_dp_group, + get_engram_dp_group, + get_engram_dp_size, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather, ) +from vllm.distributed.device_communicators.shm_broadcast import ( + SHM_PATH, + check_shm_free_space, +) +from vllm.distributed.parallel_state import GroupCoordinator +from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.utils import set_weight_attrs +from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.platform_utils import is_uva_available +from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor logger = init_logger(__name__) @@ -58,6 +76,18 @@ logger = init_logger(__name__) DEAD_ID = -1 +def _engram_lookup_thresholds(device: torch.device) -> tuple[int | None, int | None]: + """Host lookup thresholds on `device`: the rows from which to sort, and the + tokens (max over DP ranks) from which to run inline. None disables that path.""" + if current_platform.is_cuda() and current_platform.is_device_capability_family( + 100, device.index or 0 + ): + # Measured on GB200. + return 36864, 4096 + # TODO: Add hand-tuned thresholds for other GPU families. + return None, None + + def _is_prime(n: int) -> bool: """Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import).""" if n < 2: @@ -583,12 +613,14 @@ def _engram_head_shard_weight_loader( "ids_stride_t", "ids_stride_h", "GRID", + "SORTED", ] ) def _engram_lookup_kernel( weight, scales, ids, + sorted_dst, out, vocab_start, vocab_end, @@ -602,27 +634,37 @@ def _engram_lookup_kernel( QUANT_BLOCK: tl.constexpr, BLOCK_R: tl.constexpr, GRID, + SORTED, ): """Gather fp8 rows, apply their ue8m0 block scales, write bf16. Only this rank's heads are read; padded heads write zeros for all-gather. - `weight`/`scales` may address pinned host memory through UVA. + `weight`/`scales` may address pinned host memory through UVA. If SORTED, + `ids` holds rows relative to vocab_start in table order and `sorted_dst` + their output rows. """ cols = tl.arange(0, DIM) scale_cols = cols // QUANT_BLOCK for base in tl.range(tl.program_id(0) * BLOCK_R, num_rows, GRID * BLOCK_R): rows = base + tl.arange(0, BLOCK_R) valid = rows < num_rows - head = HEAD_START + rows % LOCAL_HEADS - token = (rows // LOCAL_HEADS).to(tl.int64) - index = tl.load( - ids + token * ids_stride_t + head * ids_stride_h, - mask=valid & (head < TOTAL_HEADS), - other=-1, - ).to(tl.int64) - owned = valid & (head < TOTAL_HEADS) - owned &= (index >= vocab_start) & (index < vocab_end) - local = tl.where(owned, index - vocab_start, 0) + if SORTED: + local = tl.load(ids + rows, mask=valid, other=-1).to(tl.int64) + dst = tl.load(sorted_dst + rows, mask=valid, other=0).to(tl.int32) + owned = (local >= 0) & (local < vocab_end - vocab_start) + else: + head = HEAD_START + rows % LOCAL_HEADS + token = (rows // LOCAL_HEADS).to(tl.int64) + index = tl.load( + ids + token * ids_stride_t + head * ids_stride_h, + mask=valid & (head < TOTAL_HEADS), + other=-1, + ).to(tl.int64) + dst = rows + owned = valid & (head < TOTAL_HEADS) + owned &= (index >= vocab_start) & (index < vocab_end) + local = index - vocab_start + local = tl.where(owned, local, 0) values = tl.load( weight + local[:, None] * DIM + cols[None, :], mask=owned[:, None], @@ -636,16 +678,240 @@ def _engram_lookup_kernel( # ue8m0 is a power of two, so its byte *is* the fp32 exponent field. scale = (scale.to(tl.int32) << 23).to(tl.float32, bitcast=True) tl.store( - out + rows[:, None] * DIM + cols[None, :], + out + dst[:, None] * DIM + cols[None, :], (values.to(tl.float32) * scale).to(tl.bfloat16), mask=valid[:, None], ) +def engram_head_shard_rank() -> int: + """This rank's slot among the hash-head shards of one engram table. + + TP-major, so the shards a DP gather brings in are contiguous heads and + the following TP gather completes the head order. + """ + dp_group = get_engram_dp_group() + dp_size = dp_group.world_size if dp_group is not None else 1 + dp_rank = dp_group.rank_in_group if dp_group is not None else 0 + return get_tensor_model_parallel_rank() * dp_size + dp_rank + + +def engram_gathered_num_tokens() -> int: + """Per-replica token slot for the node-local Engram DP group.""" + dp_metadata = get_forward_context().dp_metadata + if dp_metadata is None: + raise RuntimeError("a DP-shared engram table needs DP token metadata") + group = get_engram_dp_group() + assert group is not None + # Engram groups are contiguous slices of the full DP group. + start = get_dp_group().rank_in_group - group.rank_in_group + return int( + dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max() + ) + + +def gather_engram_hashes( + hash_ids: torch.Tensor, *, dp_shared_memory: bool = False +) -> torch.Tensor: + """Collect the n-gram ids of every DP replica sharing one table. + + Replicas are padded to a common token slot, so the gathered shape is + static under CUDA graph capture (where DP already pads alike). + """ + dp_group = get_engram_dp_group() + if dp_group is None or dp_shared_memory: + return hash_ids + slot = engram_gathered_num_tokens() + if hash_ids.shape[0] > slot: + raise ValueError("Engram token count exceeds the DP token slot") + if hash_ids.shape[0] < slot: + pad = hash_ids.new_full( + (slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID + ) + hash_ids = torch.cat((hash_ids, pad)) + return dp_group.all_gather(hash_ids, dim=0) + + +class DPSharedEngramStorage: + """Registered host weights shared by a node-local DP group with one writer.""" + + def __init__( + self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator + ) -> None: + self.group = group + weight_bytes = num_rows * dim + storage = self._allocate(weight_bytes + weight_bytes // block_size) + self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim) + self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size) + self._views: tuple[torch.Tensor, torch.Tensor] | None = None + + def _allocate(self, num_bytes: int) -> torch.Tensor: + """Map and register one physical allocation across a node-local DP group.""" + group = self.group + with ExitStack() as stack: + path = error = None + if group.rank_in_group == 0: + try: + check_shm_free_space(num_bytes) + backing_file = stack.enter_context( + tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH) + ) + backing_file.truncate(num_bytes) + path = backing_file.name + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + path, error = group.broadcast_object((path, error)) + if error is not None: + raise RuntimeError( + "Engram shared-memory creation failed on EDP rank 0: " + error + ) + + mapping = owner = tensor = None + try: + with open(path, "r+b") as file: + mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED) + owner = np.frombuffer(mapping, dtype=np.uint8) + pointer = owner.ctypes.data + tensor = torch.from_numpy(owner) + result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) + if result.value != 0: + raise RuntimeError(f"cudaHostRegister failed: {result}") + finalizer = weakref.finalize(owner, self._unregister, mapping, pointer) + finalizer.atexit = False # type: ignore[misc] + # The UVA helper otherwise allocates a private pinned copy. + if not tensor.is_pinned(): + raise RuntimeError( + "cudaHostRegister did not pin the shared Engram mapping" + ) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + + # Also fences peer mappings before the leader unlinks the file. + errors: list[str | None] = [None] * group.world_size + torch.distributed.all_gather_object(errors, error, group=group.cpu_group) + failures = "; ".join( + f"EDP rank {rank}: {error}" + for rank, error in enumerate(errors) + if error is not None + ) + if failures: + # Dropping the owner runs the finalizer; the mapping then unmaps. + del tensor, owner, mapping + raise RuntimeError( + "Engram shared-memory initialization failed: " + failures + ) + assert tensor is not None + return tensor + + @staticmethod + def _unregister(mapping: mmap.mmap, pointer: int) -> None: + # Torch storage retains the numpy owner, including through cached UVA views. + # Keep its mmap alive until CUDA has released the registration. + result = torch.cuda.cudart().cudaHostUnregister(pointer) + if result.value != 0: + logger.warning("Engram cudaHostUnregister failed: %s", result) + + def load_weight( + self, param: torch.nn.Parameter, loaded_weight: torch.Tensor + ) -> None: + if self.group.rank_in_group == 0: + _engram_head_shard_weight_loader(param, loaded_weight) + # Read order may differ across ranks. Equal load counts ensure all shared + # weights are ready after the last weight-loader call returns. + torch.distributed.barrier(group=self.group.cpu_group) + + def get_views( + self, weight: torch.Tensor, scales: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + if (weight.data_ptr(), scales.data_ptr()) != ( + self.weight.data_ptr(), + self.weight_scale_inv.data_ptr(), + ): + raise RuntimeError("Shared Engram parameter storage must not be replaced") + if self._views is None: + self._views = ( + get_accelerator_view_from_cpu_tensor(self.weight), + get_accelerator_view_from_cpu_tensor(self.weight_scale_inv), + ) + return self._views + + +def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool: + """Whether co-located DP replicas exist and /dev/shm can hold the full tables.""" + if get_engram_dp_size() == 1: + logger.warning_once( + "Engram DP replicas are not co-located on one node; " + "storing the offloaded tables per rank instead of sharing them." + ) + return False + num_bytes = sum(layout.num_embeddings) * ( + layout.head_dim + layout.head_dim // block_size + ) + group = get_engram_dp_group() + assert group is not None + # Only the leader allocates, so every replica follows its check. + error = None + if group.rank_in_group == 0: + if not os.path.isdir(SHM_PATH): + error = f"{SHM_PATH} is not mounted" + else: + try: + check_shm_free_space(num_bytes, allocation_name="Engram tables") + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + error = group.broadcast_object(error) + if error is not None: + logger.warning_once( + "Sharding the offloaded Engram tables across DP replicas: %s", error + ) + return error is None + + +def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None: + """Register prefaulted huge pages, or return None for pinned-memory fallback.""" + try: + mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS) + mapping.madvise(mmap.MADV_HUGEPAGE) + except OSError as exc: + logger.warning( + "Engram huge-page allocation failed (%s); using pinned memory.", exc + ) + return None + + owner = np.frombuffer(mapping, dtype=np.uint8) + # Fault the pages in before CUDA pins them, so they can be huge. + owner[:: mmap.PAGESIZE] = 0 + tensor = torch.from_numpy(owner) + pointer = tensor.data_ptr() + result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) + if result.value != 0: + logger.warning( + "Engram cudaHostRegister failed (%s); using pinned memory.", result + ) + return None + # Tensor views retain owner; its finalizer retains mapping until unregister. + finalizer = weakref.finalize( + owner, DPSharedEngramStorage._unregister, mapping, pointer + ) + finalizer.atexit = False # type: ignore[misc] + if not tensor.is_pinned(): + logger.warning( + "CUDA did not recognize Engram registration; using pinned memory." + ) + return None + return tensor + + class ParallelEngramEmbedding(nn.Module): - """TP-sharded hash heads with FP8 rows and per-block E8M0 scales.""" + """Hash heads with FP8 rows and per-block E8M0 scales. + + Heads are TP-sharded, and additionally DP-sharded across a node-local + group unless the table lives in (DP-shared) pinned host memory. + """ _weight_loader = staticmethod(_engram_head_shard_weight_loader) + _shared_memory: DPSharedEngramStorage | None = None + _packed: torch.Tensor | None = None def __init__( self, @@ -653,10 +919,32 @@ class ParallelEngramEmbedding(nn.Module): dim: int, head_sizes: tuple[int, ...], block_size: int = 32, + cpu_offload: bool = False, + dp_shared_memory: bool = False, + use_thp: bool = False, ) -> None: super().__init__() assert head_sizes and all(size > 0 for size in head_sizes) assert sum(head_sizes) <= num_embeddings + self.cpu_offload = cpu_offload + self.dp_shared_memory = dp_shared_memory + self.use_thp = use_thp + self.dp_size = get_engram_dp_size() + if dp_shared_memory: + if not cpu_offload: + raise ValueError("dp_shared_memory requires cpu_offload=True") + if self.dp_size <= 1: + raise ValueError( + "dp_shared_memory requires a node-local Engram DP " + f"group with size > 1; effective Engram DP size is {self.dp_size}. " + "Check that the node layout and rank placement allow complete " + "DP replicas to be co-located." + ) + self.dp_size = 1 + if cpu_offload and not is_uva_available(): + raise RuntimeError("Engram CPU offload requires UVA support") + self._views: tuple[torch.Tensor, torch.Tensor] | None = None + self._view_src: tuple[int, int] | None = None self.num_embeddings = num_embeddings self.dim = dim self.block_size = block_size @@ -688,20 +976,99 @@ class ParallelEngramEmbedding(nn.Module): "engram_vocab_start": self.vocab_start_idx, }, ) + if cpu_offload: + # Constant dummy values avoid randomizing huge CPU lookup tables. + set_weight_attrs(self.weight, {"dummy_weight_value": 1.0}) + # The ue8m0 encoding of scale 1.0 is exponent byte 127. + set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127}) + logger.info( + "Engram table offloaded to pinned host memory: %d rows x %d, " + "%.2f GiB %s", + self.part_num_embeddings, + dim, + self.part_num_embeddings * (dim + dim // block_size) / 1024**3, + "shared across DP replicas" if dp_shared_memory else "per rank", + ) def _get_shard_info(self) -> tuple[int, int]: - return self.tp_size, get_tensor_model_parallel_rank() + if self.dp_size == 1: + return self.tp_size, get_tensor_model_parallel_rank() + return self.tp_size * self.dp_size, engram_head_shard_rank() def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]: + if self.dp_shared_memory: + group = get_engram_dp_group() + assert group is not None + storage = DPSharedEngramStorage( + self.part_num_embeddings, self.dim, self.block_size, group + ) + self._shared_memory = storage + self._weight_loader = storage.load_weight + return storage.weight, storage.weight_scale_inv + scale_dim = self.dim // self.block_size + if not self.cpu_offload: + return ( + torch.empty( + self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn + ), + torch.empty(self.part_num_embeddings, scale_dim, dtype=torch.uint8), + ) + if self.use_thp: + weight_bytes = self.part_num_embeddings * self.dim + packed = _allocate_huge_page_storage( + weight_bytes + weight_bytes // self.block_size + ) + if packed is not None: + self._packed = packed + return ( + packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim), + packed[weight_bytes:].view(-1, scale_dim), + ) + # Model initialization may be inside a CUDA device context. return ( - torch.empty(self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn), torch.empty( - self.part_num_embeddings, self.dim // self.block_size, dtype=torch.uint8 + self.part_num_embeddings, + self.dim, + dtype=torch.float8_e4m3fn, + device="cpu", + pin_memory=True, + ), + torch.empty( + self.part_num_embeddings, + scale_dim, + dtype=torch.uint8, + device="cpu", + pin_memory=True, ), ) + def collapse_huge_pages(self) -> None: + """Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small.""" + if self._packed is None: + return + addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes + libc = ctypes.CDLL(None, use_errno=True) + libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) + if libc.madvise(addr, num_bytes, 25) != 0: + logger.warning( + "Engram MADV_COLLAPSE failed; keeping existing pages: %s", + os.strerror(ctypes.get_errno()), + ) + def _storage(self) -> tuple[torch.Tensor, torch.Tensor]: - return self.weight.data, self.weight_scale_inv.data + if self._shared_memory is not None: + return self._shared_memory.get_views(self.weight, self.weight_scale_inv) + if not self.cpu_offload: + return self.weight.data, self.weight_scale_inv.data + src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr()) + if self._view_src != src: + self._views = ( + get_accelerator_view_from_cpu_tensor(self.weight.data), + get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data), + ) + self._view_src = src + assert self._views is not None + return self._views def lookup( self, indices: torch.Tensor, out: torch.Tensor, background: bool = False @@ -714,6 +1081,24 @@ class ParallelEngramEmbedding(nn.Module): if not rows: return weight, scales = self._storage() + ids_stride_t, ids_stride_h = indices.stride() + sort_min_rows, _ = _engram_lookup_thresholds(indices.device) + sort = ( + self.cpu_offload + and self._packed is None + and sort_min_rows is not None + and rows >= sort_min_rows + ) + dst = indices + if sort: + # Small-page host tables miss the TLB per row; in table order, rows on + # one page share it. + head_end = self.head_start + self.part_n_hash_cols + local = indices[:, self.head_start : head_end] - self.vocab_start_idx + if (pad := self.part_n_hash_cols - local.shape[1]) > 0: + local = nn.functional.pad(local, (0, pad), value=-1) + indices, dst = local.flatten().sort() + dst = dst.int() # The table dwarfs TLB reach, so a persistent grid near the SM count # beats one program per row; halve it to leave SMs for the main stream. tiles = triton.cdiv(rows, 16) @@ -722,12 +1107,13 @@ class ParallelEngramEmbedding(nn.Module): weight, scales, indices, + dst, out, self.vocab_start_idx, self.vocab_end_idx, rows, - indices.stride(0), - indices.stride(1), + ids_stride_t, + ids_stride_h, HEAD_START=self.head_start, LOCAL_HEADS=self.part_n_hash_cols, TOTAL_HEADS=self.n_hash_cols, @@ -735,17 +1121,23 @@ class ParallelEngramEmbedding(nn.Module): QUANT_BLOCK=self.block_size, BLOCK_R=16, GRID=grid, + SORTED=sort, ) def forward(self, indices: torch.Tensor) -> torch.Tensor: """indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim] bf16, gathered from all shards for this replica's tokens.""" + num_tokens = indices.shape[0] + if self.dp_size > 1: + indices = gather_engram_hashes(indices) out = torch.empty( (indices.shape[0], self.part_n_hash_cols, self.dim), dtype=torch.bfloat16, device=indices.device, ) self.lookup(indices, out) + if self.dp_size > 1: + out = _gather_engram_rows(out, num_tokens) if self.tp_size > 1: out = tensor_model_parallel_all_gather(out, dim=1) return out[:, : self.n_hash_cols] @@ -891,6 +1283,25 @@ def _engram_select_rows( ) +def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor: + """Exchange DP tokens for heads, retaining only this replica's tokens.""" + dp_group = get_engram_dp_group() + assert dp_group is not None + slot, remainder = divmod(staged.shape[0], dp_group.world_size) + assert remainder == 0 and 0 <= num_tokens <= slot + gathered = dp_group.all_gather(staged, dim=0) + local_heads, dim = staged.shape[1:] + rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim)) + _engram_select_rows( + gathered, + rows, + staged.shape[0], + dp_group.rank_in_group * slot, + local_heads * dim, + ) + return rows + + class Engram(nn.Module): """Writes an n-gram lookup into the residual stream, gated by how well it matches that stream. @@ -901,6 +1312,9 @@ class Engram(nn.Module): training kernel). """ + _prefetch_stream: torch.cuda.Stream | None = None + _prefetch_done: torch.cuda.Event | None = None + def __init__( self, config, @@ -909,8 +1323,12 @@ class Engram(nn.Module): layer_hash_index: int, use_sequence_parallel: bool, prefix: str, + prefetch_stream: torch.cuda.Stream | None = None, ) -> None: super().__init__() + # Layers sharing one stream serialize their offloaded lookups, so they + # take turns instead of jointly starving decoder compute of SMs. + self._prefetch_stream = prefetch_stream self.layer_hash_index = layer_hash_index self.dim = config.hidden_size self.hc_mult = config.hc_mult @@ -950,28 +1368,78 @@ class Engram(nn.Module): def _create_embedding( self, layout: EngramLayout, layer_hash_index: int ) -> ParallelEngramEmbedding: + engram_config = get_current_vllm_config().engram_config + assert engram_config is not None return ParallelEngramEmbedding( layout.num_embeddings[layer_hash_index], layout.head_dim, tuple(size for order in layout.primes[layer_hash_index] for size in order), + cpu_offload=engram_config.cpu_offload, + dp_shared_memory=bool(engram_config.dp_shared_memory), + use_thp=engram_config.use_thp, ) def _init_staging(self, max_tokens: int, head_dim: int) -> None: # Persistent storage keeps lookup addresses stable across graph replays. self.staged_rows = torch.empty( - max_tokens, + max_tokens * self.embed_tokens.dp_size, self.embed_tokens.part_n_hash_cols, head_dim, dtype=torch.bfloat16, ) + if not self.embed_tokens.cpu_offload: + self._prefetch_stream = None + return + if self._prefetch_stream is None: + self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device) + self._prefetch_done = torch.cuda.Event() def prepare_embeddings(self, hash_ids: torch.Tensor) -> None: - """Look up rows before the decoder layers consume them.""" + """Look up, or prefetch from host memory, rows before the decoder layers + consume them; DP-sharded tables take the group's gathered hash IDs.""" rows = self.staged_rows[: hash_ids.shape[0]] assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small" - self.embed_tokens.lookup(hash_ids, rows) + if self._prefetch_stream is None: + self.embed_tokens.lookup(hash_ids, rows) + else: + self._start_prefetch(hash_ids, rows) + + @eager_break_during_capture + def _start_prefetch(self, hash_ids: torch.Tensor, rows: torch.Tensor) -> None: + # Eager boundaries let the lookup span piecewise graph segments, and + # decide per replay (on the replay stream) whether it runs inline. + num_tokens = hash_ids.shape[0] + # All DP ranks must agree, or EP collectives wait on the slowest one. + if is_forward_context_available() and (dp := get_forward_context().dp_metadata): + num_tokens = int(dp.num_tokens_across_dp_cpu.max()) + current = stream = torch.cuda.current_stream() + # Big lookups stall persistent main-stream kernels anyway: run them inline. + _, inline_min_tokens = _engram_lookup_thresholds(hash_ids.device) + if inline_min_tokens is not None and num_tokens >= inline_min_tokens: + self.embed_tokens.lookup(hash_ids, rows) + else: + stream = self._prefetch_stream + assert stream is not None + stream.wait_stream(current) + # Keep temporary hash storage alive until lookup finishes reading it. + hash_ids.record_stream(stream) + with torch.cuda.stream(stream): + self.embed_tokens.lookup(hash_ids, rows, background=True) + assert self._prefetch_done is not None + self._prefetch_done.record(stream) + + @eager_break_during_capture + def _finish_prefetch(self, event: torch.cuda.Event) -> None: + # Wait for this layer's lookup only, not for later layers on the stream. + torch.cuda.current_stream().wait_event(event) def _ready_rows(self, num_tokens: int) -> torch.Tensor: + if self._prefetch_done is not None: + self._finish_prefetch(self._prefetch_done) + if self.embed_tokens.dp_size > 1: + slot = engram_gathered_num_tokens() + staged = self.staged_rows[: slot * self.embed_tokens.dp_size] + return _gather_engram_rows(staged, num_tokens) return self.staged_rows[:num_tokens] def embed(self, hash_ids: torch.Tensor) -> torch.Tensor: diff --git a/vllm/models/deepseek_v41/nvidia/engram.py b/vllm/models/deepseek_v41/nvidia/engram.py deleted file mode 100644 index 7d1e0f8a4b..0000000000 --- a/vllm/models/deepseek_v41/nvidia/engram.py +++ /dev/null @@ -1,497 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch.""" - -import ctypes -import mmap -import os -import shutil -import tempfile -import weakref -from contextlib import ExitStack - -import numpy as np -import torch - -from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import get_current_vllm_config -from vllm.distributed import ( - get_dp_group, - get_engram_dp_group, - get_engram_dp_size, - get_tensor_model_parallel_rank, - tensor_model_parallel_all_gather, -) -from vllm.distributed.device_communicators.shm_broadcast import ( - SHM_PATH, - check_shm_free_space, -) -from vllm.distributed.parallel_state import GroupCoordinator -from vllm.forward_context import get_forward_context -from vllm.logger import init_logger -from vllm.model_executor.utils import set_weight_attrs -from vllm.models.deepseek_v41.common.engram import ( - DEAD_ID, - EngramLayout, - _engram_head_shard_weight_loader, - _engram_select_rows, -) -from vllm.models.deepseek_v41.common.engram import ( - Engram as BaseEngram, -) -from vllm.models.deepseek_v41.common.engram import ( - ParallelEngramEmbedding as BaseParallelEngramEmbedding, -) -from vllm.utils.platform_utils import is_uva_available -from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor - -logger = init_logger(__name__) - - -def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None: - """Register prefaulted huge pages, or return None for pinned-memory fallback.""" - try: - mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS) - mapping.madvise(mmap.MADV_HUGEPAGE) - except OSError as exc: - logger.warning( - "Engram huge-page allocation failed (%s); using pinned memory.", exc - ) - return None - - owner = np.frombuffer(mapping, dtype=np.uint8) - # Fault the pages in before CUDA pins them, so they can be huge. - owner[:: mmap.PAGESIZE] = 0 - tensor = torch.from_numpy(owner) - pointer = tensor.data_ptr() - result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) - if result.value != 0: - logger.warning( - "Engram cudaHostRegister failed (%s); using pinned memory.", result - ) - return None - # Tensor views retain owner; its finalizer retains mapping until unregister. - finalizer = weakref.finalize( - owner, DPSharedEngramStorage._unregister, mapping, pointer - ) - finalizer.atexit = False # type: ignore[misc] - if not tensor.is_pinned(): - logger.warning( - "CUDA did not recognize Engram registration; using pinned memory." - ) - return None - return tensor - - -def engram_head_shard_rank() -> int: - """This rank's slot among the hash-head shards of one engram table. - - TP-major, so the shards a DP gather brings in are contiguous heads and - the following TP gather completes the head order. - """ - dp_group = get_engram_dp_group() - dp_size = dp_group.world_size if dp_group is not None else 1 - dp_rank = dp_group.rank_in_group if dp_group is not None else 0 - return get_tensor_model_parallel_rank() * dp_size + dp_rank - - -def engram_gathered_num_tokens() -> int: - """Per-replica token slot for the node-local Engram DP group.""" - dp_metadata = get_forward_context().dp_metadata - if dp_metadata is None: - raise RuntimeError("a DP-shared engram table needs DP token metadata") - group = get_engram_dp_group() - assert group is not None - # Engram groups are contiguous slices of the full DP group. - start = get_dp_group().rank_in_group - group.rank_in_group - return int( - dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max() - ) - - -def gather_engram_hashes( - hash_ids: torch.Tensor, *, dp_shared_memory: bool = False -) -> torch.Tensor: - """Collect the n-gram ids of every DP replica sharing one table. - - Replicas are padded to a common token slot, so the gathered shape is - static under CUDA graph capture (where DP already pads alike). - """ - dp_group = get_engram_dp_group() - if dp_group is None or dp_shared_memory: - return hash_ids - slot = engram_gathered_num_tokens() - if hash_ids.shape[0] > slot: - raise ValueError("Engram token count exceeds the DP token slot") - if hash_ids.shape[0] < slot: - pad = hash_ids.new_full( - (slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID - ) - hash_ids = torch.cat((hash_ids, pad)) - return dp_group.all_gather(hash_ids, dim=0) - - -def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool: - """Whether co-located DP replicas exist and /dev/shm can hold the full tables.""" - if not os.path.isdir(SHM_PATH): - logger.warning_once( - "%s is not mounted; storing the offloaded Engram tables per rank " - "instead of sharing them across DP replicas.", - SHM_PATH, - ) - return False - if get_engram_dp_size() == 1: - logger.warning_once( - "Engram DP replicas are not co-located on one node; " - "storing the offloaded tables per rank instead of sharing them." - ) - return False - num_bytes = sum(layout.num_embeddings) * ( - layout.head_dim + layout.head_dim // block_size - ) - if shutil.disk_usage(SHM_PATH).total < num_bytes: - logger.warning_once( - "Sharing Engram tables across DP replicas needs %.1f GiB of %s " - "(--shm-size or --ipc=host); sharding them across replicas instead.", - num_bytes / 1024**3, - SHM_PATH, - ) - return False - return True - - -class DPSharedEngramStorage: - """Registered host weights shared by a node-local DP group with one writer.""" - - def __init__( - self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator - ) -> None: - self.group = group - weight_bytes = num_rows * dim - storage = self._allocate(weight_bytes + weight_bytes // block_size) - self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim) - self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size) - self._views: tuple[torch.Tensor, torch.Tensor] | None = None - - def _allocate(self, num_bytes: int) -> torch.Tensor: - """Map and register one physical allocation across a node-local DP group.""" - group = self.group - with ExitStack() as stack: - path = error = None - if group.rank_in_group == 0: - try: - check_shm_free_space(num_bytes) - backing_file = stack.enter_context( - tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH) - ) - backing_file.truncate(num_bytes) - path = backing_file.name - except Exception as exc: - error = f"{type(exc).__name__}: {exc}" - path, error = group.broadcast_object((path, error)) - if error is not None: - raise RuntimeError( - "Engram shared-memory creation failed on EDP rank 0: " + error - ) - - mapping = owner = tensor = None - try: - with open(path, "r+b") as file: - mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED) - owner = np.frombuffer(mapping, dtype=np.uint8) - pointer = owner.ctypes.data - tensor = torch.from_numpy(owner) - result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) - if result.value != 0: - raise RuntimeError(f"cudaHostRegister failed: {result}") - finalizer = weakref.finalize(owner, self._unregister, mapping, pointer) - finalizer.atexit = False # type: ignore[misc] - # The UVA helper otherwise allocates a private pinned copy. - if not tensor.is_pinned(): - raise RuntimeError( - "cudaHostRegister did not pin the shared Engram mapping" - ) - except Exception as exc: - error = f"{type(exc).__name__}: {exc}" - - # Also fences peer mappings before the leader unlinks the file. - errors: list[str | None] = [None] * group.world_size - torch.distributed.all_gather_object(errors, error, group=group.cpu_group) - failures = "; ".join( - f"EDP rank {rank}: {error}" - for rank, error in enumerate(errors) - if error is not None - ) - if failures: - # Dropping the owner runs the finalizer; the mapping then unmaps. - del tensor, owner, mapping - raise RuntimeError( - "Engram shared-memory initialization failed: " + failures - ) - assert tensor is not None - return tensor - - @staticmethod - def _unregister(mapping: mmap.mmap, pointer: int) -> None: - # Torch storage retains the numpy owner, including through cached UVA views. - # Keep its mmap alive until CUDA has released the registration. - result = torch.cuda.cudart().cudaHostUnregister(pointer) - if result.value != 0: - logger.warning("Engram cudaHostUnregister failed: %s", result) - - def load_weight( - self, param: torch.nn.Parameter, loaded_weight: torch.Tensor - ) -> None: - if self.group.rank_in_group == 0: - _engram_head_shard_weight_loader(param, loaded_weight) - # Read order may differ across ranks. Equal load counts ensure all shared - # weights are ready after the last weight-loader call returns. - torch.distributed.barrier(group=self.group.cpu_group) - - def get_views( - self, weight: torch.Tensor, scales: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - if (weight.data_ptr(), scales.data_ptr()) != ( - self.weight.data_ptr(), - self.weight_scale_inv.data_ptr(), - ): - raise RuntimeError("Shared Engram parameter storage must not be replaced") - if self._views is None: - self._views = ( - get_accelerator_view_from_cpu_tensor(self.weight), - get_accelerator_view_from_cpu_tensor(self.weight_scale_inv), - ) - return self._views - - -class ParallelEngramEmbedding(BaseParallelEngramEmbedding): - """Extend TP lookup with DP head sharding or shared, CPU-offloaded TP slices.""" - - _shared_memory: DPSharedEngramStorage | None = None - - def __init__( - self, - num_embeddings: int, - dim: int, - head_sizes: tuple[int, ...], - block_size: int = 32, - cpu_offload: bool = False, - dp_shared_memory: bool = False, - use_thp: bool = False, - ) -> None: - self.cpu_offload = cpu_offload - self.dp_shared_memory = dp_shared_memory - self.use_thp = use_thp - self._packed: torch.Tensor | None = None - self.dp_size = get_engram_dp_size() - if dp_shared_memory: - if not cpu_offload: - raise ValueError("dp_shared_memory requires cpu_offload=True") - if self.dp_size <= 1: - raise ValueError( - "dp_shared_memory requires a node-local Engram DP " - f"group with size > 1; effective Engram DP size is {self.dp_size}. " - "Check that the node layout and rank placement allow complete " - "DP replicas to be co-located." - ) - self.dp_size = 1 - if cpu_offload and not is_uva_available(): - raise RuntimeError("Engram CPU offload requires UVA support") - self._views: tuple[torch.Tensor, torch.Tensor] | None = None - self._view_src: tuple[int, int] | None = None - super().__init__(num_embeddings, dim, head_sizes, block_size) - if cpu_offload: - # Constant dummy values avoid randomizing huge CPU lookup tables. - set_weight_attrs(self.weight, {"dummy_weight_value": 1.0}) - # The ue8m0 encoding of scale 1.0 is exponent byte 127. - set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127}) - logger.info( - "Engram table offloaded to pinned host memory: %d rows x %d, " - "%.2f GiB %s", - self.part_num_embeddings, - dim, - self.part_num_embeddings * (dim + dim // block_size) / 1024**3, - "shared across DP replicas" if dp_shared_memory else "per rank", - ) - - def _get_shard_info(self) -> tuple[int, int]: - if self.dp_size == 1: - return super()._get_shard_info() - return self.tp_size * self.dp_size, engram_head_shard_rank() - - def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]: - if self.dp_shared_memory: - group = get_engram_dp_group() - assert group is not None - storage = DPSharedEngramStorage( - self.part_num_embeddings, self.dim, self.block_size, group - ) - self._shared_memory = storage - self._weight_loader = storage.load_weight - return storage.weight, storage.weight_scale_inv - if not self.cpu_offload: - return super()._allocate_weights() - if self.use_thp: - weight_bytes = self.part_num_embeddings * self.dim - packed = _allocate_huge_page_storage( - weight_bytes + weight_bytes // self.block_size - ) - if packed is not None: - self._packed = packed - return ( - packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim), - packed[weight_bytes:].view(-1, self.dim // self.block_size), - ) - # Model initialization may be inside a CUDA device context. - return ( - torch.empty( - self.part_num_embeddings, - self.dim, - dtype=torch.float8_e4m3fn, - device="cpu", - pin_memory=True, - ), - torch.empty( - self.part_num_embeddings, - self.dim // self.block_size, - dtype=torch.uint8, - device="cpu", - pin_memory=True, - ), - ) - - def collapse_huge_pages(self) -> None: - """Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small.""" - if self._packed is None: - return - addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes - libc = ctypes.CDLL(None, use_errno=True) - libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) - if libc.madvise(addr, num_bytes, 25) != 0: - logger.warning( - "Engram MADV_COLLAPSE failed; keeping existing pages: %s", - os.strerror(ctypes.get_errno()), - ) - - def _storage(self) -> tuple[torch.Tensor, torch.Tensor]: - if self._shared_memory is not None: - return self._shared_memory.get_views(self.weight, self.weight_scale_inv) - if not self.cpu_offload: - return super()._storage() - src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr()) - if self._view_src != src: - self._views = ( - get_accelerator_view_from_cpu_tensor(self.weight.data), - get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data), - ) - self._view_src = src - assert self._views is not None - return self._views - - def forward(self, indices: torch.Tensor) -> torch.Tensor: - if self.dp_size == 1: - return super().forward(indices) - num_tokens = indices.shape[0] - indices = gather_engram_hashes(indices) - out = torch.empty( - (indices.shape[0], self.part_n_hash_cols, self.dim), - dtype=torch.bfloat16, - device=indices.device, - ) - self.lookup(indices, out) - out = _gather_engram_rows(out, num_tokens) - if self.tp_size > 1: - out = tensor_model_parallel_all_gather(out, dim=1) - return out[:, : self.n_hash_cols] - - -def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor: - """Exchange DP tokens for heads, retaining only this replica's tokens.""" - dp_group = get_engram_dp_group() - assert dp_group is not None - slot, remainder = divmod(staged.shape[0], dp_group.world_size) - assert remainder == 0 and 0 <= num_tokens <= slot - gathered = dp_group.all_gather(staged, dim=0) - local_heads, dim = staged.shape[1:] - rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim)) - _engram_select_rows( - gathered, - rows, - staged.shape[0], - dp_group.rank_in_group * slot, - local_heads * dim, - ) - return rows - - -class Engram(BaseEngram): - """NVIDIA Engram with asynchronous offload and node-local DP lookup.""" - - _prefetch_stream: torch.cuda.Stream | None = None - _prefetch_done: torch.cuda.Event | None = None - - def __init__( - self, *args, prefetch_stream: torch.cuda.Stream | None = None, **kwargs - ) -> None: - # Layers sharing one stream serialize their offloaded lookups, so they - # take turns instead of jointly starving decoder compute of SMs. - self._prefetch_stream = prefetch_stream - super().__init__(*args, **kwargs) - - def _create_embedding( - self, layout: EngramLayout, layer_hash_index: int - ) -> ParallelEngramEmbedding: - engram_config = get_current_vllm_config().engram_config - assert engram_config is not None - return ParallelEngramEmbedding( - layout.num_embeddings[layer_hash_index], - layout.head_dim, - tuple(size for order in layout.primes[layer_hash_index] for size in order), - cpu_offload=engram_config.cpu_offload, - dp_shared_memory=bool(engram_config.dp_shared_memory), - use_thp=engram_config.use_thp, - ) - - def _init_staging(self, max_tokens: int, head_dim: int) -> None: - super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim) - if not self.embed_tokens.cpu_offload: - self._prefetch_stream = None - return - if self._prefetch_stream is None: - self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device) - self._prefetch_done = torch.cuda.Event() - - def prepare_embeddings(self, hash_ids: torch.Tensor) -> None: - """Prefetch local shared rows or the DP group's gathered hash IDs.""" - if self._prefetch_stream is None: - return super().prepare_embeddings(hash_ids) - rows = self.staged_rows[: hash_ids.shape[0]] - assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small" - self._start_prefetch(hash_ids, rows, self._prefetch_stream) - - @eager_break_during_capture - def _start_prefetch( - self, hash_ids: torch.Tensor, rows: torch.Tensor, stream: torch.cuda.Stream - ) -> None: - # Eager boundaries let the lookup span piecewise graph segments. - stream.wait_stream(torch.cuda.current_stream()) - # Keep temporary hash storage alive until lookup finishes reading it. - hash_ids.record_stream(stream) - with torch.cuda.stream(stream): - self.embed_tokens.lookup(hash_ids, rows, background=True) - assert self._prefetch_done is not None - self._prefetch_done.record(stream) - - @eager_break_during_capture - def _finish_prefetch(self, event: torch.cuda.Event) -> None: - # Wait for this layer's lookup only, not for later layers on the stream. - torch.cuda.current_stream().wait_event(event) - - def _ready_rows(self, num_tokens: int) -> torch.Tensor: - if self._prefetch_done is not None: - self._finish_prefetch(self._prefetch_done) - if self.embed_tokens.dp_size > 1: - slot = engram_gathered_num_tokens() - staged = self.staged_rows[: slot * self.embed_tokens.dp_size] - return _gather_engram_rows(staged, num_tokens) - return super()._ready_rows(num_tokens) diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index f7647ec81d..b8fa7754b7 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -98,9 +98,14 @@ from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.worker.ubatching import dbo_current_ubatch_id -from ..common.engram import EngramLayout, NgramHashState +from ..common.engram import ( + Engram, + EngramLayout, + NgramHashState, + can_share_engram_tables, + gather_engram_hashes, +) from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask -from .engram import Engram, can_share_engram_tables, gather_engram_hashes from .ops.mhc import ( MHC_OVERLAP_MAX_TOKENS, init_mhc_all_reduce, From ec5ef92c87a66cb9e69c349d82d5e0980d494ab5 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 1 Oct 2026 12:21:50 +0800 Subject: [PATCH 176/248] [Rust Frontend] Anchor tokens after pending UTF-8 bytes at their own first byte (#58357) Signed-off-by: Bugen Zhao Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/src/tokenizer/src/incremental.rs | 76 ++++++++++++- .../src/incremental/attribution/tests.rs | 100 ++++++++++++++++++ 2 files changed, 172 insertions(+), 4 deletions(-) diff --git a/rust/src/tokenizer/src/incremental.rs b/rust/src/tokenizer/src/incremental.rs index bdc50d175c..9dfc345f26 100644 --- a/rust/src/tokenizer/src/incremental.rs +++ b/rust/src/tokenizer/src/incremental.rs @@ -42,6 +42,10 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> { prefix: String, prefix_index: usize, prefix_seeded: bool, + /// Joint decode from the last `push_token` that left tokens pending, i.e. + /// `decode(context ++ pending)`. Used to split the pending group from the + /// next token when that token provably does not merge with it. + pending_string: Option, decoded: AttributedTextBuffer, } @@ -60,6 +64,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> { prefix: String::new(), prefix_index: 0, prefix_seeded: prompt_token_ids.is_empty(), + pending_string: None, decoded: AttributedTextBuffer::default(), } } @@ -72,6 +77,44 @@ const SAFE_SUFFIX_MIN: usize = 4; const SAFE_SUFFIX_MAX: usize = 6; impl DecodeStream<'_, T> { + /// Decide whether `token_id` can be anchored at its own first byte inside + /// `new_chunk`, separately from the older tokens still pending. + /// + /// Pending tokens (an incomplete UTF-8 sequence, or text still empty in + /// context) normally share one anchor with the token that completes them, + /// because the tokenizer only defines their joint decode. That sharing is + /// wrong when the new token does *not* merge with them, e.g. a stray + /// incomplete byte followed by a special token or plain ASCII: the joint + /// decode is then exactly `decode(pending) ++ decode([token_id])`, and the + /// new token's first byte is known. Returns that split offset, or `None` + /// when the tokens merge (byte fallback completing a character), when + /// nothing is pending, or when the isolated decode is not provably + /// consistent with the joint one. `pending` is the joint decode kept from the + /// push that left tokens pending. Decoded text is never changed by this; + /// only anchor placement is. + fn pending_group_split( + &self, + pending: Option<&str>, + token_id: u32, + new_chunk: &str, + ) -> Result> { + let Some(pending) = pending else { + return Ok(None); + }; + let prefix_len = self.prefix.len(); + let pending_chunk = &pending[pending.floor_char_boundary(prefix_len)..]; + // Empty pending text: today's shared anchor is already this token's first + // byte. Not a prefix: the new token completed the pending character. + if pending_chunk.is_empty() || !new_chunk.starts_with(pending_chunk) { + return Ok(None); + } + let own = self.tokenizer.decode(&[token_id], self.skip_special_tokens)?; + let independent = !own.is_empty() + && new_chunk.len() == pending_chunk.len() + own.len() + && new_chunk.ends_with(&own); + Ok(independent.then_some(pending_chunk.len())) + } + /// Decode prompt-only context for prefix seeding. /// /// Prompt ids may come from the model vocabulary rather than the local @@ -145,16 +188,40 @@ impl IncrementalDecoder for DecodeStream<'_, T> { || (!produced_text && self.tokenizer.id_to_token(token_id).is_none()); if zero_width { self.decoded.record_zero_width_token(token_id); - } else { - self.decoded.record_pending_token(token_id); } - if !produced_text { + if !zero_width { + self.decoded.record_pending_token(token_id); + self.pending_string = Some(string); + } return Ok(0); } + // Ensure we split at a utf-8 char boundary. let new_chunk = &string[string.floor_char_boundary(prefix_len)..]; - self.decoded.append_visible_text(new_chunk); + // Text is released, so the pending group resolves in this push either + // way; never carry its joint decode into a later push. + let pending = self.pending_string.take(); + let split = if zero_width { + None + } else { + self.pending_group_split(pending.as_deref(), token_id, new_chunk)? + }; + match split { + Some(split) => { + // Older pending tokens own the prefix (including any U+FFFD from + // an incomplete character); this token owns exactly its own text. + self.decoded.append_visible_text(&new_chunk[..split]); + self.decoded.record_pending_token(token_id); + self.decoded.append_visible_text(&new_chunk[split..]); + } + None => { + if !zero_width { + self.decoded.record_pending_token(token_id); + } + self.decoded.append_visible_text(new_chunk); + } + } self.ids.drain(..self.prefix_index); self.prefix = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?; self.prefix_index = self.ids.len(); @@ -184,6 +251,7 @@ impl IncrementalDecoder for DecodeStream<'_, T> { } } self.decoded.resolve_pending_zero_width(); + self.pending_string = None; self.ids.clear(); self.prefix.clear(); self.prefix_index = 0; diff --git a/rust/src/tokenizer/src/incremental/attribution/tests.rs b/rust/src/tokenizer/src/incremental/attribution/tests.rs index cd39a1c03c..7764173ebf 100644 --- a/rust/src/tokenizer/src/incremental/attribution/tests.rs +++ b/rust/src/tokenizer/src/incremental/attribution/tests.rs @@ -520,6 +520,106 @@ fn attribution_incomplete_utf8_tokens_share_replacement_anchor_on_flush() { }); } +#[test] +fn attribution_special_token_after_incomplete_utf8_anchors_at_its_own_spelling() { + // A stray incomplete byte followed by a special token: the joint decode is + // exactly `decode([0xe4]) ++ decode([special])`, so the special token owns + // its own first byte instead of sharing the group's anchor at 0. + let tokenizer = tokenizer_with_special_token(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![0xe4, SPECIAL_TOKEN_ID], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded( + "�", + &[0xe4, SPECIAL_TOKEN_ID], + &[visible(0), visible(3)], + )], + expected_full: decoded( + "�", + &[0xe4, SPECIAL_TOKEN_ID], + &[visible(0), visible(3)], + ), + }); +} + +#[test] +fn attribution_ascii_token_after_incomplete_utf8_anchors_at_its_own_byte() { + // Same rule for an ordinary token: the split is decided by the non-merge + // equality, not by the token being special. + let tokenizer = TestTokenizer::new(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![0xe4, b'a' as u32], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded( + "�a", + &[0xe4, b'a' as u32], + &[visible(0), visible(3)], + )], + expected_full: decoded("�a", &[0xe4, b'a' as u32], &[visible(0), visible(3)]), + }); +} + +#[test] +fn attribution_byte_fallback_completion_keeps_shared_anchor_before_special_token() { + // The completing byte merges with the pending group (no split); the special + // token that follows a complete character is an ordinary independent token. + let tokenizer = tokenizer_with_special_token(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![0xe4, 0xbd, 0xa0, SPECIAL_TOKEN_ID], + skip_special_tokens: false, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![ + decoded( + "你", + &[0xe4, 0xbd, 0xa0], + &[visible(0), visible(0), visible(0)], + ), + decoded("", &[SPECIAL_TOKEN_ID], &[visible(0)]), + ], + expected_full: decoded( + "你", + &[0xe4, 0xbd, 0xa0, SPECIAL_TOKEN_ID], + &[visible(0), visible(0), visible(0), visible(3)], + ), + }); +} + +#[test] +fn attribution_filtered_special_between_incomplete_utf8_and_independent_token() { + // A skipped special token sits between the pending byte and the independent + // token; it stays zero-width in generation order and the split still applies. + let tokenizer = tokenizer_with_special_token(); + run_attribution_case(AttributionCase { + tokenizer: &tokenizer, + prompt_token_ids: vec![], + token_ids: vec![0xe4, SPECIAL_TOKEN_ID, b'a' as u32], + skip_special_tokens: true, + min_bytes_to_buffer: 0, + truncate_output_to: None, + expected_chunks: vec![decoded( + "�a", + &[0xe4, SPECIAL_TOKEN_ID, b'a' as u32], + &[visible(0), zero_width(0), visible(3)], + )], + expected_full: decoded( + "�a", + &[0xe4, SPECIAL_TOKEN_ID, b'a' as u32], + &[visible(0), zero_width(0), visible(3)], + ), + }); +} + #[test] fn attribution_truncation_converts_fully_removed_token_to_zero_width() { run_attribution_case(AttributionCase { From 36229e1337ae0e44e60075a11ebfde210c11d706 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 1 Oct 2026 12:21:50 +0800 Subject: [PATCH 177/248] [Rust Frontend] Token-aware marker parsing for Kimi K3 (#58358) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/Cargo.lock | 2 + rust/Cargo.toml | 1 + rust/src/chat/src/parser/unified.rs | 3 +- rust/src/parser/Cargo.toml | 2 + rust/src/parser/src/unified/kimi_k3.rs | 394 +++++++++++++-- rust/src/parser/src/unified/mod.rs | 9 + rust/src/parser/src/utils.rs | 197 +++++--- rust/src/parser/src/utils/marker.rs | 660 +++++++++++++++++++++++++ 8 files changed, 1135 insertions(+), 133 deletions(-) create mode 100644 rust/src/parser/src/utils/marker.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index d6143f941b..bd5a63e81d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5964,8 +5964,10 @@ dependencies = [ name = "vllm-parser" version = "0.1.0" dependencies = [ + "auto_impl", "criterion", "easy-ext", + "educe", "expect-test", "futures", "openai-protocol", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 0002796d40..a2e6d0c1ed 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,6 +31,7 @@ async-openai = { version = "0.33.1", default-features = false, features = ["nati async-trait = "0.1.89" asynk-strim-attr = "0.1.0" auto_enums = { version = "0.8.9", features = ["tokio1"] } +auto_impl = "1.3.0" axum = "0.8.8" base64 = "0.22.1" bytemuck = { version = "1.25.0", features = ["extern_crate_alloc"] } diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 5ab830150a..52cabaf162 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -141,7 +141,8 @@ mod tests { let tokenizer = TestTokenizer::new() .with_regular_token("<|open|>", 1001) .with_regular_token("<|close|>", 1002) - .with_regular_token("<|sep|>", 1003); + .with_regular_token("<|sep|>", 1003) + .with_regular_token("<|end_of_msg|>", 1004); assert!(factory.contains(names::KIMI_K3)); assert_eq!( diff --git a/rust/src/parser/Cargo.toml b/rust/src/parser/Cargo.toml index 25546cb1df..3086de03e9 100644 --- a/rust/src/parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -8,7 +8,9 @@ license.workspace = true test-util = [] [dependencies] +auto_impl.workspace = true easy-ext.workspace = true +educe.workspace = true serde.workspace = true serde_json.workspace = true serde_with.workspace = true diff --git a/rust/src/parser/src/unified/kimi_k3.rs b/rust/src/parser/src/unified/kimi_k3.rs index 08ad91193c..d6e32463c1 100644 --- a/rust/src/parser/src/unified/kimi_k3.rs +++ b/rust/src/parser/src/unified/kimi_k3.rs @@ -26,6 +26,14 @@ //! starts *inside* that channel without re-emitting the open tag; //! [`UnifiedParser::initialize`] detects this from the prompt token IDs. //! +//! Channel markers are structural only when their `<|open|>` / `<|close|>` / +//! `<|sep|>` bytes come from the dedicated special tokens. The model can spell +//! the same text with ordinary BPE tokens (for example while quoting a +//! configuration snippet in its reasoning); such text stays content in the +//! current channel. The parser therefore expects token-attributed input from +//! the incremental detokenizer: fed unattributed text, it treats every marker +//! as content. Channel names such as `think` are ordinary text either way. +//! //! Argument decoding mirrors the renderer's type tagging (inverse encoding): //! `type="string"` values pass the raw text through, other types are //! JSON-decoded, and a raw `json` block is passed through unmodified. Attribute @@ -34,29 +42,33 @@ //! Known limitation (shared with the Python parser): string argument and //! response bodies are emitted raw, so a value that literally contains //! `<|close|>argument<|sep|>` or `<|close|>response<|sep|>` is -//! indistinguishable from a real closing marker. +//! indistinguishable from a real closing marker. With token attribution this +//! now only applies to `argument` and `json` blocks inside a call body, which +//! are still parsed from text. mod structural_tag; pub use structural_tag::KimiK3StructuralTagBuilder; use serde_json::{Map, Value}; -use vllm_tokenizer::{DecodedText, DynTokenizer}; +use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer}; use winnow::ascii::{multispace0 as ws0, multispace1 as ws1}; use winnow::combinator::{alt, delimited, eof, preceded, repeat, seq, terminated}; use winnow::error::{ContextError, ErrMode, ModalResult, StrContext}; use winnow::prelude::*; -use winnow::stream::Partial; use winnow::token::{literal, rest, take_till, take_until, take_while}; use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER; -use super::{Result, UnifiedParser, UnifiedParserOutput, token_id}; +use super::{Result, UnifiedParser, UnifiedParserOutput, special_token}; use crate::output_grammar::{ self, BuiltOutputGrammar, OutputGrammarContext, visible_format_from_builder, }; use crate::tool::{Tool, ToolCallDelta}; use crate::unified::parsing_failed; -use crate::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker}; +use crate::utils::{ + Attributed, AttributedExt as _, Marker, MarkerScanState, SpecialToken, + parse_buffered_event_attributed, safe_text_len_mul, take_until_marker, +}; const OPEN: &str = "<|open|>"; const CLOSE: &str = "<|close|>"; @@ -70,31 +82,82 @@ const RESPONSE_CLOSE: &str = "<|close|>response<|sep|>"; const TOOLS_OPEN: &str = "<|open|>tools<|sep|>"; const TOOLS_CLOSE: &str = "<|close|>tools<|sep|>"; const MESSAGE_CLOSE: &str = "<|close|>message<|sep|>"; -const CALL_OPEN: &str = "<|open|>call"; const CALL_CLOSE: &str = "<|close|>call<|sep|>"; const ARG_OPEN: &str = "<|open|>argument"; const ARG_CLOSE: &str = "<|close|>argument<|sep|>"; const JSON_OPEN: &str = "<|open|>json"; const JSON_CLOSE: &str = "<|close|>json<|sep|>"; -const IDLE_MARKERS: &[&str] = &[ - THINK_OPEN, - RESPONSE_OPEN, - TOOLS_OPEN, - MESSAGE_CLOSE, - END_OF_MSG, -]; -const REASONING_MARKERS: &[&str] = &[THINK_CLOSE, END_OF_MSG]; -const RESPONSE_MARKERS: &[&str] = &[RESPONSE_CLOSE, TOOLS_OPEN, MESSAGE_CLOSE, END_OF_MSG]; -const EPILOGUE_MARKERS: &[&str] = &[TOOLS_OPEN, MESSAGE_CLOSE, END_OF_MSG]; -const TOOLS_MARKERS: &[&str] = &[CALL_OPEN, TOOLS_CLOSE, MESSAGE_CLOSE, END_OF_MSG]; - /// Channel tags are a couple of text tokens; longer `<|open|>…<|sep|>` spans in /// the prompt tail (attribute-bearing message opens, message bodies) never name /// a generation channel. const MAX_PREFILL_TAG_TOKENS: usize = 8; -type KimiK3Input<'i> = Partial<&'i str>; +type KimiK3Input<'i> = Attributed<'i, Markers>; + +/// The K3 structural special tokens, resolved once from the active tokenizer. +struct SpecialTokens { + open: SpecialToken, + close: SpecialToken, + sep: SpecialToken, + end_of_msg: SpecialToken, +} + +impl SpecialTokens { + fn new(tokenizer: &dyn Tokenizer) -> Result { + Ok(Self { + open: special_token(tokenizer, OPEN)?, + close: special_token(tokenizer, CLOSE)?, + sep: special_token(tokenizer, SEP)?, + end_of_msg: special_token(tokenizer, END_OF_MSG)?, + }) + } +} + +/// The K3 channel markers, with their special-token segments guarded by the +/// active tokenizer's IDs. +/// +/// One definition serves both the `alt` branches that consume a marker and the +/// safe-text scans that stop in front of one, so the two can never disagree on +/// whether a spelling is structural. +#[derive(Debug)] +struct Markers { + think_open: Marker, + think_close: Marker, + response_open: Marker, + response_close: Marker, + tools_open: Marker, + tools_close: Marker, + message_close: Marker, + end_of_msg: Marker, + /// `<|open|>call`; the tag attributes and the terminating `<|sep|>` follow. + call_open: Marker, + call_close: Marker, + sep: Marker, +} + +impl Markers { + fn new(tokens: &SpecialTokens) -> Self { + let open = + |tag: &str| Marker::special(&tokens.open).then_text(tag).then_special(&tokens.sep); + let close = + |tag: &str| Marker::special(&tokens.close).then_text(tag).then_special(&tokens.sep); + + Self { + think_open: open("think"), + think_close: close("think"), + response_open: open("response"), + response_close: close("response"), + tools_open: open("tools"), + tools_close: close("tools"), + message_close: close("message"), + end_of_msg: Marker::special(&tokens.end_of_msg), + call_open: Marker::special(&tokens.open).then_text("call"), + call_close: close("call"), + sep: Marker::special(&tokens.sep), + } + } +} #[derive(Debug, Clone, PartialEq, Eq)] enum KimiK3Event { @@ -148,26 +211,25 @@ enum KimiK3Mode { pub struct KimiK3UnifiedParser { buffer: DecodedText, mode: KimiK3Mode, + markers: Markers, /// Number of calls emitted in the current response. emitted_call_count: usize, tokenizer: DynTokenizer, - open_token_id: u32, - sep_token_id: u32, + special_tokens: SpecialTokens, } impl KimiK3UnifiedParser { - /// Create a Kimi K3 parser. + /// Create a Kimi K3 parser expecting token-attributed input. pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result { - let open_token_id = token_id(tokenizer.as_ref(), OPEN)?; - let sep_token_id = token_id(tokenizer.as_ref(), SEP)?; + let special_tokens = SpecialTokens::new(tokenizer.as_ref())?; Ok(Self { buffer: DecodedText::default(), mode: KimiK3Mode::default(), + markers: Markers::new(&special_tokens), emitted_call_count: 0, tokenizer, - open_token_id, - sep_token_id, + special_tokens, }) } @@ -180,11 +242,14 @@ impl KimiK3UnifiedParser { fn initialize_mode(&mut self, prompt_token_ids: &[u32]) { self.mode = KimiK3Mode::Idle; - let Some(sep_pos) = prompt_token_ids.iter().rposition(|&id| id == self.sep_token_id) else { + let Some(sep_pos) = + prompt_token_ids.iter().rposition(|&id| id == self.special_tokens.sep.id) + else { return; }; - let Some(open_pos) = - prompt_token_ids[..sep_pos].iter().rposition(|&id| id == self.open_token_id) + let Some(open_pos) = prompt_token_ids[..sep_pos] + .iter() + .rposition(|&id| id == self.special_tokens.open.id) else { return; }; @@ -291,9 +356,11 @@ impl UnifiedParser for KimiK3UnifiedParser { fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> { self.buffer.append(delta); - while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer.text, |input| { - parse_next_kimi_k3_event(input, &mut self.mode) - })? { + while let Some((event, consumed_len)) = + parse_buffered_event_attributed(&self.buffer, &self.markers, |input| { + parse_next_kimi_k3_event(input, &mut self.mode) + })? + { let piece = self.buffer.drain_prefix(consumed_len); self.apply_event(event, piece, output)?; } @@ -347,10 +414,11 @@ fn parse_next_kimi_k3_event( /// Parse an event while waiting for the next channel open. fn parse_idle_event(input: &mut KimiK3Input<'_>) -> ModalResult { + let markers = input.markers(); alt(( - literal(THINK_OPEN).value(KimiK3Event::ThinkOpen), - literal(RESPONSE_OPEN).value(KimiK3Event::ResponseOpen), - literal(TOOLS_OPEN).value(KimiK3Event::ToolsOpen), + markers.think_open.value(KimiK3Event::ThinkOpen), + markers.response_open.value(KimiK3Event::ResponseOpen), + markers.tools_open.value(KimiK3Event::ToolsOpen), message_end_event, safe_idle_text_event, )) @@ -359,11 +427,12 @@ fn parse_idle_event(input: &mut KimiK3Input<'_>) -> ModalResult { /// Parse an event inside the `think` channel. fn parse_reasoning_event(input: &mut KimiK3Input<'_>) -> ModalResult { + let markers = input.markers(); alt(( - literal(THINK_CLOSE).value(KimiK3Event::ThinkClose), + markers.think_close.value(KimiK3Event::ThinkClose), // `<|end_of_msg|>` can reach the parser under `ignore_eos` or // `include_stop_str_in_output`; never leak it into reasoning. - literal(END_OF_MSG).value(KimiK3Event::MessageEnd), + markers.end_of_msg.value(KimiK3Event::MessageEnd), safe_reasoning_event, )) .parse_next(input) @@ -371,10 +440,11 @@ fn parse_reasoning_event(input: &mut KimiK3Input<'_>) -> ModalResult) -> ModalResult { + let markers = input.markers(); alt(( - literal(RESPONSE_CLOSE).value(KimiK3Event::ResponseClose), + markers.response_close.value(KimiK3Event::ResponseClose), // The response body also implicitly ends at a `tools` channel. - literal(TOOLS_OPEN).value(KimiK3Event::ToolsOpen), + markers.tools_open.value(KimiK3Event::ToolsOpen), message_end_event, safe_response_text_event, )) @@ -383,8 +453,9 @@ fn parse_response_event(input: &mut KimiK3Input<'_>) -> ModalResult /// Parse an event after the response channel closed. fn parse_epilogue_event(input: &mut KimiK3Input<'_>) -> ModalResult { + let markers = input.markers(); alt(( - literal(TOOLS_OPEN).value(KimiK3Event::ToolsOpen), + markers.tools_open.value(KimiK3Event::ToolsOpen), message_end_event, skip_epilogue_noise_event, )) @@ -393,9 +464,10 @@ fn parse_epilogue_event(input: &mut KimiK3Input<'_>) -> ModalResult /// Parse an event inside the `tools` channel, between `call` blocks. fn parse_tools_event(input: &mut KimiK3Input<'_>) -> ModalResult { + let markers = input.markers(); alt(( call_open_event, - literal(TOOLS_CLOSE).value(KimiK3Event::ToolsClose), + markers.tools_close.value(KimiK3Event::ToolsClose), // Defensive: an unterminated tools channel still ends with the message. message_end_event, skip_tools_noise_event, @@ -405,7 +477,8 @@ fn parse_tools_event(input: &mut KimiK3Input<'_>) -> ModalResult { /// Parse a message close or end-of-message marker. fn message_end_event(input: &mut KimiK3Input<'_>) -> ModalResult { - alt((literal(MESSAGE_CLOSE), literal(END_OF_MSG))) + let markers = input.markers(); + alt((&markers.message_close, &markers.end_of_msg)) .value(KimiK3Event::MessageEnd) .parse_next(input) } @@ -417,35 +490,82 @@ fn parse_done_event(input: &mut KimiK3Input<'_>) -> ModalResult { /// Parse safe text while waiting for the next channel marker. fn safe_idle_text_event(input: &mut KimiK3Input<'_>) -> ModalResult { - safe_text_len_mul(input, IDLE_MARKERS).map(|_| KimiK3Event::Text) + let markers = input.markers(); + safe_text_len_mul( + input, + &[ + &markers.think_open, + &markers.response_open, + &markers.tools_open, + &markers.message_close, + &markers.end_of_msg, + ], + ) + .map(|_| KimiK3Event::Text) } /// Parse safe reasoning before the think close marker. fn safe_reasoning_event(input: &mut KimiK3Input<'_>) -> ModalResult { - safe_text_len_mul(input, REASONING_MARKERS).map(|_| KimiK3Event::Reasoning) + let markers = input.markers(); + safe_text_len_mul(input, &[&markers.think_close, &markers.end_of_msg]) + .map(|_| KimiK3Event::Reasoning) } /// Parse safe response text before the next channel marker. fn safe_response_text_event(input: &mut KimiK3Input<'_>) -> ModalResult { - safe_text_len_mul(input, RESPONSE_MARKERS).map(|_| KimiK3Event::Text) + let markers = input.markers(); + safe_text_len_mul( + input, + &[ + &markers.response_close, + &markers.tools_open, + &markers.message_close, + &markers.end_of_msg, + ], + ) + .map(|_| KimiK3Event::Text) } /// Skip non-content noise after the response channel closed. fn skip_epilogue_noise_event(input: &mut KimiK3Input<'_>) -> ModalResult { - safe_text_len_mul(input, EPILOGUE_MARKERS).map(|_| KimiK3Event::Skip) + let markers = input.markers(); + safe_text_len_mul( + input, + &[ + &markers.tools_open, + &markers.message_close, + &markers.end_of_msg, + ], + ) + .map(|_| KimiK3Event::Skip) } /// Skip non-content noise between `call` blocks. fn skip_tools_noise_event(input: &mut KimiK3Input<'_>) -> ModalResult { - safe_text_len_mul(input, TOOLS_MARKERS).map(|_| KimiK3Event::Skip) + let markers = input.markers(); + safe_text_len_mul( + input, + &[ + &markers.call_open, + &markers.tools_close, + &markers.message_close, + &markers.end_of_msg, + ], + ) + .map(|_| KimiK3Event::Skip) } /// Parse a `call` open tag into its tool name and one-based index. +/// +/// The attributes end at the structural `<|sep|>`; an ordinary spelling of +/// `<|sep|>` inside an attribute value stays attribute text. fn call_open_event(input: &mut KimiK3Input<'_>) -> ModalResult { + let markers = input.markers(); + let mut scan = MarkerScanState::default(); let (attrs,) = seq!( - _: literal(CALL_OPEN), - take_until(0.., SEP), - _: literal(SEP), + _: &markers.call_open, + take_until_marker(&markers.sep, &mut scan), + _: &markers.sep, ) .parse_next(input)?; let attrs = parse_tag_attrs(attrs)?; @@ -464,9 +584,10 @@ fn call_body_event( input: &mut KimiK3Input<'_>, scan: &mut MarkerScanState, ) -> ModalResult { + let markers = input.markers(); let (body,) = seq!( - take_until_marker(CALL_CLOSE, scan), - _: literal(CALL_CLOSE), + take_until_marker(&markers.call_close, scan), + _: &markers.call_close, ) .parse_next(input)?; let arguments = parse_call_arguments(body)?; @@ -614,9 +735,11 @@ mod tests { } impl UnifiedParserTestExt for KimiK3UnifiedParser { + /// Parse `chunk` as the detokenizer would deliver it: markers spelled in + /// full are structural special tokens, everything else ordinary text. fn parse_chunk(&mut self, chunk: &str) -> super::Result { let mut output = UnifiedParserOutput::default(); - self.parse_into(DecodedText::unattributed(chunk), &mut output)?; + self.parse_into(decode_ids(&structural(chunk)), &mut output)?; Ok(output) } @@ -665,14 +788,41 @@ mod tests { } } + /// The parser as constructed in production: markers must be spelled by + /// their special tokens. fn test_parser() -> KimiK3UnifiedParser { KimiK3UnifiedParser::new(&[], Arc::new(tokenizer())).unwrap() } + /// Run token IDs through the real incremental decoder to obtain attributed + /// input, exactly as the frontend does. + fn decode_ids(ids: &[u32]) -> DecodedText { + let tokenizer = tokenizer(); + let mut decoder = tokenizer.create_decode_stream(&[], false, 0); + for &id in ids { + decoder.push_token(id).unwrap(); + } + decoder.flush(None).unwrap().1 + } + + /// Encode with the special tokens, so markers are structural. + fn structural(text: &str) -> Vec { + tokenizer().encode(text, false).unwrap() + } + + /// Encode as ordinary bytes only, so marker spellings are content. + fn ordinary(text: &str) -> Vec { + tokenizer().encode_ordinary(text).unwrap() + } + + /// Stream the structural encoding of the concatenated `chunks`, split at the + /// chunk boundaries, so a marker split across chunks keeps its token anchor. fn collect_stream(parser: &mut KimiK3UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput { + let mut rest = decode_ids(&structural(&chunks.concat())); let mut output = UnifiedParserOutput::default(); for chunk in chunks { - output.append(parser.parse_chunk(chunk).unwrap()); + let piece = rest.drain_prefix(chunk.len()); + parser.parse_into(piece, &mut output).unwrap(); } output.append(parser.finish().unwrap()); output @@ -1136,4 +1286,140 @@ mod tests { assert_eq!(raw, "<|close|>resp"); } + + // --- Ordinary marker spellings ------------------------------------------ + + fn start_in_reasoning(parser: &mut KimiK3UnifiedParser) { + parser.initialize(&structural(THINK_OPEN)).unwrap(); + } + + fn parse_decoded( + parser: &mut KimiK3UnifiedParser, + decoded: DecodedText, + ) -> UnifiedParserOutput { + let mut output = UnifiedParserOutput::default(); + parser.parse_into(decoded, &mut output).unwrap(); + output.append(parser.finish().unwrap()); + output + } + + #[test] + fn kimi_k3_markers_spell_the_documented_strings() { + let special_tokens = super::SpecialTokens::new(&tokenizer()).unwrap(); + let markers = super::Markers::new(&special_tokens); + + assert_eq!(markers.think_open.as_str(), THINK_OPEN); + assert_eq!(markers.think_close.as_str(), THINK_CLOSE); + assert_eq!(markers.response_open.as_str(), RESPONSE_OPEN); + assert_eq!(markers.response_close.as_str(), RESPONSE_CLOSE); + assert_eq!(markers.tools_open.as_str(), TOOLS_OPEN); + assert_eq!(markers.tools_close.as_str(), TOOLS_CLOSE); + assert_eq!(markers.message_close.as_str(), super::MESSAGE_CLOSE); + assert_eq!(markers.call_close.as_str(), super::CALL_CLOSE); + assert_eq!(markers.end_of_msg.as_str(), END_OF_MSG); + } + + #[test] + fn kimi_k3_ordinary_close_marker_text_stays_in_reasoning() { + // The production incident: the model spells `<|close|>think<|sep|>` with + // ordinary tokens while reasoning, then really closes the channel later. + let mut ids = ordinary(THINK_CLOSE); + ids.extend(structural(&format!( + " after{THINK_CLOSE}{RESPONSE_OPEN}answer{RESPONSE_CLOSE}<|close|>message{SEP}" + ))); + let decoded = decode_ids(&ids); + assert!( + decoded.text.starts_with(THINK_CLOSE), + "same bytes as the structural marker" + ); + + let mut parser = test_parser(); + start_in_reasoning(&mut parser); + let output = parse_decoded(&mut parser, decoded); + + assert_eq!(output.reasoning_text(), format!("{THINK_CLOSE} after")); + assert_eq!(output.normal_text(), "answer"); + } + + #[test] + fn kimi_k3_ordinary_open_marker_text_stays_in_reasoning() { + let mut ids = structural("config: "); + ids.extend(ordinary(THINK_OPEN)); + ids.extend(structural(&format!( + "{THINK_CLOSE}{RESPONSE_OPEN}ok{RESPONSE_CLOSE}" + ))); + + let mut parser = test_parser(); + start_in_reasoning(&mut parser); + let output = parse_decoded(&mut parser, decode_ids(&ids)); + + assert_eq!(output.reasoning_text(), format!("config: {THINK_OPEN}")); + assert_eq!(output.normal_text(), "ok"); + } + + #[test] + fn kimi_k3_attributed_streaming_splits_at_every_char_boundary() { + // A lookalike close inside reasoning, then genuine markers and a call. + let mut ids = structural(THINK_OPEN); + ids.extend(ordinary(THINK_CLOSE)); + ids.extend(structural(&format!( + " step{THINK_CLOSE}{RESPONSE_OPEN}the answer{RESPONSE_CLOSE}{TOOLS_OPEN}{}{TOOLS_CLOSE}<|close|>message{SEP}", + call("tool=\"calc\" index=\"1\"", &arg("x", "number", "42")), + ))); + let full = decode_ids(&ids); + + for split in (0..=full.text.len()).filter(|&index| full.text.is_char_boundary(index)) { + let mut rest = full.clone(); + let head = rest.drain_prefix(split); + let mut parser = test_parser(); + let mut output = UnifiedParserOutput::default(); + parser.parse_into(head, &mut output).unwrap(); + parser.parse_into(rest, &mut output).unwrap(); + output.append(parser.finish().unwrap()); + + assert_eq!( + output.reasoning_text(), + format!("{THINK_CLOSE} step"), + "split {split}" + ); + assert_eq!(output.normal_text(), "the answer", "split {split}"); + assert_eq!( + first_call(&output).arguments, + r#"{"x":42}"#, + "split {split}" + ); + } + } + + #[test] + fn kimi_k3_incomplete_utf8_before_structural_close_still_closes() { + // A stray incomplete byte right before the close marker: the decoder + // anchors the special token at its own spelling, so the boundary holds. + let mut ids = vec![0xe4]; + ids.extend(structural(&format!( + "{THINK_CLOSE}{RESPONSE_OPEN}ok{RESPONSE_CLOSE}" + ))); + + let mut parser = test_parser(); + start_in_reasoning(&mut parser); + let output = parse_decoded(&mut parser, decode_ids(&ids)); + + assert_eq!(output.reasoning_text(), "\u{FFFD}"); + assert_eq!(output.normal_text(), "ok"); + } + + #[test] + fn kimi_k3_ordinary_sep_text_inside_call_attributes_stays_attribute_text() { + let mut ids = structural(&format!("{TOOLS_OPEN}{OPEN}call tool=\"a")); + ids.extend(ordinary(SEP)); + ids.extend(structural(&format!( + "b\" index=\"1\"{SEP}{}<|close|>call{SEP}{TOOLS_CLOSE}", + arg("x", "number", "1") + ))); + + let output = parse_decoded(&mut test_parser(), decode_ids(&ids)); + + assert_eq!(first_call(&output).name, Some(format!("a{SEP}b"))); + assert_eq!(first_call(&output).arguments, r#"{"x":1}"#); + } } diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index e65f2de087..c621075a02 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -21,6 +21,7 @@ use vllm_tokenizer::{DecodedText, DynTokenizer}; use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext}; use crate::reasoning::ReasoningError; use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput}; +use crate::utils::SpecialToken; /// Result alias for unified parser operations. pub type Result = std::result::Result; @@ -288,3 +289,11 @@ fn token_id(tokenizer: &dyn vllm_tokenizer::Tokenizer, token: &str) -> Result Result { + Ok(SpecialToken { + id: token_id(tokenizer, token)?, + text: token.to_string(), + }) +} diff --git a/rust/src/parser/src/utils.rs b/rust/src/parser/src/utils.rs index e8ef70a861..1324e297fb 100644 --- a/rust/src/parser/src/utils.rs +++ b/rust/src/parser/src/utils.rs @@ -3,14 +3,23 @@ //! Shared helpers for streaming parsers. +use std::fmt::Debug; + +use vllm_tokenizer::DecodedText; use winnow::Parser; use winnow::error::{ContextError, ErrMode, ModalResult, Needed, StrContext, StrContextValue}; -use winnow::stream::{FindSlice, Offset, Partial, Stream}; +use winnow::stream::{Partial, Stream}; use crate::tool::{Result, ToolParserError}; +pub(crate) mod marker; pub(crate) mod recursion; +pub use marker::{ + Attributed, AttributedExt, Marker, MarkerLike, MarkerStream, SpecialToken, + attributed_with_markers, +}; + /// Return the byte length of the longest proper prefix of `token` that is also /// a suffix of `buffer`. /// @@ -48,48 +57,81 @@ pub fn partial_prefix_len(buffer: &str, token: &str) -> usize { 0 } +/// Where the next marker may be, in the coordinate system of +/// [`MarkerStream::offset`]. +enum Scan { + /// A complete marker starts at this offset. + Found(usize), + /// A marker may start at this offset but needs more input to decide; text + /// from here on must be held back. + Pending(usize), + /// Every candidate before this offset (the end of the input) was rejected. + Clear(usize), +} + +/// Scan for the earliest of `markers` at or after `from`. +/// +/// Candidates come from the stream ([`MarkerStream::next_candidate`]): text +/// occurrences for plain markers, visible anchors for markers guarded by a +/// special token. Each candidate is decided by probing the marker's own parser +/// on a cloned cursor, so the scanner and the consuming `alt` branch can never +/// disagree; a rejected candidate (an ordinary lookalike) is skipped and the +/// scan continues. +fn scan_markers<'i, I: MarkerStream<'i>, M: MarkerLike>( + input: &I, + markers: &[M], + from: usize, +) -> ModalResult { + let start = input.offset(); + let mut from = from.max(start); + while let Some(at) = input.next_candidate(markers, from) { + for marker in markers { + let mut probe = input.clone(); + probe.next_slice(at - start); + match marker.as_marker().parse_next(&mut probe) { + Ok(_) => return Ok(Scan::Found(at)), + Err(ErrMode::Backtrack(_)) => {} + Err(ErrMode::Incomplete(_)) => return Ok(Scan::Pending(at)), + Err(error @ ErrMode::Cut(_)) => return Err(error), + } + } + from = at + 1; + } + Ok(Scan::Clear(start + input.eof_offset())) +} + /// Parse a safe text run before the next marker. /// This is the single-marker variant of [`safe_text_len_mul`]. /// /// Returns the text length in bytes, and advances the input. -pub fn safe_text_len(input: &mut Partial<&str>, marker: &str) -> ModalResult { - let text = **input; - if text.is_empty() { - return incomplete(); - } - - if let Some(start_idx) = text.find(marker) { - input.next_slice(start_idx); - return Ok(start_idx); - } - - let keep_len = partial_prefix_len(text, marker); - let emit_len = text.len().saturating_sub(keep_len); - if emit_len == 0 { - return incomplete(); - } - - input.next_slice(emit_len); - Ok(emit_len) +pub fn safe_text_len<'i, I: MarkerStream<'i>, M: MarkerLike>( + input: &mut I, + marker: M, +) -> ModalResult { + safe_text_len_mul(input, &[marker]) } /// Parse a safe text run before the earliest next marker. /// This is the multi-marker variant of [`safe_text_len`]. /// +/// Text that may still grow into a marker is held back: a partial spelling at +/// the end of the input for plain markers, and only a genuine special token's +/// incomplete marker for guarded ones. +/// /// Returns the text length in bytes, and advances the input. -pub fn safe_text_len_mul(input: &mut Partial<&str>, markers: &[&str]) -> ModalResult { - let text = **input; - if text.is_empty() { +pub fn safe_text_len_mul<'i, I: MarkerStream<'i>, M: MarkerLike>( + input: &mut I, + markers: &[M], +) -> ModalResult { + if input.eof_offset() == 0 { return incomplete(); } - if let Some(start_idx) = find_slice_mul(text, markers) { - input.next_slice(start_idx); - return Ok(start_idx); - } - - let keep_len = markers.iter().map(|marker| partial_prefix_len(text, marker)).max().unwrap_or(0); - let emit_len = text.len().saturating_sub(keep_len); + let start = input.offset(); + let stop = match scan_markers(input, markers, start)? { + Scan::Found(at) | Scan::Pending(at) | Scan::Clear(at) => at, + }; + let emit_len = stop - start; if emit_len == 0 { return incomplete(); } @@ -98,23 +140,11 @@ pub fn safe_text_len_mul(input: &mut Partial<&str>, markers: &[&str]) -> ModalRe Ok(emit_len) } -#[inline(always)] -fn find_slice_mul(text: &str, markers: &[&str]) -> Option { - let range = match markers { - // Use the fast specialized `winnow::stream::FindSlice` impl for 1-3 markers. - [first] => text.find_slice(*first), - [first, second] => text.find_slice((*first, *second)), - [first, second, third] => text.find_slice((*first, *second, *third)), - // Fall back to a linear scan for 4+ markers. - _ => return markers.iter().filter_map(|marker| text.find(marker)).min(), - }; - range.map(|range| range.start) -} - /// Streaming scan state for a buffered marker search [`take_until_marker`], /// so that we don't have to rescan the whole buffered prefix when resuming. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct MarkerScanState { + /// Offset relative to the cursor at call time from which the scan resumes. scan_start: usize, } @@ -128,55 +158,47 @@ impl MarkerScanState { /// /// This is the streaming-buffered variant of `winnow::token::take_until(0.., /// marker)`: it returns the slice before `marker` and leaves `marker` for the -/// caller to consume. On incomplete input, it stores the earliest byte offset -/// that can still match `marker` and returns `Incomplete` without consuming -/// input, so the next parse can avoid rescanning the whole buffered prefix. +/// caller to consume. On incomplete input, it stores the earliest offset that +/// can still start `marker` and returns `Incomplete` without consuming input, +/// so the next parse can avoid rescanning the whole buffered prefix. /// /// Use this for outer parser states that keep the full buffered input across /// chunks while waiting for a closing marker. Plain `take_until` is still a /// better fit for one-shot parsers over a complete body, and for `1..` cases /// where an empty slice before the marker should be rejected. -pub fn take_until_marker<'i, 'a>( - marker: &'a str, +pub fn take_until_marker<'i, 'a, I: MarkerStream<'i>, M: MarkerLike + 'a>( + marker: M, state: &'a mut MarkerScanState, -) -> impl Parser, &'i str, ErrMode> + 'a { - move |input: &mut Partial<&'i str>| take_until_marker_(input, marker, state) +) -> impl Parser> + 'a { + move |input: &mut I| take_until_marker_(input, &marker, state) } -fn take_until_marker_<'i>( - input: &mut Partial<&'i str>, - marker: &str, +fn take_until_marker_<'i, I: MarkerStream<'i>, M: MarkerLike>( + input: &mut I, + marker: &M, state: &mut MarkerScanState, ) -> ModalResult<&'i str> { - debug_assert!(!marker.is_empty()); + debug_assert!(!marker.as_marker().as_str().is_empty()); - let text = **input; - if text.is_empty() { + if input.eof_offset() == 0 { return incomplete(); } - // Normal updates store a char boundary; this keeps stale or misused state from panicking. - let scan_start = floor_char_boundary(text, state.scan_start); - - if let Some(offset) = text[scan_start..].find(marker) { - let marker_start = scan_start + offset; - let body = &text[..marker_start]; - input.next_slice(marker_start); - state.reset(); - return Ok(body); + let start = input.offset(); + match scan_markers( + input, + std::slice::from_ref(marker), + start + state.scan_start, + )? { + Scan::Found(at) => { + state.reset(); + Ok(input.next_slice(at - start)) + } + Scan::Pending(at) | Scan::Clear(at) => { + state.scan_start = at - start; + incomplete() + } } - - let keep_len = partial_prefix_len(text, marker); - state.scan_start = text.len() - keep_len; - incomplete() -} - -fn floor_char_boundary(text: &str, index: usize) -> usize { - let mut index = index.min(text.len()); - while !text.is_char_boundary(index) { - index -= 1; - } - index } /// Streaming lexical state for a top-level JSON object. @@ -394,7 +416,26 @@ pub fn parse_buffered_event( buffer: &str, parse: impl FnOnce(&mut Partial<&str>) -> ModalResult, ) -> Result> { - let mut input = Partial::new(buffer); + parse_buffered_stream_event(Partial::new(buffer), parse) +} + +/// Parse one event from a buffered [`DecodedText`], for token-aware parsers. +/// +/// Same contract as [`parse_buffered_event`]; the input also carries the +/// buffer's token anchors and the parser's `markers` ([`Attributed`]). +pub fn parse_buffered_event_attributed( + buffer: &DecodedText, + markers: &M, + parse: impl FnOnce(&mut Attributed<'_, M>) -> ModalResult, +) -> Result> { + parse_buffered_stream_event(attributed_with_markers(buffer, markers), parse) +} + +fn parse_buffered_stream_event<'i, I: MarkerStream<'i>, E>( + mut input: I, + parse: impl FnOnce(&mut I) -> ModalResult, +) -> Result> { + let buffer = input.remaining(); let checkpoint = input.checkpoint(); let event = match parse(&mut input) { Ok(event) => event, diff --git a/rust/src/parser/src/utils/marker.rs b/rust/src/parser/src/utils/marker.rs new file mode 100644 index 0000000000..cc36f8e6bf --- /dev/null +++ b/rust/src/parser/src/utils/marker.rs @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Token-aware marker matching for streaming parsers. +//! +//! Model protocols delimit channels with dedicated special tokens such as +//! `<|close|>` and `<|sep|>`. Ordinary BPE tokens can decode to the very same +//! spelling, so matching a marker by text alone mistakes model-written content +//! for structure. The helpers here keep text as the parse axis and consult the +//! generated-token attribution carried by [`DecodedText`] only where a marker's +//! bytes must have come from a special token. +//! +//! - [`Attributed`] is a winnow input over one `DecodedText` buffer: the text +//! plus its buffer-local token anchors, composed from winnow's own stream +//! wrappers so every existing combinator keeps working. +//! - [`Marker`] is a fixed marker string whose special-token segments are +//! *guarded*: they match only when the expected special token produced them. +//! A marker without guards matches by spelling alone, which is exactly what a +//! plain `&str` marker is; [`MarkerRef`] is the borrowed view shared by both. +//! One marker definition is both the parser that consumes the marker and the +//! definition the safe-text scanner in the parent module stops in front of, +//! so the two never disagree. +//! - [`MarkerStream`] abstracts over `Partial<&str>` (text-only parsers) and +//! [`Attributed`], letting one generic scanner serve both. + +use std::fmt::Debug; +use std::ops::Range; + +use auto_impl::auto_impl; +use educe::Educe; +use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution}; +use winnow::Parser; +use winnow::error::{ContextError, ErrMode, ModalResult}; +use winnow::stream::{ + Compare, FindSlice, LocatingSlice, Location, Partial, Stateful, Stream, StreamIsPartial, +}; +use winnow::token::literal; + +use super::partial_prefix_len; + +/// Streaming winnow input over one [`DecodedText`] buffer. +/// +/// `LocatingSlice` reports the cursor's byte offset from the buffer start, which +/// is the coordinate system of the anchors; `Partial` keeps the incomplete-input +/// semantics streaming parsers rely on; `Stateful` carries the immutable +/// [`AttributedState`]. winnow forwards `Compare`, `FindSlice`, `Location`, and +/// `UpdateSlice` through all three, so `literal`, `alt`, `seq!`, `take_until`, +/// and `rest` work unchanged. +pub type Attributed<'i, M = ()> = Stateful>, AttributedState<'i, M>>; + +/// Immutable state of an [`Attributed`] input. +#[derive(Debug, Educe)] +#[educe(Clone, Copy)] +pub struct AttributedState<'i, M = ()> { + /// Token anchors of the buffer, sorted by byte offset. + anchors: &'i [TokenAttribution], + /// The parser's markers; see [`AttributedExt::markers`]. + markers: &'i M, +} + +#[easy_ext::ext(AttributedExt)] +pub impl<'i, M> Attributed<'i, M> { + /// The parser's markers, reachable from every grammar function through the + /// input instead of an extra parameter. + fn markers(&self) -> &'i M { + self.state.markers + } +} + +/// Create an [`Attributed`] input over `buffer`, carrying `markers`. +pub fn attributed_with_markers<'i, M>( + buffer: &'i DecodedText, + markers: &'i M, +) -> Attributed<'i, M> { + Stateful { + input: Partial::new(LocatingSlice::new(buffer.text.as_str())), + state: AttributedState { + anchors: &buffer.attributions, + markers, + }, + } +} + +/// A `&str`-sliced streaming input that can say where markers may begin. +/// +/// Implemented for `Partial<&str>` (text-only parsers, no anchors) and +/// [`Attributed`]. +pub trait MarkerStream<'i>: + Stream + StreamIsPartial + Clone + for<'m> Compare<&'m str> +{ + /// Remaining input from the cursor. + fn remaining(&self) -> &'i str; + + /// Byte offset of the cursor in the coordinate system of + /// [`Self::anchors`] and of resumable scan state. + /// + /// `Partial<&str>` tracks no origin and reports 0, so its offsets are + /// relative to the cursor at the time of the call; [`Attributed`] reports + /// the offset from the buffer start. + fn offset(&self) -> usize; + + /// Token anchors of the buffer, sorted by byte offset; empty for text-only + /// input. + fn anchors(&self) -> &'i [TokenAttribution]; + + /// Earliest offset at or after `from` where one of `markers` may begin. + /// + /// A guarded marker can only begin at a visible anchor of its first special + /// token, so text-only input never yields one. Any other marker begins at a + /// complete occurrence of its spelling, or at a proper prefix of it that + /// ends the input. + fn next_candidate(&self, markers: &[M], from: usize) -> Option { + let start = self.offset(); + let from = from.max(start); + let markers = || markers.iter().map(MarkerLike::as_marker); + + let anchored = markers().any(|marker| marker.first_token_id().is_some()).then(|| { + visible_in(self.anchors(), from..usize::MAX).find_map(|(at, token_id)| { + markers().any(|marker| marker.first_token_id() == Some(token_id)).then_some(at) + }) + }); + let by_text = text_candidate( + self.remaining(), + markers() + .filter(|marker| marker.first_token_id().is_none()) + .map(MarkerRef::as_str), + from - start, + ) + .map(|at| start + at); + + match (anchored.flatten(), by_text) { + (Some(anchored), Some(by_text)) => Some(anchored.min(by_text)), + (anchored, by_text) => anchored.or(by_text), + } + } +} + +impl<'i> MarkerStream<'i> for Partial<&'i str> { + fn remaining(&self) -> &'i str { + **self + } + + fn offset(&self) -> usize { + 0 + } + + fn anchors(&self) -> &'i [TokenAttribution] { + &[] + } +} + +impl<'i, M: Debug> MarkerStream<'i> for Attributed<'i, M> { + fn remaining(&self) -> &'i str { + ****self + } + + fn offset(&self) -> usize { + self.current_token_start() + } + + fn anchors(&self) -> &'i [TokenAttribution] { + self.state.anchors + } +} + +fn anchor_offset(anchor: TokenAnchor) -> usize { + match anchor { + TokenAnchor::Visible { byte_offset } | TokenAnchor::ZeroWidth { byte_offset } => { + byte_offset as usize + } + } +} + +/// Visible anchors within `range`, as `(byte_offset, token_id)`, in generation +/// order. +fn visible_in( + anchors: &[TokenAttribution], + range: Range, +) -> impl Iterator + '_ { + let first = + anchors.partition_point(|attribution| anchor_offset(attribution.anchor) < range.start); + anchors[first..] + .iter() + .take_while(move |attribution| anchor_offset(attribution.anchor) < range.end) + .filter_map(|attribution| match attribution.anchor { + TokenAnchor::Visible { byte_offset } => { + Some((byte_offset as usize, attribution.token_id)) + } + TokenAnchor::ZeroWidth { .. } => None, + }) +} + +/// Return whether `range` is exactly the visible span of the single token +/// `token_id`: one visible anchor at `range.start` with that ID and no other +/// visible anchor inside. Zero-width records are ignored. +/// +/// Special tokens never merge with neighbouring text, so this is the identity +/// test for a special token's spelling. Anchors beyond the available input do +/// not exist yet, so a partially arrived genuine spelling passes and is left to +/// `literal` to report as incomplete. +fn is_single_token(anchors: &[TokenAttribution], range: Range, token_id: u32) -> bool { + let mut visible = visible_in(anchors, range.clone()); + visible.next() == Some((range.start, token_id)) && visible.next().is_none() +} + +/// Earliest text position at or after `from` where one of `spellings` occurs +/// in full, or where a proper prefix of one of them ends the input. +fn text_candidate<'m>( + text: &str, + spellings: impl Iterator + Clone, + from: usize, +) -> Option { + // Round up: a marker starts on a char boundary at or after `from`, and a + // rejected candidate inside a multi-byte char must not be found again. + let from = text.ceil_char_boundary(from); + let tail = &text[from..]; + if tail.is_empty() { + return None; + } + if let Some(at) = find_earliest(tail, spellings.clone()) { + return Some(from + at); + } + let keep_len = spellings.map(|spelling| partial_prefix_len(tail, spelling)).max().unwrap_or(0); + (keep_len > 0).then(|| text.len() - keep_len) +} + +/// Earliest occurrence of any spelling in `text`. +#[inline(always)] +fn find_earliest<'m>(text: &str, mut spellings: impl Iterator) -> Option { + let first = spellings.next()?; + let second = spellings.next(); + let third = second.and_then(|_| spellings.next()); + let fourth = third.and_then(|_| spellings.next()); + // Use the fast specialized `FindSlice` impls for 1-3 spellings, and fall + // back to a linear scan for 4+. + let range = match (second, third, fourth) { + (None, _, _) => text.find_slice(first), + (Some(second), None, _) => text.find_slice((first, second)), + (Some(second), Some(third), None) => text.find_slice((first, second, third)), + (Some(second), Some(third), Some(fourth)) => { + return [first, second, third, fourth] + .into_iter() + .chain(spellings) + .filter_map(|spelling| text.find(spelling)) + .min(); + } + }; + range.map(|range| range.start) +} + +/// A special token's spelling and its ID in the active tokenizer, resolved +/// together so a marker cannot pair one token's spelling with another's ID. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SpecialToken { + pub text: String, + pub id: u32, +} + +/// A fixed marker string whose special-token segments are guarded by token +/// identity. +/// +/// Build it once from tokenizer IDs, then use it both as the parser that +/// consumes the marker (it implements [`Parser`] by reference) and as the +/// definition the safe-text scanner stops in front of. +/// +/// ```ignore +/// let think_close = Marker::special(&close).then_text("think").then_special(&sep); +/// ``` +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Marker { + text: String, + guards: Vec, +} + +/// One special-token segment of a marker. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Guard { + /// Byte range of the spelling within the marker text. + range: Range, + token_id: u32, +} + +impl Marker { + /// A marker matched by spelling alone. + pub fn text(text: impl Into) -> Self { + Self { + text: text.into(), + guards: Vec::new(), + } + } + + /// A marker beginning with the special token `token`. + pub fn special(token: &SpecialToken) -> Self { + Self::text(String::new()).then_special(token) + } + + /// Append ordinary text, matched by spelling. + #[must_use] + pub fn then_text(mut self, text: &str) -> Self { + self.text.push_str(text); + self + } + + /// Append the special token `token`. + #[must_use] + pub fn then_special(mut self, token: &SpecialToken) -> Self { + let start = self.text.len(); + self.text.push_str(&token.text); + self.guards.push(Guard { + range: start..self.text.len(), + token_id: token.id, + }); + self + } + + /// Drop the token-identity guards, so the marker matches by spelling alone, + /// for callers whose input carries no token attribution. + #[must_use] + pub fn without_guards(mut self) -> Self { + self.guards.clear(); + self + } + + /// The full spelling of the marker. + pub fn as_str(&self) -> &str { + &self.text + } +} + +/// Anything the scanning helpers accept as a marker, viewed through +/// [`MarkerRef`]. +/// +/// A plain `str` is a marker without guards, so text-only parsers keep passing +/// `&[&str]` (or `&String`); token-aware parsers pass `&Marker`. +#[auto_impl(&)] +pub trait MarkerLike { + /// Borrow the marker's spelling and guards. + fn as_marker(&self) -> MarkerRef<'_>; +} + +impl MarkerLike for str { + fn as_marker(&self) -> MarkerRef<'_> { + MarkerRef { + text: self, + guards: &[], + } + } +} + +impl MarkerLike for String { + fn as_marker(&self) -> MarkerRef<'_> { + self.as_str().as_marker() + } +} + +impl MarkerLike for Marker { + fn as_marker(&self) -> MarkerRef<'_> { + MarkerRef { + text: &self.text, + guards: &self.guards, + } + } +} + +/// Borrowed view of a marker: its spelling and guarded segments. +/// +/// A plain `&str` is a marker without guards; see [`MarkerLike`]. +#[derive(Clone, Copy, Debug)] +pub struct MarkerRef<'m> { + text: &'m str, + guards: &'m [Guard], +} + +impl<'m> MarkerRef<'m> { + /// The full spelling of the marker. + pub fn as_str(self) -> &'m str { + self.text + } + + /// The special token the marker begins with, if it begins with one. + pub fn first_token_id(self) -> Option { + self.guards + .first() + .filter(|guard| guard.range.start == 0) + .map(|guard| guard.token_id) + } + + /// Parse the marker at the cursor, returning its spelling. + /// + /// Every guard whose first byte is available is checked before the spelling: + /// when those bytes do not come from the expected special token the parser + /// backtracks at once instead of asking for more input, so an ordinary + /// partial lookalike streams as content immediately. A genuine special token + /// split by output holdback still reports incomplete, because its anchor + /// arrived with its first byte. Nothing is consumed on failure. + pub fn parse_next<'i, I: MarkerStream<'i>>(self, input: &mut I) -> ModalResult<&'i str> { + let start = input.offset(); + let available = input.eof_offset(); + let anchors = input.anchors(); + for guard in self.guards.iter().take_while(|guard| guard.range.start < available) { + let range = start + guard.range.start..start + guard.range.end; + if !is_single_token(anchors, range, guard.token_id) { + return Err(ErrMode::Backtrack(ContextError::new())); + } + } + literal(self.text).parse_next(input) + } +} + +impl<'i, I: MarkerStream<'i>> Parser> for &Marker { + fn parse_next(&mut self, input: &mut I) -> ModalResult<&'i str> { + self.as_marker().parse_next(input) + } +} + +#[cfg(test)] +mod tests { + use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution}; + use winnow::Parser; + use winnow::error::ErrMode; + use winnow::stream::{Partial, Stream}; + + use super::{ + Attributed, Marker, MarkerLike, MarkerStream, SpecialToken, attributed_with_markers, + }; + use crate::utils::safe_text_len_mul; + + const CLOSE_ID: u32 = 256; + const SEP_ID: u32 = 258; + + /// Build attributed text from `(piece, token_id)` pairs, one visible anchor + /// at the first byte of each piece. + fn pieces(pieces: &[(&str, u32)]) -> DecodedText { + let mut decoded = DecodedText::default(); + for &(piece, token_id) in pieces { + decoded.attributions.push(TokenAttribution { + token_id, + anchor: TokenAnchor::Visible { + byte_offset: decoded.text.len() as u32, + }, + }); + decoded.text.push_str(piece); + } + decoded + } + + fn token(text: &str, id: u32) -> SpecialToken { + SpecialToken { + text: text.to_string(), + id, + } + } + + /// An input over `buffer` without parser markers. + fn attributed(buffer: &DecodedText) -> Attributed<'_> { + attributed_with_markers(buffer, &()) + } + + fn close() -> Marker { + Marker::special(&token("<|close|>", CLOSE_ID)) + } + + fn think_close() -> Marker { + close().then_text("think").then_special(&token("<|sep|>", SEP_ID)) + } + + /// `<|close|>` spelled by ordinary tokens. + fn ordinary_close() -> Vec<(&'static str, u32)> { + vec![("<", 60), ("|", 124), ("close", 900), ("|", 124), (">", 62)] + } + + #[test] + fn special_marker_accepts_single_token_with_expected_id() { + let buffer = pieces(&[("<|close|>", CLOSE_ID), ("x", 1)]); + let mut input = attributed(&buffer); + + let matched = (&close()).parse_next(&mut input).unwrap(); + + assert_eq!(matched, "<|close|>"); + assert_eq!(input.remaining(), "x"); + } + + #[test] + fn special_marker_rejects_ordinary_tokens_with_same_spelling() { + let mut fixture = ordinary_close(); + fixture.push(("x", 1)); + let buffer = pieces(&fixture); + let mut input = attributed(&buffer); + + let error = (&close()).parse_next(&mut input).unwrap_err(); + + assert!(matches!(error, ErrMode::Backtrack(_))); + assert_eq!(input.remaining(), "<|close|>x", "nothing consumed"); + } + + #[test] + fn special_marker_rejects_single_ordinary_token_with_wrong_id() { + let buffer = pieces(&[("<|close|>", 900)]); + let mut input = attributed(&buffer); + + let error = (&close()).parse_next(&mut input).unwrap_err(); + + assert!(matches!(error, ErrMode::Backtrack(_))); + } + + #[test] + fn special_marker_reports_incomplete_for_split_genuine_spelling() { + // Output holdback delivered only the first bytes; the anchor came with them. + let buffer = pieces(&[("<|clo", CLOSE_ID)]); + let mut input = attributed(&buffer); + + let error = (&close()).parse_next(&mut input).unwrap_err(); + + assert!(matches!(error, ErrMode::Incomplete(_))); + } + + #[test] + fn special_marker_rejects_partial_lookalike_without_waiting() { + let buffer = pieces(&[("<", 60), ("|", 124), ("clo", 901)]); + let mut input = attributed(&buffer); + + let error = (&close()).parse_next(&mut input).unwrap_err(); + + assert!(matches!(error, ErrMode::Backtrack(_))); + } + + #[test] + fn special_marker_ignores_zero_width_records_inside_span() { + let mut buffer = pieces(&[("<|close|>", CLOSE_ID)]); + buffer.attributions.push(TokenAttribution { + token_id: 7, + anchor: TokenAnchor::ZeroWidth { byte_offset: 3 }, + }); + let mut input = attributed(&buffer); + + (&close()).parse_next(&mut input).unwrap(); + } + + #[test] + fn marker_without_guards_matches_spelling_only() { + let buffer = pieces(&ordinary_close()); + let marker = close().without_guards(); + + (&marker).parse_next(&mut attributed(&buffer)).unwrap(); + (&marker).parse_next(&mut Partial::new("<|close|>")).unwrap(); + } + + #[test] + fn guarded_marker_never_matches_text_only_input() { + let error = (&close()).parse_next(&mut Partial::new("<|close|>")).unwrap_err(); + + assert!(matches!(error, ErrMode::Backtrack(_))); + } + + #[test] + fn marker_requires_every_special_segment() { + let marker = think_close(); + let genuine = pieces(&[("<|close|>", CLOSE_ID), ("think", 5), ("<|sep|>", SEP_ID)]); + assert_eq!( + (&marker).parse_next(&mut attributed(&genuine)).unwrap(), + "<|close|>think<|sep|>" + ); + + // Real close, ordinary separator: the strict rule keeps it as content. + let mixed = pieces(&[("<|close|>", CLOSE_ID), ("think", 5), ("<|sep|>", 902)]); + let mut input = attributed(&mixed); + assert!(matches!( + (&marker).parse_next(&mut input).unwrap_err(), + ErrMode::Backtrack(_) + )); + assert_eq!( + input.remaining(), + "<|close|>think<|sep|>", + "nothing consumed" + ); + } + + #[test] + fn marker_first_token_id_requires_leading_special() { + assert_eq!(think_close().as_marker().first_token_id(), Some(CLOSE_ID)); + assert_eq!(Marker::text("").as_marker().first_token_id(), None); + assert_eq!( + Marker::text("x") + .then_special(&token("<|sep|>", SEP_ID)) + .as_marker() + .first_token_id(), + None + ); + assert_eq!("".as_marker().first_token_id(), None); + } + + #[test] + fn attributed_next_candidate_uses_anchors_for_guarded_markers() { + let mut fixture = vec![("say ", 1)]; + fixture.extend(ordinary_close()); // lookalike at 4 + fixture.extend([("think", 5), ("<|sep|>", 903), (" then ", 2)]); + let genuine_at = fixture.iter().map(|(piece, _)| piece.len()).sum::(); + fixture.extend([("<|close|>", CLOSE_ID), ("think", 5), ("<|sep|>", SEP_ID)]); + let buffer = pieces(&fixture); + let input = attributed(&buffer); + + let marker = think_close(); + assert_eq!(input.next_candidate(&[&marker], 0), Some(genuine_at)); + assert_eq!(input.next_candidate(&[&marker], genuine_at + 1), None); + } + + #[test] + fn attributed_next_candidate_falls_back_to_text_for_unguarded_markers() { + let buffer = pieces(&[("a", 1), ("", 2), ("b", 3)]); + let input = attributed(&buffer); + + assert_eq!(input.next_candidate(&[""], 0), Some(1)); + assert_eq!(input.next_candidate(&[""], 0), None); + // Partial spelling at the end of the input is a candidate too. + let buffer = pieces(&[("a", 1), (""], 0), Some(1)); + } + + #[test] + fn text_next_candidate_matches_previous_scan_semantics() { + let input = Partial::new("hello<|tool"); + + assert_eq!( + input.next_candidate(&["<|tool_call>", "<|channel>thought\n"], 0), + Some("hello".len()) + ); + assert_eq!(input.next_candidate(&["<|tool_call>"], 18), None); + assert_eq!( + Partial::new("hello<|tool_call>") + .next_candidate(&["<|tool_call>", ""], 0), + Some(5) + ); + } + + #[test] + fn scan_skips_rejected_text_candidate_starting_with_multibyte_char() { + // A marker whose leading text is found by spelling but whose guard then + // fails: the scan must move past the whole char, not retry inside it. + let marker = Marker::text("é").then_special(&token("<|sep|>", SEP_ID)); + let buffer = pieces(&[("aé", 1), ("<", 60), ("|sep|>", 904), ("b", 2)]); + let mut input = attributed(&buffer); + + let len = safe_text_len_mul(&mut input, &[&marker]).unwrap(); + + assert_eq!(len, buffer.text.len()); + } + + #[test] + fn attributed_offsets_follow_the_cursor() { + let buffer = pieces(&[("ab", 1), ("<|close|>", CLOSE_ID)]); + let mut input = attributed(&buffer); + input.next_slice(2); + + assert_eq!(input.offset(), 2); + assert_eq!(input.remaining(), "<|close|>"); + (&close()).parse_next(&mut input).unwrap(); + assert_eq!(input.offset(), 11); + } +} From ac9126e58aa7bbab1856ba6593ba4d5003fea516 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 1 Oct 2026 13:09:43 +0800 Subject: [PATCH 178/248] [Rust Frontend] Snapshot structural-tag grammars as readable outlines (#59393) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- rust/src/chat/Cargo.toml | 1 + .../grammar_replay/DeepSeek-V3.2-Exp.txt | 26 + .../grammar_replay/DeepSeek-V4-Flash.txt | 26 + .../grammar_replay/DeepSeek-V4.1-Flash.txt | 26 + .../tests/grammar_replay/GLM-4.7-Flash.txt | 18 + .../chat/tests/grammar_replay/GLM-5.2-FP8.txt | 18 + rust/src/chat/tests/grammar_replay/Hy3.txt | 112 +++ .../chat/tests/grammar_replay/Kimi-K2.5.txt | 46 ++ .../src/chat/tests/grammar_replay/Kimi-K3.txt | 67 ++ .../grammar_replay/MiMo-V2.6-Flash-RL.txt | 18 + .../tests/grammar_replay/MiniMax-M2.5.txt | 32 + .../chat/tests/grammar_replay/Qwen3-0.6B.txt | 27 + .../chat/tests/grammar_replay/Qwen3.5-4B.txt | 24 + rust/src/chat/tests/roundtrip.rs | 32 +- rust/src/parser/src/output_grammar.rs | 3 + .../parser/src/output_grammar/test_utils.rs | 676 ++++++++++++++++++ rust/src/parser/src/tool/hy/structural_tag.rs | 148 +++- .../src/unified/kimi_k3/structural_tag.rs | 23 +- 18 files changed, 1290 insertions(+), 33 deletions(-) create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt create mode 100644 rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt create mode 100644 rust/src/chat/tests/grammar_replay/Hy3.txt create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K3.txt create mode 100644 rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt create mode 100644 rust/src/parser/src/output_grammar/test_utils.rs diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index d145fc237a..fdaa5ad114 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -52,6 +52,7 @@ tempfile.workspace = true tokio.workspace = true uuid.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } +vllm-parser = { workspace = true, features = ["test-util"] } vllm-tokenizer = { workspace = true, features = ["test-utils"] } zeromq.workspace = true diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt new file mode 100644 index 0000000000..5cf40f3218 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.txt @@ -0,0 +1,26 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [`<|DSML|function_calls>`] excluding [``, ``] + tag `<|DSML|function_calls>\n` .. `` + tags_with_separator `` at_least_one + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + tag `<|DSML|invoke name="add">\n` deepseek_xml({ y: number, x: number, items: string[] }) `\n` + +# required (from_token_zero) +sequence + tag `` text `` + sequence + `\n\n<|DSML|function_calls>\n` + tags_with_separator `` at_least_one + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + tag `<|DSML|invoke name="add">\n` deepseek_xml({ y: number, x: number, items: string[] }) `\n` + `` + +# named (from_token_zero) +sequence + tag `` text `` + sequence + `\n\n<|DSML|function_calls>\n` + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + `` diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt new file mode 100644 index 0000000000..5e216e8ffc --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.txt @@ -0,0 +1,26 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [`<|DSML|tool_calls>`] excluding [``, ``] + tag `<|DSML|tool_calls>\n` .. `` + tags_with_separator `` at_least_one + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + tag `<|DSML|invoke name="add">\n` deepseek_xml({ y: number, x: number, items: string[] }) `\n` + +# required (from_token_zero) +sequence + tag `` text `` + sequence + `\n\n<|DSML|tool_calls>\n` + tags_with_separator `` at_least_one + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + tag `<|DSML|invoke name="add">\n` deepseek_xml({ y: number, x: number, items: string[] }) `\n` + `` + +# named (from_token_zero) +sequence + tag `` text `` + sequence + `\n\n<|DSML|tool_calls>\n` + tag `<|DSML|invoke name="get_weather">\n` deepseek_xml({ location: string }) `\n` + `` diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt new file mode 100644 index 0000000000..8c92c3a5a3 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.txt @@ -0,0 +1,26 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [`<|DSML| calls>`] excluding [``, ``] + tag `<|DSML| calls>\n` .. `` + tags_with_separator `` at_least_one + tag `<|DSML| invoke name="get_weather">\n` deepseek_v4_1_xml({ location: string }) `\n` + tag `<|DSML| invoke name="add">\n` deepseek_v4_1_xml({ y: number, x: number, items: string[] }) `\n` + +# required (from_token_zero) +sequence + tag `` text `` + sequence + `\n\n<|DSML| calls>\n` + tags_with_separator `` at_least_one + tag `<|DSML| invoke name="get_weather">\n` deepseek_v4_1_xml({ location: string }) `\n` + tag `<|DSML| invoke name="add">\n` deepseek_v4_1_xml({ y: number, x: number, items: string[] }) `\n` + `` + +# named (from_token_zero) +sequence + tag `` text `` + sequence + `\n\n<|DSML| calls>\n` + tag `<|DSML| invoke name="get_weather">\n` deepseek_v4_1_xml({ location: string }) `\n` + `` diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt new file mode 100644 index 0000000000..58b4bef2da --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.txt @@ -0,0 +1,18 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [``] excluding [``, ``, ``, ``, ``, ``, ``] + tag `get_weather` glm_xml({ location: string }) `` + tag `add` glm_xml({ y: number, x: number, items: string[] }) `` + +# required (from_token_zero) +sequence + tag `` text `` + triggered_tags [``] excluding [``, ``, ``, ``, ``, ``, ``] at_least_one + tag `get_weather` glm_xml({ location: string }) `` + tag `add` glm_xml({ y: number, x: number, items: string[] }) `` + +# named (from_token_zero) +sequence + tag `` text `` + tag `get_weather` glm_xml({ location: string }) `` diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt new file mode 100644 index 0000000000..58b4bef2da --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.txt @@ -0,0 +1,18 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [``] excluding [``, ``, ``, ``, ``, ``, ``] + tag `get_weather` glm_xml({ location: string }) `` + tag `add` glm_xml({ y: number, x: number, items: string[] }) `` + +# required (from_token_zero) +sequence + tag `` text `` + triggered_tags [``] excluding [``, ``, ``, ``, ``, ``, ``] at_least_one + tag `get_weather` glm_xml({ location: string }) `` + tag `add` glm_xml({ y: number, x: number, items: string[] }) `` + +# named (from_token_zero) +sequence + tag `` text `` + tag `get_weather` glm_xml({ location: string }) `` diff --git a/rust/src/chat/tests/grammar_replay/Hy3.txt b/rust/src/chat/tests/grammar_replay/Hy3.txt new file mode 100644 index 0000000000..dcd6bb08ec --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Hy3.txt @@ -0,0 +1,112 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [``] + tag `\n` .. `\n` + tags_with_separator `\n` at_least_one + tag `get_weather\n` .. `` + sequence + sequence + `` + `location` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + tag `add\n` .. `` + sequence + sequence + `` + `y` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `x` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `items` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + +# required (from_token_zero) +sequence + tag `` text `` + sequence + `\n` + tags_with_separator `\n` at_least_one + tag `get_weather\n` .. `` + sequence + sequence + `` + `location` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + tag `add\n` .. `` + sequence + sequence + `` + `y` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `x` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `items` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + `\n` + +# named (from_token_zero) +sequence + tag `` text `` + sequence + `\n` + tag `get_weather\n` .. `` + sequence + sequence + `` + `location` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + `\n` diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt new file mode 100644 index 0000000000..553c1ce6f4 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.txt @@ -0,0 +1,46 @@ +# auto (from_token_zero) +sequence + tag `` text `` + triggered_tags [`<|tool_calls_section_begin|>`] excluding [``, ``, `<|tool_call_begin|>`] + tag `<|tool_calls_section_begin|>` .. `<|tool_calls_section_end|>` + tags_with_separator `` at_least_one + tag `<|tool_call_begin|>functions.get_weather:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ location: string }) + tag `<|tool_call_begin|>functions.add:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ y: number, x: number, items: string[] }) + +# required (from_token_zero) +sequence + tag `` text `` + sequence + `<|tool_calls_section_begin|>` + tags_with_separator `` at_least_one + tag `<|tool_call_begin|>functions.get_weather:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ location: string }) + tag `<|tool_call_begin|>functions.add:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ y: number, x: number, items: string[] }) + `<|tool_calls_section_end|>` + +# named (from_token_zero) +sequence + tag `` text `` + sequence + `<|tool_calls_section_begin|>` + tag `<|tool_call_begin|>functions.get_weather:` .. `<|tool_call_end|>` + sequence + /\d+/ + `<|tool_call_argument_begin|>` + json({ location: string }) + `<|tool_calls_section_end|>` diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt new file mode 100644 index 0000000000..9b2eca8b2a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt @@ -0,0 +1,67 @@ +# auto (final_output_only) +sequence + optional `<|open|>response<|sep|>` + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + optional + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + plus tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + plus + or + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` + optional `<|close|>message<|sep|>` + +# required (final_output_only) +sequence + optional `<|open|>response<|sep|>` + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + plus tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + plus + or + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` + optional `<|close|>message<|sep|>` + +# named (final_output_only) +sequence + optional `<|open|>response<|sep|>` + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + plus tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` + optional `<|close|>message<|sep|>` diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt new file mode 100644 index 0000000000..1d12560eb0 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.txt @@ -0,0 +1,18 @@ +# auto (from_token_zero) +sequence + optional tag `` text `` + triggered_tags [``] excluding [``, ``, ``, `` qwen_xml({ location: string }) `` + tag `` qwen_xml({ y: number, x: number, items: string[] }) `` + +# required (from_token_zero) +sequence + optional tag `` text `` + triggered_tags [``] excluding [``, ``, ``, `` qwen_xml({ location: string }) `` + tag `` qwen_xml({ y: number, x: number, items: string[] }) `` + +# named (from_token_zero) +sequence + optional tag `` text `` + tag `` qwen_xml({ location: string }) `` diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt new file mode 100644 index 0000000000..431238009a --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.txt @@ -0,0 +1,32 @@ +# auto (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + triggered_tags [``] excluding [``, ``] + tag `\n` .. `` + tags_with_separator `` at_least_one + tag `\n` minimax_xml({ location: string }) `\n` + tag `\n` minimax_xml({ y: number, x: number, items: string[] }) `\n` + +# required (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + sequence + `\n\n` + tags_with_separator `` at_least_one + tag `\n` minimax_xml({ location: string }) `\n` + tag `\n` minimax_xml({ y: number, x: number, items: string[] }) `\n` + `` + +# named (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + sequence + `\n\n` + tag `\n` minimax_xml({ location: string }) `\n` + `` diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt new file mode 100644 index 0000000000..fe38f5e9ff --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.txt @@ -0,0 +1,27 @@ +# auto (from_token_zero) +sequence + optional + sequence + tag `` text `` + `\n\n` + triggered_tags [``] excluding [``, ``] + tag `\n{"name": "get_weather", "arguments": ` json({ location: string }) `}\n` + tag `\n{"name": "add", "arguments": ` json({ y: number, x: number, items: string[] }) `}\n` + +# required (from_token_zero) +sequence + optional + sequence + tag `` text `` + `\n\n` + triggered_tags [``] excluding [``, ``] at_least_one + tag `\n{"name": "get_weather", "arguments": ` json({ location: string }) `}\n` + tag `\n{"name": "add", "arguments": ` json({ y: number, x: number, items: string[] }) `}\n` + +# named (from_token_zero) +sequence + optional + sequence + tag `` text `` + `\n\n` + tag `\n{"name": "get_weather", "arguments": ` json({ location: string }) `}\n` diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt new file mode 100644 index 0000000000..b057780649 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.txt @@ -0,0 +1,24 @@ +# auto (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + triggered_tags [`\n`, ``] + tag `\n\n` qwen_xml({ location: string }) `\n\n` + tag `\n\n` qwen_xml({ y: number, x: number, items: string[] }) `\n\n` + +# required (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + triggered_tags [`\n`, ``] at_least_one + tag `\n\n` qwen_xml({ location: string }) `\n\n` + tag `\n\n` qwen_xml({ y: number, x: number, items: string[] }) `\n\n` + +# named (from_token_zero) +sequence + sequence + tag `` text `` + `\n\n` + tag `\n\n` qwen_xml({ location: string }) `\n\n` diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 6925ae1b02..de813e3045 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -1150,6 +1150,7 @@ fn check_grammar_replay_file( ) -> Result<()> { let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name); let path = grammar_replay_dir().join(format!("{model_name}.json")); + let outline_path = path.with_extension("txt"); let results = results .into_iter() .filter_map(|(variant, result)| { @@ -1157,11 +1158,13 @@ fn check_grammar_replay_file( }) .collect::>(); if results.is_empty() { - ensure!( - !path.exists(), - "{} builds no output grammar, but grammar replay file {path:?} exists", - case.model_id - ); + for path in [&path, &outline_path] { + ensure!( + !path.exists(), + "{} builds no output grammar, but grammar replay file {path:?} exists", + case.model_id + ); + } return Ok(()); } @@ -1205,6 +1208,25 @@ fn check_grammar_replay_file( ); } + // A readable outline of the same grammars, for review only. + let mut outline = String::new(); + for (variant, case) in &cases { + if !outline.is_empty() { + outline.push('\n'); + } + let variant = serde_json::to_value(variant)?; + let coverage = serde_json::to_value(case.grammar.coverage)?; + outline.push_str(&format!( + "# {} ({})\n", + variant.as_str().context("variant name")?, + coverage.as_str().context("coverage name")?, + )); + outline.push_str(&vllm_parser::output_grammar::test_utils::outline( + &case.grammar.format, + )); + } + expect_test::expect_file![outline_path].assert_eq(&outline); + let file = GrammarReplayFile { model_id: case.model_id, vocab_size: backends.text_backend.model_vocab_size(), diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index db56f97708..29f91a4c25 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -17,6 +17,9 @@ use xgrammar_structural_tag::{ use crate::tool::Tool; +#[cfg(any(test, feature = "test-util"))] +pub mod test_utils; + /// Result alias for output grammar construction. pub type Result = std::result::Result; diff --git a/rust/src/parser/src/output_grammar/test_utils.rs b/rust/src/parser/src/output_grammar/test_utils.rs new file mode 100644 index 0000000000..c745152556 --- /dev/null +++ b/rust/src/parser/src/output_grammar/test_utils.rs @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Readable renderings of structural-tag formats for test snapshots. + +use std::fmt::Write; + +use serde_json::{Map, Value}; +use xgrammar_structural_tag::format::{EndBoundary, Format, TagBoundary, TagFormat, TokenValue}; + +/// Render `format` as an indented outline, one node per line. +/// +/// Each node starts with its format type, and composite nodes list their +/// children one level deeper. Literal text is quoted in backticks with `\`, +/// `` ` ``, and control characters escaped, so markers read as written. Other +/// leaves are compact: `/regex/`, `text`, `json(SCHEMA)` or `STYLE(SCHEMA)` +/// for styled schemas, and `token ID` or ``token `TEXT` ``. Lists such as +/// exclusions and triggers are bracketed, and schemas read as types, see +/// [`schema`]. A single leaf child of `optional`, `star`, `plus`, `repeat`, +/// and dispatch rules stays on its parent's line, as does a tag's leaf +/// content between its boundaries. Options that keep their defaults are +/// omitted. +pub fn outline(format: &Format) -> String { + let mut out = String::new(); + write(&mut out, 0, format_node(format)); + out +} + +/// A child line of an outline node. +enum Child<'a> { + Format(&'a Format), + Tag(&'a TagFormat), + Rule(String, &'a Format), + Text(&'a str), +} + +/// An outline node: its header line and children. +struct Node<'a> { + header: String, + children: Vec>, + /// Whether a single leaf child joins the header line. + inline_leaf: bool, +} + +impl<'a> Node<'a> { + fn leaf(header: String) -> Self { + Self { + header, + children: vec![], + inline_leaf: false, + } + } + + fn branch(header: impl Into, children: Vec>) -> Self { + Self { + header: header.into(), + children, + inline_leaf: false, + } + } + + fn wrapper(header: impl Into, child: Child<'a>) -> Self { + Self { + header: header.into(), + children: vec![child], + inline_leaf: true, + } + } +} + +fn write(out: &mut String, depth: usize, node: Node<'_>) { + let Node { + mut header, + children, + inline_leaf, + } = node; + let mut children = children.into_iter().map(self::node).collect::>(); + if inline_leaf + && let [child] = &children[..] + && child.children.is_empty() + { + let _ = write!(header, " {}", child.header); + children.clear(); + } + let _ = writeln!(out, "{}{header}", " ".repeat(depth)); + for child in children { + write(out, depth + 1, child); + } +} + +fn node(child: Child<'_>) -> Node<'_> { + match child { + Child::Format(format) => format_node(format), + Child::Tag(tag) => tag_node(tag), + Child::Rule(pattern, format) => { + Node::wrapper(format!("on {pattern}"), Child::Format(format)) + } + Child::Text(text) => Node::leaf(text.to_string()), + } +} + +fn format_node(format: &Format) -> Node<'_> { + match format { + Format::ConstString(format) => Node::leaf(quote(&format.value)), + Format::Regex(format) => Node::leaf(format!("/{}/", format.pattern)), + Format::AnyText(format) => { + let mut header = "text".to_string(); + push_excluding(&mut header, format.excludes.iter().map(|text| quote(text))); + push_limit(&mut header, "max_tokens", format.max_tokens); + push_limit(&mut header, "max_chars", format.max_chars); + Node::leaf(header) + } + Format::JsonSchema(format) => { + let style = serde_json::to_value(format.style) + .ok() + .and_then(|style| style.as_str().map(str::to_string)) + .unwrap_or_else(|| format!("{:?}", format.style)); + let mut header = format!("{style}({})", schema(&format.json_schema)); + if format.any_order { + header.push_str(" any_order"); + } + if let Some(max) = format.max_whitespace_cnt { + let _ = write!(header, " max_whitespace={max}"); + } + push_excluding(&mut header, format.excludes.iter().map(|text| quote(text))); + Node::leaf(header) + } + Format::Grammar(format) => { + Node::branch("grammar", format.grammar.lines().map(Child::Text).collect()) + } + Format::Token(format) => Node::leaf(format!("token {}", token(&format.token))), + Format::ExcludeToken(format) => { + let mut header = "token".to_string(); + push_excluding(&mut header, format.exclude_tokens.iter().map(token)); + Node::leaf(header) + } + Format::AnyTokens(format) => { + let mut header = "tokens".to_string(); + push_excluding(&mut header, format.exclude_tokens.iter().map(token)); + push_limit(&mut header, "max_tokens", format.max_tokens); + Node::leaf(header) + } + Format::Sequence(format) => Node::branch( + "sequence", + format.elements.iter().map(Child::Format).collect(), + ), + Format::Or(format) => { + Node::branch("or", format.elements.iter().map(Child::Format).collect()) + } + Format::Optional(format) => Node::wrapper("optional", Child::Format(&format.content)), + Format::Plus(format) => Node::wrapper("plus", Child::Format(&format.content)), + Format::Star(format) => Node::wrapper("star", Child::Format(&format.content)), + Format::Repeat(format) => { + let max = if format.max < 0 { + String::new() + } else { + format.max.to_string() + }; + Node::wrapper( + format!("repeat {}..{max}", format.min), + Child::Format(&format.content), + ) + } + Format::Tag(tag) => tag_node(tag), + Format::TriggeredTags(format) => { + let triggers = format.triggers.iter().map(|trigger| quote(trigger)); + let mut header = format!( + "triggered_tags [{}]", + triggers.collect::>().join(", ") + ); + push_excluding(&mut header, format.excludes.iter().map(|text| quote(text))); + push_flags(&mut header, format.at_least_one, format.stop_after_first); + Node::branch(header, format.tags.iter().map(Child::Tag).collect()) + } + Format::TokenTriggeredTags(format) => { + let triggers = format.trigger_tokens.iter().map(token); + let mut header = format!( + "token_triggered_tags [{}]", + triggers.collect::>().join(", ") + ); + push_excluding(&mut header, format.exclude_tokens.iter().map(token)); + push_flags(&mut header, format.at_least_one, format.stop_after_first); + Node::branch(header, format.tags.iter().map(Child::Tag).collect()) + } + Format::TagsWithSeparator(format) => { + let mut header = format!("tags_with_separator {}", quote(&format.separator)); + push_flags(&mut header, format.at_least_one, format.stop_after_first); + Node::branch(header, format.tags.iter().map(Child::Tag).collect()) + } + Format::Dispatch(format) => { + let mut header = "dispatch".to_string(); + if !format.r#loop { + header.push_str(" once"); + } + push_excluding(&mut header, format.excludes.iter().map(|text| quote(text))); + Node::branch( + header, + format + .rules + .iter() + .map(|(pattern, format)| Child::Rule(quote(pattern), format)) + .collect(), + ) + } + Format::TokenDispatch(format) => { + let mut header = "token_dispatch".to_string(); + if !format.r#loop { + header.push_str(" once"); + } + push_excluding(&mut header, format.exclude_tokens.iter().map(token)); + Node::branch( + header, + format + .rules + .iter() + .map(|(trigger, format)| Child::Rule(token(trigger), format)) + .collect(), + ) + } + } +} + +/// `tag BEGIN CONTENT END` for leaf content, or `tag BEGIN .. END` above the +/// content's own node. +fn tag_node(tag: &TagFormat) -> Node<'_> { + let begin = match &tag.begin { + TagBoundary::Text(text) => quote(text), + TagBoundary::Token(boundary) => format!("token {}", token(&boundary.token)), + }; + let end = match &tag.end { + EndBoundary::Text(text) => quote(text), + EndBoundary::Texts(texts) => { + texts.iter().map(|text| quote(text)).collect::>().join(" | ") + } + EndBoundary::Token(boundary) => format!("token {}", token(&boundary.token)), + }; + let content = format_node(&tag.content); + if content.children.is_empty() { + Node::leaf(format!("tag {begin} {} {end}", content.header)) + } else { + Node::branch( + format!("tag {begin} .. {end}"), + vec![Child::Format(&tag.content)], + ) + } +} + +/// Schema keywords that do not constrain the generated text. +const ANNOTATIONS: &[&str] = &[ + "$comment", + "$id", + "$schema", + "default", + "deprecated", + "description", + "examples", + "readOnly", + "title", + "writeOnly", +]; + +/// Render a JSON schema as a TypeScript-like type. +/// +/// Objects are `{ key: T, optional?: T }`, with `...` for any additional +/// property and `...: T` for additional properties under `T`. Arrays are +/// `T[]`. Unions, type arrays, and enums join their options with `|`, +/// `oneOf` and `allOf` keep their names, local references show the +/// definition name, a `type` next to a reference intersects it as +/// `NAME & T`, and `true` / `false` are `any` / `never`. Every other +/// keyword follows the type as `(key=VALUE, ...)`, and definitions follow the +/// root as `where NAME = T, ...`. Annotations such as `description` are +/// dropped. +pub fn schema(schema: &Value) -> String { + typed(schema).text +} + +/// A rendered type, and whether it is a top-level union or intersection that +/// an operand position must parenthesize. +struct Typed { + text: String, + compound: bool, +} + +impl Typed { + fn plain(text: impl Into) -> Self { + Self { + text: text.into(), + compound: false, + } + } + + fn compound(text: String) -> Self { + Self { + text, + compound: true, + } + } + + fn union(options: Vec) -> Self { + Self { + compound: options.len() > 1, + text: options.join(" | "), + } + } + + /// The type as an operand of `&` or of a suffix such as `[]` or `(...)`. + fn grouped(self) -> String { + if self.compound { + format!("({})", self.text) + } else { + self.text + } + } +} + +fn typed(schema: &Value) -> Typed { + let schema = match schema { + Value::Bool(true) => return Typed::plain("any"), + Value::Bool(false) => return Typed::plain("never"), + Value::Object(schema) => schema, + schema => return Typed::plain(schema.to_string()), + }; + let mut rendered = SchemaType::new(schema).render(); + let definitions = ["$defs", "definitions"] + .into_iter() + .filter_map(|key| schema.get(key)?.as_object()) + .flatten() + .map(|(name, definition)| format!("{name} = {}", self::schema(definition))) + .collect::>(); + if !definitions.is_empty() { + rendered = Typed::plain(format!( + "{} where {}", + rendered.text, + definitions.join(", ") + )); + } + rendered +} + +/// One schema object being rendered, with the keywords its type consumed. +struct SchemaType<'a> { + schema: &'a Map, + consumed: Vec<&'static str>, +} + +impl<'a> SchemaType<'a> { + fn new(schema: &'a Map) -> Self { + Self { + schema, + consumed: vec!["$defs", "definitions"], + } + } + + fn take(&mut self, key: &'static str) -> Option<&'a Value> { + let value = self.schema.get(key)?; + self.consumed.push(key); + Some(value) + } + + fn render(mut self) -> Typed { + let mut rendered = self.body(); + let rest = self + .schema + .iter() + .filter(|(key, _)| { + !self.consumed.contains(&key.as_str()) && !ANNOTATIONS.contains(&key.as_str()) + }) + .map(|(key, value)| format!("{key}={value}")) + .collect::>(); + if !rest.is_empty() { + rendered = Typed::plain(format!("{}({})", rendered.grouped(), rest.join(", "))); + } + rendered + } + + fn body(&mut self) -> Typed { + if let Some(Value::String(reference)) = self.take("$ref") { + let name = ["#/$defs/", "#/definitions/"] + .into_iter() + .find_map(|prefix| reference.strip_prefix(prefix)) + .unwrap_or(reference); + // Keywords next to `$ref` apply together with it. + return match self.take("type") { + Some(type_name) => { + Typed::compound(format!("{name} & {}", type_names(type_name).grouped())) + } + None => Typed::plain(name), + }; + } + if let Some(value) = self.take("const") { + self.take_redundant_type([value]); + return Typed::plain(value.to_string()); + } + if let Some(Value::Array(values)) = self.take("enum") { + self.take_redundant_type(values); + return Typed::union(values.iter().map(Value::to_string).collect()); + } + if let Some(Value::Array(options)) = self.take("anyOf") { + return Typed::union(options.iter().map(schema).collect()); + } + for combinator in ["oneOf", "allOf"] { + if let Some(Value::Array(options)) = self.take(combinator) { + let options = options.iter().map(schema).collect::>(); + return Typed::plain(format!("{combinator}({})", options.join(", "))); + } + } + let is = |name: &str| self.schema.get("type").is_none_or(|type_name| type_name == name); + if is("object") + && (self.schema.contains_key("properties") + || self.schema.contains_key("additionalProperties")) + { + self.take("type"); + return Typed::plain(self.object()); + } + if is("array") && self.schema.contains_key("items") { + self.take("type"); + let items = self.take("items").map_or_else(|| Typed::plain("any"), typed); + return Typed::plain(format!("{}[]", items.grouped())); + } + match self.take("type") { + Some(type_name) => type_names(type_name), + None => Typed::plain("any"), + } + } + + /// Consume `type` when every literal already has that type. + fn take_redundant_type<'v>(&mut self, values: impl IntoIterator) { + let Some(Value::String(type_name)) = self.schema.get("type") else { + return; + }; + let matches = |value: &Value| match value { + Value::String(_) => type_name == "string", + Value::Number(number) => { + type_name == "number" || (type_name == "integer" && !number.is_f64()) + } + Value::Bool(_) => type_name == "boolean", + Value::Null => type_name == "null", + Value::Object(_) => type_name == "object", + Value::Array(_) => type_name == "array", + }; + if values.into_iter().all(matches) { + self.take("type"); + } + } + + fn object(&mut self) -> String { + let required = self + .take("required") + .and_then(Value::as_array) + .map(|required| required.iter().filter_map(Value::as_str).collect::>()) + .unwrap_or_default(); + let properties = self.take("properties").and_then(Value::as_object); + let mut fields = properties + .into_iter() + .flatten() + .map(|(key, value)| { + let optional = if required.contains(&key.as_str()) { + "" + } else { + "?" + }; + format!("{key}{optional}: {}", schema(value)) + }) + .collect::>(); + // Required keys without a declared schema. + fields.extend( + required + .iter() + .filter(|key| properties.is_none_or(|properties| !properties.contains_key(**key))) + .map(|key| format!("{key}: any")), + ); + match self.take("additionalProperties") { + Some(Value::Bool(false)) | None => {} + Some(Value::Bool(true)) => fields.push("...".to_string()), + Some(additional) => fields.push(format!("...: {}", schema(additional))), + } + if fields.is_empty() { + "{}".to_string() + } else { + format!("{{ {} }}", fields.join(", ")) + } + } +} + +/// A `type` keyword's type names. +fn type_names(type_name: &Value) -> Typed { + match type_name { + Value::String(name) if name == "array" => Typed::plain("any[]"), + Value::String(name) => Typed::plain(name.clone()), + Value::Array(names) => Typed::union( + names + .iter() + .map(|name| name.as_str().map_or_else(|| name.to_string(), str::to_string)) + .collect(), + ), + type_name => Typed::plain(type_name.to_string()), + } +} + +fn quote(text: &str) -> String { + let mut quoted = String::with_capacity(text.len() + 2); + quoted.push('`'); + for char in text.chars() { + match char { + '\\' => quoted.push_str("\\\\"), + '`' => quoted.push_str("\\`"), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + char if char.is_control() => { + let _ = write!(quoted, "\\u{{{:x}}}", char as u32); + } + char => quoted.push(char), + } + } + quoted.push('`'); + quoted +} + +fn token(token: &TokenValue) -> String { + match token { + TokenValue::Id(id) => id.to_string(), + TokenValue::Text(text) => quote(text), + } +} + +fn push_excluding(header: &mut String, excludes: impl Iterator) { + let excludes = excludes.collect::>(); + if !excludes.is_empty() { + let _ = write!(header, " excluding [{}]", excludes.join(", ")); + } +} + +fn push_limit(header: &mut String, name: &str, limit: Option) { + if let Some(limit) = limit { + let _ = write!(header, " {name}={limit}"); + } +} + +fn push_flags(header: &mut String, at_least_one: bool, stop_after_first: bool) { + if at_least_one { + header.push_str(" at_least_one"); + } + if stop_after_first { + header.push_str(" stop_after_first"); + } +} + +#[cfg(test)] +mod tests { + use expect_test::expect; + use serde_json::json; + use xgrammar_structural_tag::format::{ + AnyTokensFormat, DispatchFormat, ExcludeTokenFormat, GrammarFormat, JsonSchemaFormat, + JsonSchemaStyle, TokenBoundary, TokenDispatchFormat, TokenFormat, TokenTriggeredTagsFormat, + }; + + use super::*; + + #[test] + fn outline_renders_every_format_type() { + let format = Format::sequence(vec![ + Format::Token(TokenFormat { + token: TokenValue::Id(7), + }), + Format::ExcludeToken(ExcludeTokenFormat { + exclude_tokens: vec![TokenValue::Id(1), TokenValue::Text("".to_string())], + }), + Format::AnyTokens(AnyTokensFormat { + exclude_tokens: vec![TokenValue::Id(2)], + max_tokens: Some(16), + }), + Format::Grammar(GrammarFormat { + grammar: "root ::= a\na ::= \"x\"".to_string(), + }), + Format::repeat(Format::regex("[0-9]"), 1, -1), + Format::JsonSchema( + JsonSchemaFormat::new(json!({ "type": "string" })) + .with_style(JsonSchemaStyle::KimiK3Xml) + .with_any_order(true) + .with_max_whitespace_cnt(Some(2)) + .with_excludes(&[""]), + ), + Format::TokenTriggeredTags(TokenTriggeredTagsFormat { + trigger_tokens: vec![TokenValue::Id(9)], + tags: vec![TagFormat::new( + TokenBoundary::new(9), + Format::optional(Format::const_string("`\t")), + vec!["
", ""], + )], + exclude_tokens: vec![], + at_least_one: false, + stop_after_first: true, + }), + Format::Dispatch(DispatchFormat { + rules: vec![( + "".to_string(), + Format::sequence(vec![Format::any_text(), Format::const_string("")]), + )], + r#loop: false, + excludes: vec!["".to_string()], + }), + Format::TokenDispatch(TokenDispatchFormat { + rules: vec![( + TokenValue::Id(3), + Format::JsonSchema(JsonSchemaFormat::new(json!(true))), + )], + r#loop: true, + exclude_tokens: vec![], + }), + ]); + + expect![[r#" + sequence + token 7 + token excluding [1, ``] + tokens excluding [2] max_tokens=16 + grammar + root ::= a + a ::= "x" + repeat 1.. /[0-9]/ + kimi_k3_xml(string) any_order max_whitespace=2 excluding [``] + token_triggered_tags [9] stop_after_first + tag token 9 .. `` | `` + optional `\`\t` + dispatch once excluding [``] + on `` + sequence + text + `` + token_dispatch + on 3 json(any) + "#]] + .assert_eq(&outline(&format)); + } + + #[test] + fn schema_renders_common_shapes_as_types() { + let rendered = schema(&json!({ + "$defs": { "place": { "type": "object", "properties": { "city": { "type": "string" } } } }, + "type": "object", + "description": "ignored", + "properties": { + "location": { "type": "string" }, + "id": { "type": ["integer", "null"] }, + "unit": { "type": "string", "enum": ["celsius", "fahrenheit"] }, + "tags": { "type": "array", "items": { "anyOf": [{ "type": "string" }, { "type": "number" }] } }, + "place": { "$ref": "#/$defs/place" }, + "days": { "type": "integer", "minimum": 1 }, + "free": { "type": "object", "additionalProperties": true } + }, + "required": ["location", "id"], + "additionalProperties": false + })); + + expect![[r#"{ location: string, id: integer | null, unit?: "celsius" | "fahrenheit", tags?: (string | number)[], place?: place, days?: integer(minimum=1), free?: { ... } } where place = { city?: string }"#]].assert_eq(&rendered); + } + + #[test] + fn schema_keeps_other_keywords_after_the_type() { + let rendered = schema(&json!({ + "type": "object", + "properties": { + "code": { "type": "string", "pattern": "[A-Z]{3}" }, + "ids": { "type": "array", "items": { "type": ["integer", "null"] }, "maxItems": 3 }, + "place": { "$ref": "#/$defs/place", "type": "object" }, + "either": { "not": { "type": "null" } } + }, + "required": ["code", "tenant"], + "additionalProperties": { "type": "boolean" }, + "minProperties": 1 + })); + + expect![[r#"{ code: string(pattern="[A-Z]{3}"), ids?: (integer | null)[](maxItems=3), place?: place & object, either?: any(not={"type":"null"}), tenant: any, ...: boolean }(minProperties=1)"#]].assert_eq(&rendered); + } +} diff --git a/rust/src/parser/src/tool/hy/structural_tag.rs b/rust/src/parser/src/tool/hy/structural_tag.rs index e883acbafd..ed2dc6cec3 100644 --- a/rust/src/parser/src/tool/hy/structural_tag.rs +++ b/rust/src/parser/src/tool/hy/structural_tag.rs @@ -170,6 +170,7 @@ mod tests { }; use super::HyStructuralTagBuilder; + use crate::output_grammar::test_utils::outline; use crate::tool::{HyDialect, HyToolMarkers}; fn tool(name: &str, parameters: Value) -> ToolParam { @@ -219,7 +220,33 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"days"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}}]},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#" + sequence + `\n` + tags_with_separator `\n` at_least_one + tag `get_weather\n` .. `` + sequence + sequence + `` + `city` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + star + sequence + `` + `days` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + `\n` + "#]].assert_eq(&outline(&tag.format)); } #[test] @@ -238,7 +265,21 @@ mod tests { ToolChoice::required(), ); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"city"},{"type":"const_string","value":""},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""}]}]},"end":""}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":""}]}}"#]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#" + sequence + `` + tags_with_separator `` at_least_one + tag `get_weather` .. `` + sequence + sequence + `` + `city` + `` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``] + `` + `` + "#]].assert_eq(&outline(&tag.format)); } #[test] @@ -257,14 +298,34 @@ mod tests { )], ToolChoice::required(), ); - let value = serde_json::to_value(tag).unwrap(); - - expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"limit"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq( - &serde_json::to_string( - &value["format"]["elements"][1]["tags"][0]["content"], - ) - .unwrap(), - ); + expect![[r#" + sequence + `\n` + tags_with_separator `\n` at_least_one + tag `lookup\n` .. `` + sequence + star + or + sequence + `` + `query` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `limit` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + `\n` + "#]].assert_eq(&outline(&tag.format)); } #[test] @@ -273,11 +334,24 @@ mod tests { tool("search", json!({ "type": "object" })), tool("lookup", json!({ "type": "object" })), ]; - let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap(); - let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap(); + let auto = build("", &tools, ToolChoice::auto()); + let forced = build("", &tools, ToolChoice::function("lookup")); - expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":[""],"tags":[{"type":"tag","begin":"\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"search\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto); - expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"\n"},{"type":"tag","begin":"lookup\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":""},{"type":"const_string","value":"\n"}]}}"#]].assert_eq(&forced); + expect![[r#" + triggered_tags [``] + tag `\n` .. `\n` + tags_with_separator `\n` at_least_one + tag `search\n` text `` + tag `lookup\n` text `` + "#]] + .assert_eq(&outline(&auto.format)); + expect![[r#" + sequence + `\n` + tag `lookup\n` text `` + `\n` + "#]] + .assert_eq(&outline(&forced.format)); } #[test] @@ -291,11 +365,13 @@ mod tests { )], ToolChoice::function("get_current_date"), ); - let value = serde_json::to_value(tag).unwrap(); - - expect![[r#"{"type":"tag","begin":"get_current_date","content":{"type":"const_string","value":""},"end":""}"#]].assert_eq( - &serde_json::to_string(&value["format"]["elements"][1]).unwrap(), - ); + expect![[r#" + sequence + `` + tag `get_current_date` `` `` + `` + "#]] + .assert_eq(&outline(&tag.format)); } #[test] @@ -312,13 +388,31 @@ mod tests { )], ToolChoice::required(), ); - let value = serde_json::to_value(tag).unwrap(); - - expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"query"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":""},{"type":"const_string","value":"tenant"},{"type":"const_string","value":""},{"type":"const_string","value":"\n"},{"type":"const_string","value":""},{"type":"any_text","excludes":["","","","","","","","",""],"max_tokens":null,"max_chars":null},{"type":"const_string","value":""},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq( - &serde_json::to_string( - &value["format"]["elements"][1]["tags"][0]["content"], - ) - .unwrap(), - ); + expect![[r#" + sequence + `\n` + tags_with_separator `\n` at_least_one + tag `search\n` .. `` + sequence + sequence + `` + `query` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + sequence + `` + `tenant` + `` + `\n` + `` + text excluding [``, ``, ``, ``, ``, ``, ``, ``, ``] + `` + `\n` + `\n` + "#]].assert_eq(&outline(&tag.format)); } } diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs index f542f34e67..a80be7bb1e 100644 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs @@ -364,6 +364,7 @@ mod tests { }; use super::KimiK3StructuralTagBuilder; + use crate::output_grammar::test_utils::outline; fn tool(name: &str, parameters: serde_json::Value) -> ToolParam { ToolParam::Function(FunctionToolParam::new( @@ -395,7 +396,27 @@ mod tests { ) .unwrap(); - expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null,"excludes":[]},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null,"excludes":[]},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap()); + expect![[r#" + sequence + optional `<|open|>response<|sep|>` + tag `` text excluding [`<|open|>`, `<|close|>`, `<|end_of_msg|>`] `<|close|>response<|sep|>` + tag `<|open|>tools<|sep|>` .. `<|close|>tools<|sep|>` + tags_with_separator `` at_least_one + tag `<|open|>call tool="get_weather" index="` .. `<|close|>call<|sep|>` + sequence + /[1-9][0-9]*/ + `"<|sep|>` + or + plus + or + tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>` + or + `celsius` + `fahrenheit` + tag `<|open|>argument key="place" type="object"<|sep|>` json(place & object where place = { city?: string }) `<|close|>argument<|sep|>` + tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>` + optional `<|close|>message<|sep|>` + "#]].assert_eq(&outline(&tag.format)); } #[test] From b2d3a3b9ad7c58e96a825a33112c571610219460 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 1 Oct 2026 13:43:18 +0800 Subject: [PATCH 179/248] [Rust Frontend] Split argument grammars into schema resolution and per-model rendering (#59395) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- .../chat/tests/grammar_replay/Kimi-K3.json | 265 +++--- .../src/chat/tests/grammar_replay/Kimi-K3.txt | 24 +- rust/src/parser/src/output_grammar.rs | 1 + .../parser/src/output_grammar/arguments.rs | 845 ++++++++++++++++++ .../src/unified/kimi_k3/structural_tag.rs | 342 ++----- 5 files changed, 1050 insertions(+), 427 deletions(-) create mode 100644 rust/src/parser/src/output_grammar/arguments.rs diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json index ffe5061192..7fe53b0ab2 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -56,21 +56,18 @@ "type": "or", "elements": [ { - "type": "plus", + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", "content": { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" - } + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" }, { "type": "tag", @@ -119,60 +116,57 @@ "type": "or", "elements": [ { - "type": "plus", - "content": { - "type": "or", - "elements": [ - { - "type": "tag", - "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" }, - "end": "<|close|>argument<|sep|>" + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] }, - { - "type": "tag", - "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" }, - "end": "<|close|>argument<|sep|>" + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] }, - { - "type": "tag", - "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "array", - "items": { - "type": "string" - } - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } }, - "end": "<|close|>argument<|sep|>" - } - ] - } + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] }, { "type": "tag", @@ -287,21 +281,18 @@ "type": "or", "elements": [ { - "type": "plus", + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", "content": { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" - } + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" }, { "type": "tag", @@ -350,60 +341,57 @@ "type": "or", "elements": [ { - "type": "plus", - "content": { - "type": "or", - "elements": [ - { - "type": "tag", - "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" }, - "end": "<|close|>argument<|sep|>" + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] }, - { - "type": "tag", - "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" }, - "end": "<|close|>argument<|sep|>" + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] }, - { - "type": "tag", - "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "array", - "items": { - "type": "string" - } - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null, - "excludes": [] + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } }, - "end": "<|close|>argument<|sep|>" - } - ] - } + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] }, { "type": "tag", @@ -517,21 +505,18 @@ "type": "or", "elements": [ { - "type": "plus", + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", "content": { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" - } + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" }, { "type": "tag", diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt index 9b2eca8b2a..3aeb612642 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.txt +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.txt @@ -10,18 +10,17 @@ sequence /[1-9][0-9]*/ `"<|sep|>` or - plus tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` sequence /[1-9][0-9]*/ `"<|sep|>` or - plus - or - tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` - tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` - tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + sequence + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` @@ -36,18 +35,17 @@ sequence /[1-9][0-9]*/ `"<|sep|>` or - plus tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` tag `<|open|>call tool="add" index="` .. `<|close|>call<|sep|>` sequence /[1-9][0-9]*/ `"<|sep|>` or - plus - or - tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` - tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` - tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` + sequence + tag `<|open|>argument key="y" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="x" type="number"<|sep|>` json(number) `<|close|>argument<|sep|>` + tag `<|open|>argument key="items" type="array"<|sep|>` json(string[]) `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ y: number, x: number, items: string[] }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` @@ -62,6 +60,6 @@ sequence /[1-9][0-9]*/ `"<|sep|>` or - plus tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` + tag `<|open|>argument key="location" type="string"<|sep|>` text excluding [`<|close|>argument<|sep|>`, `<|close|>call<|sep|>`] `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ location: string }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index 29f91a4c25..9058b4a58b 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -17,6 +17,7 @@ use xgrammar_structural_tag::{ use crate::tool::Tool; +pub mod arguments; #[cfg(any(test, feature = "test-util"))] pub mod test_utils; diff --git a/rust/src/parser/src/output_grammar/arguments.rs b/rust/src/parser/src/output_grammar/arguments.rs new file mode 100644 index 0000000000..a7d9fb3078 --- /dev/null +++ b/rust/src/parser/src/output_grammar/arguments.rs @@ -0,0 +1,845 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Tool-call argument grammars for keyed-parameter protocols. +//! +//! Many tool-call protocols render the root arguments object as a sequence of +//! keyed parameters. [`arguments`] owns what follows from the JSON schema +//! alone: which parameters exist, their order and presence, and every option +//! a parameter value can take, with `$ref`s resolved and unions, type arrays, +//! and enums split per option. An [`ArgumentSyntax`], implemented next to each +//! parser, decides how one parameter and its value options are rendered. +//! Values nested below a parameter stay with XGrammar's JSON schema converter +//! through [`ValueOption::json`]. +//! +//! This is for protocols whose argument rendering XGrammar's per-model schema +//! styles do not cover, or cover without matching the parser. Parameter lists +//! follow those styles' fixed-order property semantics, with these deliberate +//! differences: +//! +//! - Unions, type arrays, and mixed enums split into one option per +//! alternative, so a syntax can keep a value's type label consistent with +//! the value. +//! - Additional-property keys are not required to differ from declared keys. +//! - String `format` and length constraints are not enforced. `pattern` is. +//! - `minProperties` and `maxProperties` are not enforced. +//! - `allOf` with more than one schema accepts any value. + +use std::collections::HashSet; + +use serde_json::{Map, Value, json}; +use xgrammar_structural_tag::format::{Format, JsonSchemaFormat}; + +/// Bound on `$ref` and combinator nesting within one value, which keeps a long +/// chain of references from overflowing the stack. +const MAX_SCHEMA_DEPTH: usize = 32; + +/// The schema accepting any value. +static ANY_SCHEMA: Value = Value::Bool(true); + +/// How one protocol renders the parameters of a call. +pub trait ArgumentSyntax { + /// Text around and between parameters, or `None` when parameters are + /// adjacent. + fn separator(&self) -> Option; + + /// The grammar of one parameter whose value takes one of `options`, or + /// `None` when none of them can be rendered. + fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option; +} + +/// The key of one parameter. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ParameterKey<'a> { + /// A property declared by the schema. + Declared(&'a str), + /// Any key allowed by `additionalProperties` or an unconstrained schema. + Free, +} + +impl ParameterKey<'_> { + /// Pattern of free keys, as in XGrammar's XML styles. + pub const FREE_PATTERN: &'static str = "[a-zA-Z_][a-zA-Z0-9_]*"; + + /// `prefix KEY suffix content end`: a tag for a declared key, or a + /// sequence matching [`Self::FREE_PATTERN`] for a free key. + pub fn tag(&self, prefix: &str, suffix: &str, content: Format, end: &str) -> Format { + match self { + Self::Declared(key) => Format::tag(format!("{prefix}{key}{suffix}"), content, end), + Self::Free => Format::sequence(vec![ + Format::const_string(prefix), + Format::regex(Self::FREE_PATTERN), + Format::const_string(suffix), + content, + Format::const_string(end), + ]), + } + } +} + +/// JSON type of a value option. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JsonType { + String, + Number, + Boolean, + Null, + Object, + Array, +} + +impl JsonType { + const ALL: [Self; 6] = [ + Self::String, + Self::Number, + Self::Boolean, + Self::Null, + Self::Object, + Self::Array, + ]; + + /// The type's JSON name. Integers are numbers. + pub fn name(self) -> &'static str { + match self { + Self::String => "string", + Self::Number => "number", + Self::Boolean => "boolean", + Self::Null => "null", + Self::Object => "object", + Self::Array => "array", + } + } + + fn of_schema_type(name: &str) -> Option { + Some(match name { + "string" => Self::String, + "integer" | "number" => Self::Number, + "boolean" => Self::Boolean, + "null" => Self::Null, + "object" => Self::Object, + "array" => Self::Array, + _ => return None, + }) + } + + fn of_value(value: &Value) -> Self { + match value { + Value::String(_) => Self::String, + Value::Number(_) => Self::Number, + Value::Bool(_) => Self::Boolean, + Value::Null => Self::Null, + Value::Object(_) => Self::Object, + Value::Array(_) => Self::Array, + } + } + + /// Schema accepting every JSON value of this type. + fn any_schema(self) -> Value { + match self { + Self::Object => json!({ "type": "object", "additionalProperties": true }), + Self::Array => json!({ "type": "array", "items": true }), + _ => json!({ "type": self.name() }), + } + } +} + +/// One option of a parameter value: a single JSON type under a narrowed +/// schema, a constant, or any value of the type. +pub struct ValueOption<'a> { + /// The option's JSON type. + pub ty: JsonType, + source: OptionSource, + cx: &'a ArgumentContext<'a>, +} + +enum OptionSource { + /// A schema narrowed to [`ValueOption::ty`]. + Schema(Map), + /// A `const` or `enum` value. + Literal(Value), + /// Any value of [`ValueOption::ty`]. + Any, +} + +impl ValueOption<'_> { + /// The option as JSON text. + pub fn json(&self) -> Format { + match &self.source { + OptionSource::Schema(schema) => self.cx.json(Value::Object(schema.clone())), + OptionSource::Literal(value) => self.cx.json(json!({ "const": value })), + OptionSource::Any => self.cx.json(self.ty.any_schema()), + } + } + + /// The option as raw text that ends before any of `terminators`, or `None` + /// for a non-string option or a constant containing a terminator or an + /// excluded substring. + pub fn raw_string(&self, terminators: &[&str]) -> Option { + if self.ty != JsonType::String { + return None; + } + let excludes = self + .cx + .options + .excludes + .iter() + .map(String::as_str) + .chain(terminators.iter().copied()) + .collect::>(); + let text = || Format::any_text_excluding(&excludes); + Some(match &self.source { + OptionSource::Literal(Value::String(value)) => { + if excludes.iter().any(|exclude| value.contains(exclude)) { + return None; + } + Format::const_string(value) + } + OptionSource::Literal(_) => unreachable!("literal options are typed by their value"), + OptionSource::Schema(schema) => match schema.get("pattern").and_then(Value::as_str) { + Some(pattern) => Format::regex(pattern), + None => text(), + }, + OptionSource::Any => text(), + }) + } +} + +/// Request options that also apply to nested JSON schema regions. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ArgumentOptions { + /// Whether object properties may appear in any order, at the parameter + /// level and in nested objects. + pub any_order: bool, + /// Maximum consecutive whitespace characters, or no limit when unset. + pub max_whitespace_cnt: Option, + /// Substrings forbidden in string values, nested JSON strings included. + pub excludes: Vec, +} + +/// Reference targets expanded while collecting one value's options. +#[derive(Default)] +struct Expansion { + expanded: HashSet<*const Value>, + /// Whether a reference was skipped because its target already expanded. + skipped: bool, +} + +/// Request state shared by the value options of one call. +struct ArgumentContext<'a> { + options: &'a ArgumentOptions, + /// The parameters schema, which local `$ref`s resolve against. + root: &'a Value, +} + +impl ArgumentContext<'_> { + /// A JSON value under `schema`, which keeps the root's definitions so its + /// local `$ref`s still resolve. + fn json(&self, mut schema: Value) -> Format { + if let (Some(schema), Some(root)) = (schema.as_object_mut(), self.root.as_object()) { + for key in ["$defs", "definitions"] { + if let Some(definitions) = root.get(key) { + schema.entry(key).or_insert_with(|| definitions.clone()); + } + } + } + Format::JsonSchema(JsonSchemaFormat { + excludes: self.options.excludes.clone(), + ..JsonSchemaFormat::new(schema) + .with_any_order(self.options.any_order) + .with_max_whitespace_cnt(self.options.max_whitespace_cnt) + }) + } + + fn resolve_ref(&self, reference: &str) -> Option<&Value> { + self.root.pointer(reference.strip_prefix('#')?) + } + + /// Follow `$ref`s and single-schema `allOf`s at the root. + fn resolve<'s>(&'s self, schema: &'s Value, depth: usize) -> &'s Value { + if depth > MAX_SCHEMA_DEPTH { + return &ANY_SCHEMA; + } + if let Some(reference) = schema.get("$ref").and_then(Value::as_str) { + return match self.resolve_ref(reference) { + Some(target) => self.resolve(target, depth + 1), + None => &ANY_SCHEMA, + }; + } + match schema.get("allOf").and_then(Value::as_array).map(Vec::as_slice) { + Some([schema]) => self.resolve(schema, depth + 1), + _ => schema, + } + } + + /// Every option of a value under `schema`. + /// + /// Each reference target expands at most once. A union gains no values + /// from a repeated alternative, and a definition shared by several + /// alternatives would otherwise expand once per path to it, exponentially + /// in the nesting. A schema whose references only lead back into it + /// accepts any value. + fn value_options(&self, schema: &Value) -> Vec> { + let mut expansion = Expansion::default(); + let options = self.value_options_at(schema, 0, &mut expansion); + if options.is_empty() && expansion.skipped { + return self.any_value(); + } + options + } + + fn value_options_at( + &self, + schema: &Value, + depth: usize, + expansion: &mut Expansion, + ) -> Vec> { + let schema = match schema { + _ if depth > MAX_SCHEMA_DEPTH => return self.any_value(), + Value::Bool(false) => return vec![], + Value::Object(schema) => schema, + _ => return self.any_value(), + }; + if let Some(reference) = schema.get("$ref").and_then(Value::as_str) { + let Some(target) = self.resolve_ref(reference) else { + return self.any_value(); + }; + if !expansion.expanded.insert(std::ptr::from_ref(target)) { + expansion.skipped = true; + return vec![]; + } + return self.value_options_at(target, depth + 1, expansion); + } + if let Some(value) = schema.get("const") { + return vec![self.literal(value)]; + } + if let Some(values) = schema.get("enum").and_then(Value::as_array) { + return values.iter().map(|value| self.literal(value)).collect(); + } + if let Some(options) = + schema.get("anyOf").or_else(|| schema.get("oneOf")).and_then(Value::as_array) + { + return options + .iter() + .flat_map(|option| self.value_options_at(option, depth + 1, expansion)) + .collect(); + } + if let Some(schemas) = schema.get("allOf").and_then(Value::as_array) { + return match schemas.as_slice() { + [schema] => self.value_options_at(schema, depth + 1, expansion), + _ => self.any_value(), + }; + } + match schema.get("type") { + Some(Value::String(name)) => self.typed(schema.clone(), name).into_iter().collect(), + Some(Value::Array(names)) => names + .iter() + .filter_map(Value::as_str) + .filter_map(|name| { + let mut schema = schema.clone(); + schema.insert("type".to_string(), Value::String(name.to_string())); + self.typed(schema, name) + }) + .collect(), + _ => self.any_value(), + } + } + + fn typed(&self, schema: Map, type_name: &str) -> Option> { + Some(ValueOption { + ty: JsonType::of_schema_type(type_name)?, + source: OptionSource::Schema(schema), + cx: self, + }) + } + + fn literal(&self, value: &Value) -> ValueOption<'_> { + ValueOption { + ty: JsonType::of_value(value), + source: OptionSource::Literal(value.clone()), + cx: self, + } + } + + fn any_value(&self) -> Vec> { + JsonType::ALL + .into_iter() + .map(|ty| ValueOption { + ty, + source: OptionSource::Any, + cx: self, + }) + .collect() + } +} + +/// Build the grammar of one call's arguments under `schema`, the tool's +/// parameters schema, rendering each parameter with `syntax`. +pub fn arguments(schema: &Value, syntax: &dyn ArgumentSyntax, options: &ArgumentOptions) -> Format { + let cx = ArgumentContext { + options, + root: schema, + }; + let parameter = + |key: ParameterKey<'_>, schema: &Value| syntax.parameter(key, &cx.value_options(schema)); + + let schema = cx.resolve(schema, 0); + let (properties, additional) = match schema { + Value::Object(schema) => { + let additional = match schema.get("additionalProperties") { + Some(Value::Bool(false)) => None, + Some(additional) => parameter(ParameterKey::Free, additional), + // Strict schemas allow no undeclared properties, and a schema + // without declared properties allows any. + None if schema.contains_key("properties") => None, + None => parameter(ParameterKey::Free, &ANY_SCHEMA), + }; + let required = schema + .get("required") + .and_then(Value::as_array) + .map(|required| required.iter().filter_map(Value::as_str).collect::>()) + .unwrap_or_default(); + let properties = schema + .get("properties") + .and_then(Value::as_object) + .into_iter() + .flatten() + .filter_map(|(key, schema)| { + let parameter = parameter(ParameterKey::Declared(key), schema)?; + Some((parameter, required.contains(&key.as_str()))) + }) + .collect(); + (properties, additional) + } + Value::Bool(false) => (vec![], None), + _ => (vec![], parameter(ParameterKey::Free, &ANY_SCHEMA)), + }; + parameter_list(options, syntax.separator(), properties, additional) +} + +/// Parameters in declared order, required ones mandatory, followed by any +/// number of additional parameters; or, under `any_order`, any sequence of +/// them with at least one when some are required. +fn parameter_list( + options: &ArgumentOptions, + separator: Option, + properties: Vec<(Format, bool)>, + additional: Option, +) -> Format { + let separated = |parameter: Format| match &separator { + Some(separator) => Format::sequence(vec![parameter, separator.clone()]), + None => parameter, + }; + let mut elements = Vec::from_iter(separator.clone()); + if options.any_order { + let any_required = properties.iter().any(|(_, required)| *required); + let parameters = properties + .into_iter() + .map(|(parameter, _)| parameter) + .chain(additional) + .collect::>(); + if !parameters.is_empty() { + let parameter = separated(one_of(parameters)); + elements.push(if any_required { + Format::plus(parameter) + } else { + Format::star(parameter) + }); + } + } else { + for (parameter, required) in properties { + let parameter = separated(parameter); + elements.push(if required { + parameter + } else { + Format::optional(parameter) + }); + } + if let Some(additional) = additional { + elements.push(Format::star(separated(additional))); + } + } + match elements.len() { + 0 => Format::const_string(""), + 1 => elements.pop().expect("one element"), + _ => Format::sequence(elements), + } +} + +/// One of `formats`, without an `or` around a single format. +pub fn one_of(mut formats: Vec) -> Format { + if formats.len() == 1 { + formats.pop().expect("one format") + } else { + Format::or(formats) + } +} + +/// `options` grouped by `key`, in order of first appearance. +pub fn group_by<'o, 'a, K: PartialEq>( + options: &'o [ValueOption<'a>], + key: impl Fn(&ValueOption<'a>) -> K, +) -> Vec<(K, Vec<&'o ValueOption<'a>>)> { + let mut groups: Vec<(K, Vec<&ValueOption<'a>>)> = Vec::new(); + for option in options { + let key = key(option); + match groups.iter_mut().find(|(existing, _)| *existing == key) { + Some((_, group)) => group.push(option), + None => groups.push((key, vec![option])), + } + } + groups +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::output_grammar::test_utils::outline; + use expect_test::{Expect, expect}; + use serde_json::json; + + /// Qwen-like: `VALUE` on separate lines, with + /// raw strings. + struct Untyped; + + impl ArgumentSyntax for Untyped { + fn separator(&self) -> Option { + Some(Format::const_string("\n")) + } + + fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option { + let values = options + .iter() + .filter_map(|option| match option.ty { + JsonType::String => option.raw_string(&[""]), + _ => Some(option.json()), + }) + .collect::>(); + (!values.is_empty()) + .then(|| key.tag("", one_of(values), "")) + } + } + + /// K3-like: `VALUE` with adjacent + /// parameters and one header per JSON type. + struct Typed; + + impl ArgumentSyntax for Typed { + fn separator(&self) -> Option { + None + } + + fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option { + let tags = group_by(options, |option| option.ty) + .into_iter() + .filter_map(|(ty, options)| { + let values = options + .into_iter() + .filter_map(|option| match ty { + JsonType::String => option.raw_string(&[""]), + _ => Some(option.json()), + }) + .collect::>(); + (!values.is_empty()).then(|| { + let suffix = format!("\" type=\"{}\">", ty.name()); + key.tag("") + }) + }) + .collect::>(); + (!tags.is_empty()).then(|| one_of(tags)) + } + } + + fn check(schema: Value, syntax: &dyn ArgumentSyntax, expected: Expect) { + check_with(schema, syntax, &ArgumentOptions::default(), expected); + } + + fn check_with( + schema: Value, + syntax: &dyn ArgumentSyntax, + options: &ArgumentOptions, + expected: Expect, + ) { + expected.assert_eq(&outline(&arguments(&schema, syntax, options))); + } + + fn weather_schema() -> Value { + json!({ + "type": "object", + "properties": { + "location": { "type": "string" }, + "unit": { "enum": ["celsius", "fahrenheit"] }, + "days": { "type": "integer", "minimum": 1 } + }, + "required": ["location"] + }) + } + + #[test] + fn parameters_follow_declared_order() { + check( + weather_schema(), + &Untyped, + expect![[r#" + sequence + `\n` + sequence + tag `` text excluding [``] `` + `\n` + optional + sequence + tag `` .. `` + or + `celsius` + `fahrenheit` + `\n` + optional + sequence + tag `` json(integer(minimum=1)) `` + `\n` + "#]], + ); + } + + #[test] + fn any_order_repeats_any_declared_parameter() { + check_with( + weather_schema(), + &Typed, + &ArgumentOptions { + any_order: true, + ..Default::default() + }, + expect![[r#" + plus + or + tag `` text excluding [``] `` + tag `` .. `` + or + `celsius` + `fahrenheit` + tag `` json(integer(minimum=1)) any_order `` + "#]], + ); + } + + #[test] + fn unions_and_mixed_enums_split_per_option() { + check( + json!({ + "type": "object", + "properties": { + "id": { "type": ["integer", "string", "null"] }, + "mode": { "enum": ["fast", 1] }, + "q": { "anyOf": [{ "type": "string", "pattern": "[a-z]+" }, { "type": "array", "items": { "type": "string" } }] } + }, + "required": ["q"] + }), + &Typed, + expect![[r#" + sequence + optional + or + tag `` json(integer) `` + tag `` text excluding [``] `` + tag `` json(null) `` + optional + or + tag `` `fast` `` + tag `` json(1) `` + or + tag `` /[a-z]+/ `` + tag `` json(string[]) `` + "#]], + ); + } + + #[test] + fn references_resolve_against_the_root_definitions() { + let schema = json!({ + "$defs": { + "place": { "type": "object", "properties": { "city": { "type": "string" } } }, + "name": { "type": "string" } + }, + "type": "object", + "properties": { + "place": { "$ref": "#/$defs/place" }, + "name": { "$ref": "#/$defs/name" } + }, + "required": ["place", "name"] + }); + let arguments = arguments(&schema, &Typed, &ArgumentOptions::default()); + expect![[r#" + sequence + tag `` json({ city?: string } where place = { city?: string }, name = string) `` + tag `` text excluding [``] `` + "#]].assert_eq(&outline(&arguments)); + + // Nested JSON regions keep the definitions for their own references. + let serialized = serde_json::to_value(&arguments).unwrap(); + assert_eq!( + serialized["elements"][0]["content"]["json_schema"]["$defs"], + schema["$defs"] + ); + } + + #[test] + fn shared_definitions_expand_once() { + // Each level unions four references to the next, so expanding every + // path would yield 4^12 options. + let mut definitions = Map::new(); + for level in 0..12 { + let next = json!({ "$ref": format!("#/$defs/level{}", level + 1) }); + definitions.insert(format!("level{level}"), json!({ "anyOf": vec![next; 4] })); + } + definitions.insert("level12".to_string(), json!({ "type": "string" })); + check( + json!({ + "$defs": definitions, + "type": "object", + "properties": { "p": { "$ref": "#/$defs/level0" } }, + "required": ["p"] + }), + &Untyped, + expect![[r#" + sequence + `\n` + sequence + tag `` text excluding [``] `` + `\n` + "#]], + ); + } + + #[test] + fn references_that_only_cycle_accept_any_value() { + check( + json!({ + "$defs": { + "a": { "anyOf": [{ "$ref": "#/$defs/b" }] }, + "b": { "$ref": "#/$defs/a" } + }, + "type": "object", + "properties": { "p": { "$ref": "#/$defs/a" } }, + "required": ["p"] + }), + &Untyped, + expect![[r#" + sequence + `\n` + sequence + tag `` .. `` + or + text excluding [``] + json(number where a = b, b = a) + json(boolean where a = b, b = a) + json(null where a = b, b = a) + json({ ... } where a = b, b = a) + json(any[] where a = b, b = a) + `\n` + "#]], + ); + } + + #[test] + fn unconstrained_arguments_accept_any_parameter() { + check( + json!(true), + &Typed, + expect![[r#" + star + or + sequence + `` + text excluding [``] + `` + sequence + `` + json(number) + `` + sequence + `` + json(boolean) + `` + sequence + `` + json(null) + `` + sequence + `` + json({ ... }) + `` + sequence + `` + json(any[]) + `` + "#]], + ); + check( + json!(false), + &Untyped, + expect![[r#" + `\n` + "#]], + ); + } + + #[test] + fn additional_properties_follow_declared_ones() { + check( + json!({ + "type": "object", + "properties": { "path": { "type": "string" } }, + "required": ["path"], + "additionalProperties": { "type": "boolean" } + }), + &Untyped, + expect![[r#" + sequence + `\n` + sequence + tag `` text excluding [``] `` + `\n` + star + sequence + sequence + `` + json(boolean) + `` + `\n` + "#]], + ); + } + + #[test] + fn raw_literals_exclude_terminators_and_request_exclusions() { + check_with( + json!({ + "type": "object", + "properties": { "tag": { "enum": ["ok", "ab", "cd"] } }, + "required": ["tag"] + }), + &Typed, + &ArgumentOptions { + excludes: vec!["".to_string()], + ..Default::default() + }, + expect![[r#" + tag `` `ok` `` + "#]], + ); + } +} diff --git a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs index a80be7bb1e..8f0f098f06 100644 --- a/rust/src/parser/src/unified/kimi_k3/structural_tag.rs +++ b/rust/src/parser/src/unified/kimi_k3/structural_tag.rs @@ -3,7 +3,7 @@ //! Structural-tag grammar for Kimi K3 XTML tool calls. -use serde_json::{Map, Value}; +use serde_json::Value; use xgrammar_structural_tag::Result; use xgrammar_structural_tag::builders::{ ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions, @@ -11,15 +11,67 @@ use xgrammar_structural_tag::builders::{ use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat}; use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters}; +use crate::output_grammar::arguments::{ + self, ArgumentOptions, ArgumentSyntax, JsonType, ParameterKey, ValueOption, group_by, one_of, +}; + use super::{ - ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, + ARG_CLOSE, ARG_OPEN, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN, }; pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder = KimiK3StructuralTagBuilder; -const XTML_TYPES: &[&str] = &["string", "number", "boolean", "null", "object", "array"]; +/// XTML arguments: +/// `<|open|>argument key="KEY" type="TYPE"<|sep|>VALUE<|close|>argument<|sep|>`, +/// adjacent to each other. +/// +/// The parser decodes each value by its `type` attribute, so every value option +/// sits under the attribute of its own JSON type. String values are raw text +/// and every other value is JSON. +struct XtmlArguments; + +impl ArgumentSyntax for XtmlArguments { + fn separator(&self) -> Option { + None + } + + fn parameter(&self, key: ParameterKey<'_>, options: &[ValueOption<'_>]) -> Option { + let escaped; + let key = match key { + ParameterKey::Declared(key) => { + escaped = escape_attr_value(key); + ParameterKey::Declared(&escaped) + } + ParameterKey::Free => ParameterKey::Free, + }; + let arguments = group_by(options, |option| option.ty) + .into_iter() + .filter_map(|(ty, options)| { + let values = options + .into_iter() + .filter_map(|option| match ty { + // The parser ends the call at its close marker, so a + // string value may contain neither close marker. + JsonType::String => option.raw_string(&[ARG_CLOSE, CALL_CLOSE]), + _ => Some(option.json()), + }) + .collect::>(); + (!values.is_empty()).then(|| { + let suffix = format!("\" type=\"{}\"{SEP}", ty.name()); + key.tag( + &format!("{ARG_OPEN} key=\""), + &suffix, + one_of(values), + ARG_CLOSE, + ) + }) + }) + .collect::>(); + (!arguments.is_empty()).then(|| one_of(arguments)) + } +} /// Kimi K3 XTML structural-tag builder. #[derive(Debug, Clone, Copy, Default)] @@ -92,8 +144,17 @@ fn tools_channel(tools: &[FunctionToolParam], options: StructuralTagOptions) -> fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat { let parameters = function_parameters(&tool.function); + let typed_arguments = arguments::arguments( + ¶meters, + &XtmlArguments, + &ArgumentOptions { + any_order: options.any_order, + max_whitespace_cnt: options.max_whitespace_cnt, + excludes: vec![], + }, + ); let call_body = Format::or(vec![ - typed_arguments(¶meters, options), + typed_arguments, raw_json_arguments(¶meters, options), ]); @@ -111,189 +172,6 @@ fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagForma ) } -fn typed_arguments(parameters: &Value, options: StructuralTagOptions) -> Format { - let Some(schema) = parameters.as_object() else { - return if parameters == &Value::Bool(false) { - Format::const_string("") - } else { - Format::star(permissive_argument()) - }; - }; - let Some(properties) = schema.get("properties").and_then(Value::as_object) else { - return Format::star(permissive_argument()); - }; - if properties.is_empty() { - return Format::star(permissive_argument()); - } - - let root_defs = root_definitions(schema); - let arguments = properties - .iter() - .flat_map(|(key, schema)| argument_tags(key, schema, &root_defs, options)) - .map(Format::Tag) - .collect::>(); - let arguments = match arguments.as_slice() { - [argument] => argument.clone(), - _ => Format::or(arguments), - }; - // Keep typed arguments order-agnostic and non-unique, but do not allow an - // empty call when the root schema declares required properties. - if schema - .get("required") - .and_then(Value::as_array) - .is_some_and(|required| !required.is_empty()) - { - Format::plus(arguments) - } else { - Format::star(arguments) - } -} - -fn argument_tags( - key: &str, - schema: &Value, - root_defs: &Map, - options: StructuralTagOptions, -) -> Vec { - let types = schema_types(schema); - types - .into_iter() - .map(|xtml_type| { - let content = if xtml_type == "string" { - string_argument_content(schema) - } else { - json_schema( - attach_root_definitions(&narrow_schema_type(schema, xtml_type), root_defs), - options, - ) - }; - TagFormat::new( - format!( - "{OPEN}argument key=\"{}\" type=\"{xtml_type}\"{SEP}", - escape_attr_value(key) - ), - content, - ARG_CLOSE, - ) - }) - .collect() -} - -fn schema_types(schema: &Value) -> Vec<&'static str> { - let Some(schema) = schema.as_object() else { - return XTML_TYPES.to_vec(); - }; - let mut types = Vec::new(); - match schema.get("type") { - Some(Value::String(value)) => push_schema_type(&mut types, value), - Some(Value::Array(values)) => { - for value in values.iter().filter_map(Value::as_str) { - push_schema_type(&mut types, value); - } - } - _ => {} - } - if types.is_empty() - && let Some(value) = schema.get("const") - { - push_value_type(&mut types, value); - } - if types.is_empty() - && let Some(values) = schema.get("enum").and_then(Value::as_array) - { - for value in values { - push_value_type(&mut types, value); - } - } - if types.is_empty() { - XTML_TYPES.to_vec() - } else { - types - } -} - -fn push_value_type(types: &mut Vec<&'static str>, value: &Value) { - let xtml_type = match value { - Value::String(_) => "string", - Value::Number(_) => "number", - Value::Bool(_) => "boolean", - Value::Null => "null", - Value::Object(_) => "object", - Value::Array(_) => "array", - }; - if !types.contains(&xtml_type) { - types.push(xtml_type); - } -} - -fn narrow_schema_type(schema: &Value, xtml_type: &str) -> Value { - let Some(mut schema) = schema.as_object().cloned() else { - return schema.clone(); - }; - let json_type = if xtml_type == "number" && explicitly_integer_only(&schema) { - "integer" - } else { - xtml_type - }; - schema.insert("type".to_string(), Value::String(json_type.to_string())); - Value::Object(schema) -} - -fn explicitly_integer_only(schema: &Map) -> bool { - match schema.get("type") { - Some(Value::String(value)) => value == "integer", - Some(Value::Array(values)) => { - let values = values.iter().filter_map(Value::as_str).collect::>(); - values.contains(&"integer") && !values.contains(&"number") - } - _ => false, - } -} - -fn push_schema_type(types: &mut Vec<&'static str>, json_type: &str) { - let xtml_type = match json_type { - "string" => Some("string"), - "integer" | "number" => Some("number"), - "boolean" => Some("boolean"), - "null" => Some("null"), - "object" => Some("object"), - "array" => Some("array"), - _ => None, - }; - if let Some(xtml_type) = xtml_type - && !types.contains(&xtml_type) - { - types.push(xtml_type); - } -} - -fn string_argument_content(schema: &Value) -> Format { - let Some(schema) = schema.as_object() else { - return Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]); - }; - let values = schema - .get("enum") - .and_then(Value::as_array) - .cloned() - .or_else(|| schema.get("const").cloned().map(|value| vec![value])); - let Some(values) = values else { - return Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]); - }; - if values.is_empty() - || values.len() > 256 - || values - .iter() - .any(|value| value.as_str().is_none_or(|value| value.contains("<|"))) - { - return Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]); - } - let values = values.iter().filter_map(Value::as_str).collect::>(); - match values.as_slice() { - [value] => Format::const_string(*value), - _ => Format::or(values.into_iter().map(Format::const_string).collect()), - } -} - fn raw_json_arguments(parameters: &Value, options: StructuralTagOptions) -> Format { Format::tag( format!("{JSON_OPEN} type=\"object\"{SEP}"), @@ -302,29 +180,6 @@ fn raw_json_arguments(parameters: &Value, options: StructuralTagOptions) -> Form ) } -fn permissive_argument() -> Format { - let key = Format::regex(r#"(?:[^<\"&]|&(?:amp|quot);|<[^|])*"#); - let alternatives = XTML_TYPES - .iter() - .map(|xtml_type| { - Format::sequence(vec![ - key.clone(), - Format::const_string(format!("\" type=\"{xtml_type}\"{SEP}")), - if *xtml_type == "string" { - Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]) - } else { - Format::json_schema(Value::Bool(true)) - }, - ]) - }) - .collect(); - Format::tag( - format!("{OPEN}argument key=\""), - Format::or(alternatives), - ARG_CLOSE, - ) -} - fn json_schema(schema: Value, options: StructuralTagOptions) -> Format { Format::JsonSchema( JsonSchemaFormat::new(schema) @@ -333,23 +188,6 @@ fn json_schema(schema: Value, options: StructuralTagOptions) -> Format { ) } -fn root_definitions(schema: &Map) -> Map { - ["$defs", "definitions"] - .into_iter() - .filter_map(|key| schema.get(key).map(|value| (key.to_string(), value.clone()))) - .collect() -} - -fn attach_root_definitions(schema: &Value, root_defs: &Map) -> Value { - let Some(mut schema) = schema.as_object().cloned() else { - return schema.clone(); - }; - for (key, value) in root_defs { - schema.entry(key.clone()).or_insert_with(|| value.clone()); - } - Value::Object(schema) -} - fn escape_attr_value(value: &str) -> String { value.replace('&', "&").replace('"', """) } @@ -407,52 +245,18 @@ mod tests { /[1-9][0-9]*/ `"<|sep|>` or - plus - or + sequence + optional tag `<|open|>argument key="unit" type="string"<|sep|>` .. `<|close|>argument<|sep|>` or `celsius` `fahrenheit` - tag `<|open|>argument key="place" type="object"<|sep|>` json(place & object where place = { city?: string }) `<|close|>argument<|sep|>` + tag `<|open|>argument key="place" type="object"<|sep|>` json({ city?: string } where place = { city?: string }) `<|close|>argument<|sep|>` tag `<|open|>json type="object"<|sep|>` json({ unit?: "celsius" | "fahrenheit", place: place & object } where place = { city?: string }) `<|close|>json<|sep|>` optional `<|close|>message<|sep|>` "#]].assert_eq(&outline(&tag.format)); } - #[test] - fn typed_arguments_require_one_tag_only_for_nonempty_required() { - let required = super::typed_arguments( - &json!({ - "type": "object", - "properties": { "query": { "type": "string" } }, - "required": ["query"] - }), - StructuralTagOptions::default(), - ); - let optional = super::typed_arguments( - &json!({ - "type": "object", - "properties": { "query": { "type": "string" } } - }), - StructuralTagOptions::default(), - ); - let empty_required = super::typed_arguments( - &json!({ - "type": "object", - "properties": { "query": { "type": "string" } }, - "required": [] - }), - StructuralTagOptions::default(), - ); - - assert_eq!(serde_json::to_value(required).unwrap()["type"], "plus"); - assert_eq!(serde_json::to_value(optional).unwrap()["type"], "star"); - assert_eq!( - serde_json::to_value(empty_required).unwrap()["type"], - "star" - ); - } - #[test] fn reasoning_grammar_starts_inside_prefilled_think_channel() { let tools = vec![tool("ping", json!({ "type": "object", "properties": {} }))]; @@ -522,14 +326,4 @@ mod tests { assert!(tag.contains(r#"type=\"null\""#)); assert!(tag.contains(r#""json_schema":{"type":"null"}"#), "{tag}"); } - - #[test] - fn unsafe_string_enum_falls_back_as_a_whole() { - let format = super::string_argument_content(&json!({ - "type": "string", - "enum": ["safe", "<|unsafe"] - })); - - assert_eq!(serde_json::to_value(format).unwrap()["type"], "any_text"); - } } From 37cecc8141e2dcba9dd534897a164d4baaf27229 Mon Sep 17 00:00:00 2001 From: Shijie Lyu Date: Wed, 30 Sep 2026 22:43:30 -0700 Subject: [PATCH 180/248] [Bugfix][Tokenizer] Fix off-by-one max_token_id from vocab_size (#59491) Signed-off-by: Shijie Lyu --- tests/tokenizers_/test_hf.py | 16 ++++++++++++++++ vllm/tokenizers/hf.py | 5 +++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/tokenizers_/test_hf.py b/tests/tokenizers_/test_hf.py index 693cda4ab0..a28c6b3b2e 100644 --- a/tests/tokenizers_/test_hf.py +++ b/tests/tokenizers_/test_hf.py @@ -65,3 +65,19 @@ def test_thread_pool_tokenizer_pickle(model_id: str): # Idempotence: wrapping an already-pooled tokenizer returns it unchanged. assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer # type: ignore[type-var] + + +@pytest.mark.parametrize("model_id", ["openai-community/gpt2"]) +def test_cached_tokenizer_max_token_id_is_a_valid_id(model_id: str): + """max_token_id is the largest valid id, one less than vocab_size.""" + reference_tokenizer = AutoTokenizer.from_pretrained(model_id) + cached_tokenizer = get_cached_tokenizer(deepcopy(reference_tokenizer)) + + expected = max( + max(reference_tokenizer.get_vocab().values()), + reference_tokenizer.vocab_size - 1, + ) + assert cached_tokenizer.max_token_id == expected + assert cached_tokenizer.max_token_id == max( + reference_tokenizer.get_vocab().values() + ) diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index a091d8e105..a2e5e3a085 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -124,10 +124,11 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: # Some tokenizers (e.g., QwenTokenizer) have special tokens that # are added and included in the implementation of the vocab_size # property, but not in get_vocab(); if there is an implementation - # of vocab size, we should take the greater value. + # of vocab size, we should take the greater value. vocab_size is a + # count, so the largest id it implies is vocab_size - 1. if hasattr(tokenizer, "vocab_size"): with contextlib.suppress(NotImplementedError): - max_token_id = max(max_token_id, tokenizer.vocab_size) + max_token_id = max(max_token_id, tokenizer.vocab_size - 1) class CachedTokenizer(tokenizer.__class__): # type: ignore @property From 70be6aae19587a251987c405b5bdf2d540da5991 Mon Sep 17 00:00:00 2001 From: vx120 <57470515+vx120@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:51:39 +0800 Subject: [PATCH 181/248] [Feature][Spec Decode] Support sampling mask replay for MRV2 MTP (#59359) Signed-off-by: Cuiping Cheng Signed-off-by: aoshen02 Co-authored-by: Cuiping Cheng Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- docs/training/sampling_mask.md | 5 +- tests/test_config.py | 91 +++++++ tests/v1/core/test_scheduler.py | 43 ++++ tests/v1/engine/test_output_processor.py | 26 ++ tests/v1/test_outputs.py | 70 +++++- .../test_gpu_rejection_sampler_chunking.py | 236 +++++++++++++++--- vllm/config/vllm.py | 10 +- vllm/v1/engine/output_processor.py | 6 +- vllm/v1/outputs.py | 14 +- vllm/v1/worker/gpu/async_utils.py | 2 +- vllm/v1/worker/gpu/sample/output.py | 91 +++++-- vllm/v1/worker/gpu/sample/sampler.py | 5 +- .../gpu/spec_decode/rejection_sampler.py | 52 +++- 13 files changed, 583 insertions(+), 68 deletions(-) diff --git a/docs/training/sampling_mask.md b/docs/training/sampling_mask.md index 67e0709388..c8425e0245 100644 --- a/docs/training/sampling_mask.md +++ b/docs/training/sampling_mask.md @@ -63,9 +63,12 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint: | `top_k > 0` | Bounds mask size; pure top-p can produce vocab-sized masks | | Model Runner V2 | Required by the async D2H copy pipeline | +Speculative decoding is supported; each emitted token gets the support of the +target distribution at its position. + The engine rejects unsupported combinations at startup or request time: -- Speculative decoding +- Speculative decoding with adaptive verification - Diffusion models - Custom logits processors (engine-level `--logits-processors`) diff --git a/tests/test_config.py b/tests/test_config.py index 712d01fd33..e7389d18b4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -147,6 +147,97 @@ def test_rocm_mm_prefix_lm_disables_chunked_mm_input( assert config.scheduler_config.disable_chunked_mm_input is expected +def _sampling_replay_config( + *, + return_sampling_mask: bool = True, + use_v2_model_runner: bool = True, + speculative_method: str | None = None, + rejection_sample_method: str = "standard", + adaptive: bool = False, + is_diffusion: bool = False, + logits_processors: list[str] | None = None, + logprobs_mode: str = "processed_logprobs", +): + speculative_config = None + if speculative_method is not None: + speculative_config = SimpleNamespace( + method=speculative_method, + enable_adaptive_verification=adaptive, + rejection_sample_method=rejection_sample_method, + ) + return SimpleNamespace( + model_config=SimpleNamespace( + return_sampling_mask=return_sampling_mask, + is_diffusion=is_diffusion, + logits_processors=logits_processors or [], + logprobs_mode=logprobs_mode, + ), + use_v2_model_runner=use_v2_model_runner, + speculative_config=speculative_config, + ) + + +@pytest.mark.parametrize( + ("config", "message"), + [ + (_sampling_replay_config(), None), + *( + (_sampling_replay_config(speculative_method=method), None) + for method in ( + "mtp", + "eagle", + "eagle3", + "dflash", + "dspark", + "draft_model", + ) + ), + *( + ( + _sampling_replay_config( + speculative_method="mtp", + rejection_sample_method=rejection_sample_method, + ), + None, + ) + for rejection_sample_method in ("standard", "block", "synthetic") + ), + ( + _sampling_replay_config( + return_sampling_mask=False, speculative_method="dspark", adaptive=True + ), + None, + ), + ( + _sampling_replay_config(speculative_method="dspark", adaptive=True), + "requires fixed verification boundaries", + ), + ( + _sampling_replay_config(is_diffusion=True), + "does not support diffusion models", + ), + ( + _sampling_replay_config(logits_processors=["custom"]), + "does not support custom logits processors", + ), + ( + _sampling_replay_config(logprobs_mode="raw_logprobs"), + "requires logprobs_mode='processed_logprobs'", + ), + ( + _sampling_replay_config(use_v2_model_runner=False), + "requires Model Runner V2", + ), + ], +) +def test_sampling_replay_config(config, message): + if message is None: + VllmConfig._verify_sampling_replay_config(config) + else: + with pytest.raises(ValueError, match=message): + VllmConfig._verify_sampling_replay_config(config) + + def test_kda_recoverssm_derivation_is_revalidated(): config = SimpleNamespace( cache_config=SimpleNamespace( diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index c8ac4dec64..722cbfb79e 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -908,6 +908,49 @@ def test_update_from_output_routes_sampling_masks_by_request(): assert all(out.new_sampling_mask.offsets is None for out in outputs) +def test_update_from_output_routes_multi_position_sampling_masks(): + scheduler = create_scheduler() + scheduler.return_sampling_mask = True + requests = create_requests(num_requests=2, max_tokens=10) + for req in requests: + req.num_computed_tokens = req.num_tokens + scheduler.requests[req.request_id] = req + scheduler.running.append(req) + req.status = RequestStatus.RUNNING + + scheduler_output = SchedulerOutput( + scheduled_new_reqs=[], + scheduled_cached_reqs=CachedRequestData.make_empty(), + num_scheduled_tokens={req.request_id: 1 for req in requests}, + total_num_scheduled_tokens=2, + scheduled_encoder_inputs={}, + scheduled_spec_decode_tokens={}, + num_common_prefix_blocks=[], + finished_req_ids=set(), + free_encoder_mm_hashes=[], + ) + model_output = ModelRunnerOutput( + req_ids=[req.request_id for req in requests], + req_id_to_index={req.request_id: i for i, req in enumerate(requests)}, + sampled_token_ids=[[1, 2], [3, 4, 5]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + sampling_masks=SamplingMaskLists( + token_ids=np.array([1, 6, 2, 3, 7, 8, 4, 5, 9], dtype=np.int32), + offsets=np.array([0, 2, 3, 6, 7, 9]), + cu_num_generated_tokens=[0, 2, 5], + ), + ) + + outputs = scheduler.update_from_output(scheduler_output, model_output)[0].outputs + + assert [out.new_sampling_mask.to_nested_list() for out in outputs] == [ + [[1, 6], [2]], + [[3, 7, 8], [4], [5, 9]], + ] + + def test_stop_via_update_from_output(): """Test stopping behavior through update_from_output.""" scheduler = create_scheduler(num_speculative_tokens=1) diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 485187f222..53b8e0f552 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -35,6 +35,7 @@ from vllm.v1.engine.output_processor import ( RequestState, ) from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats +from vllm.v1.outputs import SamplingMaskLists @pytest.mark.parametrize("flat_logprobs", [False, True]) @@ -62,6 +63,31 @@ def test_delta_output_without_new_tokens_returns_empty_logprobs( assert len(output.logprobs) == 0 +def test_completion_output_preserves_each_sampling_mask_position() -> None: + state = RequestState.__new__(RequestState) + state.detokenizer = MagicMock() + state.detokenizer.get_next_output_text.return_value = "" + state.logprobs_processor = MagicMock() + state.logprobs_processor.logprobs = None + state.logprobs_processor.cumulative_logprob = None + state.output_kind = RequestOutputKind.DELTA + state.request_index = 0 + state.sampling_mask_chunks = [ + SamplingMaskLists( + token_ids=np.array([10, 11, 20]), + offsets=np.array([0, 2, 3]), + ), + SamplingMaskLists(token_ids=np.array([30, 31, 32])), + ] + state.routed_experts_chunks = [] + state.spec_decode_metrics = None + + output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None) + + assert output.sampling_mask is not None + assert output.sampling_mask.token_ids == [[10, 11], [20], [30, 31, 32]] + + def _ref_convert_id_to_token( tokenizer: TokenizerLike, token_id: int, diff --git a/tests/v1/test_outputs.py b/tests/v1/test_outputs.py index 5d52c86374..471cac6f95 100644 --- a/tests/v1/test_outputs.py +++ b/tests/v1/test_outputs.py @@ -13,6 +13,7 @@ from vllm.v1.outputs import ( LogprobsLists, LogprobsTensors, ModelRunnerOutput, + SamplingMaskLists, ) from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors @@ -63,8 +64,6 @@ def test_logprobs_tensors_tolists_with_tensor_boundaries(): def test_sampling_mask_lists_to_nested_list(): - from vllm.v1.outputs import SamplingMaskLists - mask = SamplingMaskLists( token_ids=np.array([10, 11, 12, 20, 21]), offsets=np.array([0, 3, 5]), @@ -76,6 +75,22 @@ def test_sampling_mask_lists_to_nested_list(): assert SamplingMaskLists(np.array([7, 9])).to_nested_list() == [[7, 9]] +def test_sampling_mask_lists_slices_multiple_positions_by_request(): + masks = SamplingMaskLists( + token_ids=np.array([10, 11, 20, 30, 31, 32, 40]), + offsets=np.array([0, 2, 3, 6, 7]), + cu_num_generated_tokens=[0, 1, 3, 4], + ) + + single = masks.slice_request(0, 1) + multi = masks.slice_request(1, 2) + + assert single.to_nested_list() == [[10, 11]] + assert single.offsets is None + assert multi.to_nested_list() == [[20], [30, 31, 32]] + assert multi.cu_num_generated_tokens is None + + @pytest.mark.parametrize("max_num_kept", [512, 20_001]) @pytest.mark.skipif( current_platform.is_xpu(), @@ -102,7 +117,10 @@ def test_sampling_mask_tensors_match_finite_support(max_num_kept): expected[1] = expected[7] = [] tensors = SamplingMaskTensors.from_logits( - logits.to(DEVICE_TYPE), num_sampled_tokens.to(DEVICE_TYPE), max_num_kept + logits.to(DEVICE_TYPE), + torch.arange(len(sizes) + 1, device=DEVICE_TYPE, dtype=torch.int32), + num_sampled_tokens.to(DEVICE_TYPE), + max_num_kept, ) assert tensors.token_ids.shape[1] == min(max_num_kept, MAX_COMPACT_SUPPORT) @@ -127,6 +145,7 @@ def test_sampling_mask_matches_processed_top_k_top_p_support(): tensors = SamplingMaskTensors.from_logits( processed_logits, + cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE), num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE), max_num_kept=3, ) @@ -135,6 +154,50 @@ def test_sampling_mask_matches_processed_top_k_top_p_support(): assert result.to_nested_list() == [expected_token_ids] +def test_sampling_mask_tensors_uses_request_boundaries(): + logits = torch.full((6, 8), -float("inf"), device=DEVICE_TYPE) + logits[0, [1, 3]] = 0 + logits[2, [2, 4, 6]] = 0 + logits[5, [0, 7]] = 0 + counts = torch.tensor([1, 0, 1], device=DEVICE_TYPE, dtype=torch.int32) + + tensors = SamplingMaskTensors.from_logits( + logits, + torch.tensor([0, 2, 5, 6], device=DEVICE_TYPE, dtype=torch.int32), + counts, + max_num_kept=3, + ) + + assert tensors.tolists().to_nested_list() == [[1, 3], [], [0, 7]] + + +def test_sampling_mask_tensors_multirow_request_layout(): + rows_per_request = 4 + counts = np.array([1, 2, 4, 0], dtype=np.int32) + logits = torch.full((16, 16), -float("inf"), device=DEVICE_TYPE) + expected = [] + for row in [0, 4, 5, 8, 9, 10, 11]: + kept = torch.tensor( + [row % 16, (row + 3) % 16, (row + 7) % 16], device=DEVICE_TYPE + ).unique(sorted=True) + logits[row, kept] = 0 + expected.append(kept.tolist()) + + tensors = SamplingMaskTensors.from_logits( + logits, + torch.arange(0, 17, 4, device=DEVICE_TYPE, dtype=torch.int32), + torch.from_numpy(counts).to(DEVICE_TYPE), + max_num_kept=2, + rows_per_request=rows_per_request, + ) + result = tensors.to_cpu_nonblocking().tolists(counts) + + assert tensors.token_ids.shape == (16, 2) + assert tensors.rows_per_request == rows_per_request + assert result.to_nested_list() == expected + assert result.cu_num_generated_tokens == [0, 1, 3, 7, 7] + + def test_sampling_mask_preserves_top_k_boundary_ties(): """When the kept support is wider than `max_num_kept` (e.g. a top-k boundary tie keeps more than k logits), the mask must fall back to the @@ -147,6 +210,7 @@ def test_sampling_mask_preserves_top_k_boundary_ties(): tensors = SamplingMaskTensors.from_logits( processed_logits, + cu_num_logits=torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE), num_sampled_tokens=torch.tensor([1], device=DEVICE_TYPE), max_num_kept=3, ) diff --git a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py index efd2b59d75..a6f6731884 100644 --- a/tests/v1/worker/test_gpu_rejection_sampler_chunking.py +++ b/tests/v1/worker/test_gpu_rejection_sampler_chunking.py @@ -10,12 +10,53 @@ import torch from vllm.config.model import PROCESSED_LOGPROBS_MODES, LogprobsMode from vllm.platforms import current_platform +from vllm.v1.worker.gpu.sample.output import SamplingMaskTensors from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( RejectionSampler, _iter_request_chunks, ) +def _make_input_batch( + cu_num_logits_np: np.ndarray, + idx_mapping_np: np.ndarray, + device: torch.device, +) -> SimpleNamespace: + """Minimal InputBatch fields read by RejectionSampler._verify_in_chunks.""" + num_reqs = len(idx_mapping_np) + num_logits_per_req = np.diff(cu_num_logits_np) + return SimpleNamespace( + num_reqs=num_reqs, + cu_num_logits_np=cu_num_logits_np, + cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device), + idx_mapping_np=idx_mapping_np, + idx_mapping=torch.from_numpy(idx_mapping_np).to(device), + expanded_idx_mapping=torch.from_numpy( + np.repeat(idx_mapping_np, num_logits_per_req) + ).to(device), + expanded_local_pos=torch.from_numpy( + np.concatenate( + [np.arange(count, dtype=np.int32) for count in num_logits_per_req] + ) + ).to(device), + seq_lens_cpu_upper_bound=torch.full((num_reqs,), 64, dtype=torch.int32), + ) + + +def _make_rejection_sampler( + sampler: SimpleNamespace, num_speculative_steps: int +) -> RejectionSampler: + """RejectionSampler for standard, fixed-boundary, unwatermarked verification.""" + rejection_sampler = object.__new__(RejectionSampler) + rejection_sampler.sampler = sampler + rejection_sampler.num_speculative_steps = num_speculative_steps + rejection_sampler.enable_adaptive_verification = False + rejection_sampler.synthetic_conditional_rates = None + rejection_sampler.use_block_verification = False + rejection_sampler.watermark_key = None + return rejection_sampler + + def test_iter_request_chunks_preserves_request_boundaries(): cu_num_logits = np.array([0, 3, 4, 11, 13], dtype=np.int32) @@ -33,28 +74,11 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): cu_num_logits_np = np.array([0, 3, 4, 8, 10], dtype=np.int32) num_logits_per_req = np.diff(cu_num_logits_np) idx_mapping_np = np.array([7, 2, 9, 1], dtype=np.int32) - input_batch = SimpleNamespace( - num_reqs=4, - cu_num_logits_np=cu_num_logits_np, - cu_num_logits=torch.from_numpy(cu_num_logits_np).to(device), - idx_mapping_np=idx_mapping_np, - idx_mapping=torch.from_numpy(idx_mapping_np).to(device), - expanded_idx_mapping=torch.from_numpy( - np.repeat(idx_mapping_np, num_logits_per_req) - ).to(device), - expanded_local_pos=torch.from_numpy( - np.concatenate( - [np.arange(count, dtype=np.int32) for count in num_logits_per_req] - ) - ).to(device), - seq_lens_cpu_upper_bound=torch.from_numpy( - np.array([10, 20, 30, 40], dtype=np.int32) - ), + input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device) + rejection_sampler = _make_rejection_sampler( + SimpleNamespace(logprobs_mode=logprobs_mode, return_sampling_mask=False), + num_speculative_steps=3, ) - rejection_sampler = object.__new__(RejectionSampler) - rejection_sampler.sampler = SimpleNamespace(logprobs_mode=logprobs_mode) - rejection_sampler.num_speculative_steps = 3 - rejection_sampler.enable_adaptive_verification = False def fake_verify( self, @@ -75,14 +99,16 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): rejection_sampler._verify = MethodType(fake_verify, rejection_sampler) logits = torch.arange(170, dtype=torch.float32, device=device).view(10, 17) - sampled, num_sampled, chunked_logprobs = rejection_sampler._verify_in_chunks( - logits, - input_batch, - draft_logits=None, - draft_sampled=torch.arange(10, device=device), - pos=torch.arange(10, device=device), - max_chunk_logits=5, - max_num_logprobs=2, + sampled, num_sampled, chunked_logprobs, sampling_mask_tensors = ( + rejection_sampler._verify_in_chunks( + logits, + input_batch, + draft_logits=None, + draft_sampled=torch.arange(10, device=device), + pos=torch.arange(10, device=device), + max_chunk_logits=5, + max_num_logprobs=2, + ) ) score_logits = logits + 1 if logprobs_mode in PROCESSED_LOGPROBS_MODES else logits full_logprobs = rejection_sampler._get_logprobs_tensors( @@ -111,3 +137,155 @@ def test_chunked_scores_match_full_batch(logprobs_mode: str): chunked_logprobs.cu_num_generated_tokens == full_logprobs.cu_num_generated_tokens ) + assert sampling_mask_tensors is None + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_replay_on_off_preserves_rejection_sampling_and_rng(monkeypatch): + device = torch.device("cuda") + num_reqs = 3 + num_speculative_steps = 3 + rows_per_request = num_speculative_steps + 1 + vocab_size = 8 + cu_num_logits_np = np.arange(0, 13, rows_per_request, dtype=np.int32) + idx_mapping_np = np.arange(num_reqs, dtype=np.int32) + input_batch = _make_input_batch(cu_num_logits_np, idx_mapping_np, device) + processed_logits = torch.zeros((12, vocab_size), device=device) + processed_logits[0, 0] = -float("inf") + processed_logits[rows_per_request + 1, 1] = -float("inf") + draft_logits = torch.zeros( + (num_reqs, num_speculative_steps, vocab_size), device=device + ) + draft_sampled = torch.zeros( + (num_reqs, rows_per_request), dtype=torch.int64, device=device + ) + draft_sampled[:, 1:] = torch.tensor([0, 1, 2], device=device) + draft_sampled = draft_sampled.flatten() + sampler = SimpleNamespace( + logprobs_mode="processed_logprobs", + return_sampling_mask=False, + sampling_states=SimpleNamespace( + top_k=SimpleNamespace(np=np.full(num_reqs, vocab_size)), + temperature=SimpleNamespace(gpu=torch.ones(num_reqs, device=device)), + seeds=SimpleNamespace( + gpu=torch.arange(num_reqs, dtype=torch.int64, device=device) + ), + ), + use_fp64_gumbel=False, + apply_sampling_params=lambda logits, *_: logits, + ) + rejection_sampler = _make_rejection_sampler(sampler, num_speculative_steps) + pos = torch.arange(12, dtype=torch.int32, device=device) + + pack_calls: list[tuple[list[int], int, int]] = [] + pack = SamplingMaskTensors.from_logits.__func__ + + def track_pack( + cls, + logits, + cu_num_logits, + num_sampled_tokens, + max_num_kept, + rows_per_request=1, + ): + pack_calls.append( + (cu_num_logits.cpu().tolist(), max_num_kept, rows_per_request) + ) + return pack( + cls, + logits, + cu_num_logits, + num_sampled_tokens, + max_num_kept, + rows_per_request, + ) + + monkeypatch.setattr( + SamplingMaskTensors, + "from_logits", + classmethod(track_pack), + ) + + rng_before = torch.cuda.get_rng_state() + off_sampled, off_counts, off_logprobs, off_masks = ( + rejection_sampler._verify_in_chunks( + processed_logits, + input_batch, + draft_logits, + draft_sampled, + pos, + max_chunk_logits=8, + max_num_logprobs=0, + ) + ) + rng_after_off = torch.cuda.get_rng_state() + assert pack_calls == [] + + sampler.return_sampling_mask = True + on_sampled, on_counts, on_logprobs, on_masks = rejection_sampler._verify_in_chunks( + processed_logits, + input_batch, + draft_logits, + draft_sampled, + pos, + max_chunk_logits=8, + max_num_logprobs=0, + ) + rng_after_on = torch.cuda.get_rng_state() + + assert pack_calls == [ + ([0, 4, 8], vocab_size, rows_per_request), + ([0, 4], vocab_size, rows_per_request), + ] + assert off_masks is None and on_masks is not None + assert on_masks.rows_per_request == rows_per_request + assert on_masks.token_ids.shape == (num_reqs * rows_per_request, vocab_size) + assert on_masks.packed_mask.shape[0] == num_reqs * rows_per_request + assert off_counts.tolist() == [1, 2, 4] + assert torch.equal(off_counts, on_counts) + counts = off_counts.cpu().tolist() + for req_idx, count in enumerate(counts): + assert torch.equal(off_sampled[req_idx, :count], on_sampled[req_idx, :count]) + + mapped_rows = [ + int(input_batch.cu_num_logits_np[req_idx]) + slot_idx + for req_idx, count in enumerate(counts) + for slot_idx in range(count) + ] + assert off_logprobs is not None and on_logprobs is not None + assert torch.equal( + off_logprobs.logprob_token_ids[mapped_rows], + on_logprobs.logprob_token_ids[mapped_rows], + ) + assert torch.equal( + off_logprobs.logprobs[mapped_rows], on_logprobs.logprobs[mapped_rows] + ) + assert torch.equal( + off_logprobs.selected_token_ranks[mapped_rows], + on_logprobs.selected_token_ranks[mapped_rows], + ) + assert torch.equal(rng_before, rng_after_off) + assert torch.equal(rng_after_off, rng_after_on) + + masks = ( + on_masks.to_cpu_nonblocking().tolists(on_counts.cpu().numpy()).to_nested_list() + ) + expected_masks = [ + torch.isfinite(processed_logits[row]).nonzero().flatten().tolist() + for row in mapped_rows + ] + emitted = [ + int(on_sampled[req_idx, slot_idx]) + for req_idx, count in enumerate(on_counts.cpu().tolist()) + for slot_idx in range(count) + ] + expected_logprobs = torch.stack( + [ + torch.log_softmax(processed_logits[row], dim=-1)[token_id] + for row, token_id in zip(mapped_rows, emitted) + ] + ) + + assert masks == expected_masks + assert all(token_id in mask for token_id, mask in zip(emitted, masks)) + assert torch.equal(on_logprobs.logprobs[mapped_rows, 0], expected_logprobs) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 0fd536ff87..c25ed27a28 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1265,9 +1265,15 @@ class VllmConfig: return if not self.use_v2_model_runner: raise ValueError("sampling distribution replay requires Model Runner V2") - if self.speculative_config is not None: + speculative_config = self.speculative_config + if ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + ): raise ValueError( - "sampling distribution replay does not support speculative decoding" + "sampling distribution replay with speculative decoding " + "requires fixed verification boundaries; disable adaptive " + "verification" ) if model_config.is_diffusion: raise ValueError( diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 5ab5636f81..e291b18806 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -441,7 +441,11 @@ class RequestState: sampling_mask = None if finished and self.sampling_mask_chunks: sampling_mask = SamplingMask( - [chunk.token_ids.tolist() for chunk in self.sampling_mask_chunks] + [ + position + for chunk in self.sampling_mask_chunks + for position in chunk.to_nested_list() + ] ) # Concatenate routed experts on finish diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index 1f04c2ccf9..22169b1fda 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -57,19 +57,25 @@ class LogprobsLists(NamedTuple): class SamplingMaskLists(NamedTuple): - """CSR sampling masks; a step slice holds one position (``offsets=None``).""" + """CSR sampling masks; a request slice may contain multiple positions.""" # [num_kept_tokens] token_ids: np.ndarray # [num_positions + 1], or None for a single position offsets: np.ndarray | None = None - # Unused with one position per request; kept for the wire layout. + # [num_requests + 1] for multi-position request batches. cu_num_generated_tokens: list[int] | None = None def slice_request(self, req_idx: int, num_positions: int) -> "SamplingMaskLists": - assert num_positions == 1 and self.offsets is not None + assert self.offsets is not None + cu = self.cu_num_generated_tokens + start = req_idx if cu is None else cu[req_idx] + end = start + num_positions + lo, hi = self.offsets[start], self.offsets[end] + if num_positions == 1: + return SamplingMaskLists(self.token_ids[lo:hi]) return SamplingMaskLists( - self.token_ids[self.offsets[req_idx] : self.offsets[req_idx + 1]] + self.token_ids[lo:hi], self.offsets[start : end + 1] - lo ) def to_nested_list(self) -> list[list[int]]: diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 3c58433318..647b0c297c 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -185,7 +185,7 @@ class AsyncOutput(AsyncModelRunnerOutput): if self.sampling_mask_tensors is not None: self.model_runner_output.sampling_masks = ( - self.sampling_mask_tensors.tolists() + self.sampling_mask_tensors.tolists(self.num_sampled_tokens_np) ) if self.num_nans is not None: diff --git a/vllm/v1/worker/gpu/sample/output.py b/vllm/v1/worker/gpu/sample/output.py index aedff5ce18..2b16883486 100644 --- a/vllm/v1/worker/gpu/sample/output.py +++ b/vllm/v1/worker/gpu/sample/output.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from __future__ import annotations +from collections.abc import Sequence from dataclasses import dataclass from typing import NamedTuple @@ -27,6 +28,7 @@ def _compact_sampling_mask_kernel( logits_ptr, logits_row_stride, logits_col_stride, + cu_num_logits_ptr, num_sampled_tokens_ptr, token_ids_ptr, token_ids_row_stride, @@ -35,25 +37,32 @@ def _compact_sampling_mask_kernel( counts_ptr, vocab_size, max_num_kept, + ROWS_PER_REQUEST: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): """Per row: first ``max_num_kept`` finite-logit ids, the full count, the bitmask.""" - req_idx = tl.program_id(0) - is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0 + output_row = tl.program_id(0) + req_idx = output_row // ROWS_PER_REQUEST + slot_idx = output_row % ROWS_PER_REQUEST + if slot_idx >= tl.load(num_sampled_tokens_ptr + req_idx): + tl.store(counts_ptr + output_row, 0) + return + source_row = tl.load(cu_num_logits_ptr + req_idx) + slot_idx + count = tl.zeros((), dtype=tl.int32) for start_idx in range(0, vocab_size, BLOCK_SIZE): offsets = start_idx + tl.arange(0, BLOCK_SIZE) logits = tl.load( - logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride, + logits_ptr + source_row * logits_row_stride + offsets * logits_col_stride, mask=offsets < vocab_size, other=-float("inf"), ) - keep = (logits > -float("inf")) & (logits < float("inf")) & is_active + keep = (logits > -float("inf")) & (logits < float("inf")) keep_i32 = keep.to(tl.int32) pos = count + tl.cumsum(keep_i32, axis=0) - keep_i32 tl.store( - token_ids_ptr + req_idx * token_ids_row_stride + pos, + token_ids_ptr + output_row * token_ids_row_stride + pos, offsets.to(tl.int32), mask=keep & (pos < max_num_kept), ) @@ -62,12 +71,12 @@ def _compact_sampling_mask_kernel( bits = tl.reshape(keep_i32, (BLOCK_SIZE // 8, 8)) << tl.arange(0, 8)[None, :] byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8) tl.store( - packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets, + packed_mask_ptr + output_row * packed_mask_row_stride + byte_offsets, tl.sum(bits, axis=1).to(tl.uint8), mask=byte_offsets < tl.cdiv(vocab_size, 8), ) - tl.store(counts_ptr + req_idx, count) + tl.store(counts_ptr + output_row, count) # Bounds the [num_reqs, width] int32 buffer; wider rows use the bitmask instead. @@ -78,37 +87,46 @@ class SamplingMaskTensors(NamedTuple): """Device-side masks pending async D2H: compact ids, plus the bitmask as the exact fallback for rows wider than ``max_num_kept``.""" - # [num_requests, max_num_kept] + # [num_requests * rows_per_request, max_num_kept] token_ids: torch.Tensor - # [num_requests, ceil(vocab_size / 8)] + # [num_requests * rows_per_request, ceil(vocab_size / 8)] packed_mask: torch.Tensor - # [num_requests] + # [num_requests * rows_per_request] counts: torch.Tensor vocab_size: int + # Fixed request-major output slots. Ordinary sampling has one slot. + rows_per_request: int = 1 @classmethod def from_logits( cls, logits: torch.Tensor, + cu_num_logits: torch.Tensor, num_sampled_tokens: torch.Tensor, max_num_kept: int, + rows_per_request: int = 1, ) -> SamplingMaskTensors: - """Capture the finite-logit support of every row with a sampled token.""" - num_reqs, vocab_size = logits.shape + """Capture committed target supports in fixed request-major slots.""" + num_reqs = num_sampled_tokens.shape[0] + vocab_size = logits.shape[1] max_num_kept = min(max_num_kept, vocab_size, MAX_COMPACT_SUPPORT) + num_output_rows = num_reqs * rows_per_request device = logits.device token_ids = torch.empty( - (num_reqs, max_num_kept), dtype=torch.int32, device=device + (num_output_rows, max_num_kept), dtype=torch.int32, device=device ) packed_mask = torch.empty( - (num_reqs, (vocab_size + 7) // 8), dtype=torch.uint8, device=device + (num_output_rows, (vocab_size + 7) // 8), + dtype=torch.uint8, + device=device, ) - counts = torch.empty(num_reqs, dtype=torch.int32, device=device) - _compact_sampling_mask_kernel[(num_reqs,)]( + counts = torch.empty(num_output_rows, dtype=torch.int32, device=device) + _compact_sampling_mask_kernel[(num_output_rows,)]( logits, logits.stride(0), logits.stride(1), + cu_num_logits, num_sampled_tokens, token_ids, token_ids.stride(0), @@ -117,9 +135,23 @@ class SamplingMaskTensors(NamedTuple): counts, vocab_size, max_num_kept, + ROWS_PER_REQUEST=rows_per_request, BLOCK_SIZE=8192, ) - return cls(token_ids, packed_mask, counts, vocab_size) + return cls(token_ids, packed_mask, counts, vocab_size, rows_per_request) + + @classmethod + def cat(cls, chunks: Sequence[SamplingMaskTensors]) -> SamplingMaskTensors: + """Join request-contiguous chunks that share one output layout.""" + first = chunks[0] + assert len({(c.vocab_size, c.rows_per_request) for c in chunks}) == 1 + return cls( + torch.cat([chunk.token_ids for chunk in chunks]), + torch.cat([chunk.packed_mask for chunk in chunks]), + torch.cat([chunk.counts for chunk in chunks]), + first.vocab_size, + first.rows_per_request, + ) def to_cpu_nonblocking(self) -> SamplingMaskTensors: if self.token_ids.device.type == "cpu": @@ -129,15 +161,31 @@ class SamplingMaskTensors(NamedTuple): self.packed_mask.to("cpu", non_blocking=True), self.counts.to("cpu", non_blocking=True), self.vocab_size, + self.rows_per_request, ) - def tolists(self) -> SamplingMaskLists: - """CSR over all requests; rows without a sampled token are empty.""" + def tolists( + self, num_sampled_tokens: np.ndarray | None = None + ) -> SamplingMaskLists: + """Convert fixed output rows to the scheduler's CSR representation.""" counts = self.counts.cpu().numpy() token_ids = self.token_ids.cpu().numpy() packed_mask = self.packed_mask.cpu().numpy() width = token_ids.shape[1] + cu_num_generated_tokens = None + if self.rows_per_request == 1: + sampled_rows = np.arange(len(counts)) + else: + assert num_sampled_tokens is not None + active_slots = ( + np.arange(self.rows_per_request)[None, :] < num_sampled_tokens[:, None] + ) + sampled_rows = np.flatnonzero(active_slots) + cu_num_generated_tokens = np.cumsum( + np.concatenate(([0], num_sampled_tokens)) + ).tolist() + def support(row: int) -> np.ndarray: if counts[row] <= width: return token_ids[row, : counts[row]] @@ -147,7 +195,8 @@ class SamplingMaskTensors(NamedTuple): ) return np.flatnonzero(bits).astype(np.int32, copy=False) - supports = [support(row) for row in range(len(counts))] + supports = [support(row) for row in sampled_rows] offsets = np.zeros(len(supports) + 1, dtype=np.int64) np.cumsum([len(s) for s in supports], out=offsets[1:]) - return SamplingMaskLists(np.concatenate(supports), offsets) + flat = np.concatenate([np.empty(0, dtype=np.int32), *supports]) + return SamplingMaskLists(flat, offsets, cu_num_generated_tokens) diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 802adb6f64..7ffc5bf493 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -203,7 +203,10 @@ class Sampler: # Size by the validated top_k batch max; wider supports use the bitmask. max_num_kept = int(np.max(self.sampling_states.top_k.np[idx_mapping_np])) sampling_mask_tensors = SamplingMaskTensors.from_logits( - processed_logits, num_sampled, max_num_kept + processed_logits, + input_batch.cu_num_logits, + num_sampled, + max_num_kept, ) # These are GPU tensors. diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 1983126b77..b96007b6c1 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -18,7 +18,7 @@ from vllm.v1.worker.gpu.input_batch import ( ) from vllm.v1.worker.gpu.metrics.logits import get_num_nans from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores -from vllm.v1.worker.gpu.sample.output import SamplerOutput +from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS from vllm.v1.worker.gpu.spec_decode.rejection_sampler_utils import ( @@ -296,7 +296,12 @@ class RejectionSampler: max_chunk_logits: int, max_num_logprobs: int, verify_draft_sampled: torch.Tensor | None = None, - ) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]: + ) -> tuple[ + torch.Tensor, + torch.Tensor, + LogprobsTensors | None, + SamplingMaskTensors | None, + ]: if verify_draft_sampled is None: verify_draft_sampled = draft_sampled cu_num_logits_np = input_batch.cu_num_logits_np @@ -316,6 +321,14 @@ class RejectionSampler: sampled_chunks: list[torch.Tensor] = [] num_sampled_chunks: list[torch.Tensor] = [] logprobs_chunks: list[LogprobsTensors] = [] + sampling_mask_chunks: list[SamplingMaskTensors] = [] + max_num_kept = None + if self.sampler.return_sampling_mask: + max_num_kept = int( + np.max( + self.sampler.sampling_states.top_k.np[input_batch.idx_mapping_np] + ) + ) for start, end in request_chunks: lo = int(cu_num_logits_np[start]) @@ -346,13 +359,31 @@ class RejectionSampler: ) if chunk_logprobs is not None: logprobs_chunks.append(chunk_logprobs) + if max_num_kept is not None: + sampling_mask_chunks.append( + SamplingMaskTensors.from_logits( + processed_logits, + chunk_cu_num_logits, + num_sampled, + max_num_kept, + self.num_speculative_steps + 1, + ) + ) del processed_logits sampled_chunks.append(sampled) num_sampled_chunks.append(num_sampled) if len(sampled_chunks) == 1: logprobs_tensors = logprobs_chunks[0] if logprobs_chunks else None - return sampled_chunks[0], num_sampled_chunks[0], logprobs_tensors + sampling_mask_tensors = ( + sampling_mask_chunks[0] if sampling_mask_chunks else None + ) + return ( + sampled_chunks[0], + num_sampled_chunks[0], + logprobs_tensors, + sampling_mask_tensors, + ) logprobs_tensors = None if logprobs_chunks: @@ -366,7 +397,12 @@ class RejectionSampler: sampled = torch.cat(sampled_chunks) num_sampled = torch.cat(num_sampled_chunks) - return sampled, num_sampled, logprobs_tensors + sampling_mask_tensors = ( + SamplingMaskTensors.cat(sampling_mask_chunks) + if sampling_mask_chunks + else None + ) + return sampled, num_sampled, logprobs_tensors, sampling_mask_tensors def __call__( self, @@ -400,7 +436,12 @@ class RejectionSampler: input_batch.idx_mapping_np ) chunk_logit_limit = get_max_chunk_logits(logits.shape[1]) - sampled, num_sampled, logprobs_tensors = self._verify_in_chunks( + ( + sampled, + num_sampled, + logprobs_tensors, + sampling_mask_tensors, + ) = self._verify_in_chunks( logits, input_batch, draft_logits, @@ -425,4 +466,5 @@ class RejectionSampler: num_nans=num_nans, num_sampled=num_sampled, num_rejected=num_rejected, + sampling_mask_tensors=sampling_mask_tensors, ) From c9578f10ab3eaf252742960db3fd08f11cf6d25a Mon Sep 17 00:00:00 2001 From: Shuolei Wang <46160365+ShuoleiWang@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:21:06 +0800 Subject: [PATCH 182/248] [Qwen3.8-Flash-Next] Fuse main QK-norm/RoPE/gate and KV-cache write into the QSA pre-indexer launch (#57097) Signed-off-by: Shuolei Wang Signed-off-by: Thien Co-authored-by: Thien --- .buildkite/test_areas/models_basic.yaml | 2 +- ...qsa_pre_indexer.py => test_qsa_prepare.py} | 93 ++++++++++- tests/models/qwen4_exp/test_qsa_reference.py | 19 ++- vllm/models/qwen4_exp/nvidia/indexer_qsa.py | 46 ++++-- .../{qsa_pre_indexer.py => qsa_prepare.py} | 154 +++++++++++++++++- vllm/models/qwen4_exp/nvidia/qsa.py | 58 +++++-- 6 files changed, 325 insertions(+), 47 deletions(-) rename tests/models/qwen4_exp/{test_qsa_pre_indexer.py => test_qsa_prepare.py} (77%) rename vllm/models/qwen4_exp/nvidia/ops/{qsa_pre_indexer.py => qsa_prepare.py} (75%) diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 7c03db5cb4..b817b9a1fb 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -178,7 +178,7 @@ steps: - vllm/v1/attention/backends/short_conv_attn.py - tests/models/qwen4_exp/ commands: - - pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_pre_indexer.py models/qwen4_exp/test_qsa_reference.py + - pytest -v -s models/qwen4_exp/test_hc_ops.py models/qwen4_exp/test_qsa_prepare.py models/qwen4_exp/test_qsa_reference.py mirror: amd: label: ":amd: (MI355 DPX) Qwen4 Exp" diff --git a/tests/models/qwen4_exp/test_qsa_pre_indexer.py b/tests/models/qwen4_exp/test_qsa_prepare.py similarity index 77% rename from tests/models/qwen4_exp/test_qsa_pre_indexer.py rename to tests/models/qwen4_exp/test_qsa_prepare.py index 8d4df3dbc9..14b836c132 100644 --- a/tests/models/qwen4_exp/test_qsa_pre_indexer.py +++ b/tests/models/qwen4_exp/test_qsa_prepare.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Correctness tests for the fused QSA pre-indexer.""" +"""Correctness tests for the fused QSA prepare kernel.""" import pytest import torch @@ -15,9 +15,7 @@ from vllm.models.qwen4_exp.nvidia.ops.qsa import ( qsa_compress_groups_with_ratio, qsa_store_cache_rows, ) -from vllm.models.qwen4_exp.nvidia.ops.qsa_pre_indexer import ( - qsa_pre_indexer, -) +from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON @@ -36,6 +34,7 @@ ROPE_POS_OFFSET = D RTOL = 1.6e-2 ATOL = 1e-2 MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8]) +MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16 def _make_block_table(block_counts): @@ -61,6 +60,84 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all()) +def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict: + """Random main-attention arguments with a paged cache in vLLM's layout.""" + num_blocks = num_tokens // MAIN_PAGE + 2 + cache = torch.zeros( + num_blocks, + MAIN_HK, + MAIN_PAGE, + 2 * MAIN_D, + dtype=torch.uint8 if fp8_cache else torch.bfloat16, + device="cuda", + ).transpose(1, 2) + slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens] + slots[-1] = -1 + norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2 + return dict( + main_qkv=torch.randn( + num_tokens, + 2 * (MAIN_HQ + MAIN_HK) * MAIN_D, + dtype=torch.bfloat16, + device="cuda", + ), + main_q_norm_weight=norm_weights[0], + main_k_norm_weight=norm_weights[1], + main_eps=EPS, + main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache, + main_slot_mapping=slots, + main_k_scale=0.5, + main_v_scale=2.0, + ) + + +def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None: + """Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash.""" + from vllm._custom_ops import reshape_and_cache_flash + from vllm.model_executor.layers.fused_qk_norm_rope import ( + fused_qk_rmsnorm_rope_gate, + ) + + q_gate, k, v = main["main_qkv"].split( + [2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1 + ) + q, k, gate = fused_qk_rmsnorm_rope_gate( + q_gate, + k, + main["main_q_norm_weight"], + main["main_k_norm_weight"], + rope.cos_sin_cache, + positions, + EPS, + MAIN_HQ, + MAIN_HK, + MAIN_D, + rope.rotary_dim, + mrope_section=MROPE_SECTION if positions.ndim == 2 else None, + norm_beta=1.0, + ) + kv_cache = main["main_kv_cache"] + fp8_cache = kv_cache.dtype == torch.float8_e4m3fn + cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache) + key_cache, value_cache = cache.split(MAIN_D, dim=-1) + reshape_and_cache_flash( + k.view(-1, MAIN_HK, MAIN_D), + v.view(-1, MAIN_HK, MAIN_D), + key_cache, + value_cache, + main["main_slot_mapping"], + "fp8" if fp8_cache else "auto", + torch.tensor(main["main_k_scale"], device="cuda"), + torch.tensor(main["main_v_scale"], device="cuda"), + ) + torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL) + assert torch.equal(gate_out.flatten(1), gate) + if fp8_cache: + assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn)) + else: + torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL) + + @requires_qsa_kernels @pytest.mark.usefixtures("default_vllm_config") @pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn]) @@ -86,7 +163,7 @@ def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> N pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"), ], ) -def test_qsa_fused_pre_indexer_matches_unfused( +def test_qsa_fused_prepare_matches_unfused( indexer_dtype, mrope, is_2d_positions, @@ -246,7 +323,9 @@ def test_qsa_fused_pre_indexer_matches_unfused( k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2 fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device) - qsa_pre_indexer( + # The main-attention cache follows the recipe's pairing with the indexer. + main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn) + main_q_out, main_gate_out = qsa_prepare( projected_qk[:, : HQ * D], projected_qk[:, HQ * D :], positions, @@ -266,7 +345,9 @@ def test_qsa_fused_pre_indexer_matches_unfused( compress_ratio=CR, mrope_section=MROPE_SECTION if mrope else None, rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None, + **main, ) + _check_main_outputs(main, main_q_out, main_gate_out, rope, positions) unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D) unfused_query = gemma_rmsnorm( diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index a6f3b6557d..dc431aed7a 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -56,16 +56,24 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( rope_position_offset=0, ), compressed_key_cache=SimpleNamespace(kv_cache=torch.empty(0)), - use_fused_pre_indexer=True, rotary_emb=SimpleNamespace(cos_sin_cache=torch.empty(0)), q_layernorm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6), k_layernorm=SimpleNamespace(weight=torch.ones(1)), compress_ratio=2, ) + attn = SimpleNamespace( + use_fused_qsa_prepare=True, + kv_cache=torch.empty(0, 1, 1, 2), + kv_cache_dtype="auto", + q_norm=SimpleNamespace(weight=torch.ones(1), variance_epsilon=1e-6), + k_norm=SimpleNamespace(weight=torch.ones(1)), + _k_scale_float=1.0, + _v_scale_float=1.0, + ) monkeypatch.setattr( indexer_qsa, - "qsa_pre_indexer", + "qsa_prepare", lambda *args, **kwargs: updates.append((args, kwargs)), ) monkeypatch.setattr( @@ -79,11 +87,14 @@ def test_qsa_mtp_index_share_updates_cache_but_skips_selection( lambda *args, **kwargs: selections.append((args, kwargs)), ) - actual = indexer_qsa.QSAIndexer.forward( + actual, _ = indexer_qsa.QSAIndexer.forward( indexer, torch.zeros(2, 2), torch.tensor([7, 8]), rows, + attn=attn, + qkv=torch.zeros(2, 4), + slot_mapping=torch.arange(2), ) assert actual is rows @@ -512,7 +523,6 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None: _metadata=lambda: (raw_metadata, compressed_metadata), skip_topk=True, index_kv_heads=1, - use_fused_pre_indexer=False, index_n_heads=1, index_head_dim=64, indexer_dtype=torch.bfloat16, @@ -535,6 +545,7 @@ def test_qsa_unfused_cache_update_ignores_padded_qk() -> None: torch.cat((torch.ones_like(padded_keys), padded_keys), dim=-1), torch.zeros(8, dtype=torch.long, device=device), torch.full((5, 5), -1, dtype=torch.int32, device=device), + attn=SimpleNamespace(use_fused_qsa_prepare=False), ) torch.testing.assert_close(raw_cache[0, :, 0], keys[[4, 1, 2, 3]]) expected_compressed = torch.zeros_like(compressed_cache) diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index f4b968f152..14137de695 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Qwen4Exp weight-free QSA indexer.""" -from typing import cast +from typing import TYPE_CHECKING, cast import torch from torch import nn @@ -23,7 +23,10 @@ from ..common.qsa_cache import ( QSAKeyStateCache, canonical_qsa_rope_positions, ) -from .ops.qsa_pre_indexer import qsa_pre_indexer +from .ops.qsa_prepare import qsa_prepare + +if TYPE_CHECKING: + from .qsa import Qwen4ExpQSAAttention def apply_qsa_rope( @@ -235,7 +238,11 @@ class QSAIndexer(nn.Module): projected_qk: torch.Tensor, positions: torch.Tensor, out: torch.Tensor | None = None, - ) -> torch.Tensor: + *, + attn: "Qwen4ExpQSAAttention", + qkv: torch.Tensor | None = None, + slot_mapping: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: """Update side caches and select token indices from pre-projected Q/K. Returns the packed buffer of shape [num_tokens, output_width + 1]: @@ -243,12 +250,17 @@ class QSAIndexer(nn.Module): request-relative token indices, and the trailing column is the row's valid-entry count (the attention kernel's loop bound, never a token index). + + With ``attn.use_fused_qsa_prepare``, the same launch also writes + ``attn``'s K/V into ``attn.kv_cache`` at ``slot_mapping`` and prepares + its Q and gate from ``qkv``, returned as the second element (None + otherwise). ``qkv`` and ``slot_mapping`` are only read in that mode. """ metadata = self._metadata() if metadata is None: # Preserve step-0 indices when later MTP steps reuse the buffer. if self.skip_topk and out is not None: - return out + return out, None result = torch.full( (projected_qk.shape[0], self.packed_output_width), -1, @@ -259,8 +271,8 @@ class QSAIndexer(nn.Module): result[:, -1] = 0 if out is not None: out.copy_(result) - return out - return result + return out, None + return result, None from .ops.qsa import qsa_compress_groups_with_ratio, qsa_store_cache_rows from .ops.qsa_indexer import ( @@ -284,14 +296,20 @@ class QSAIndexer(nn.Module): raw_key_state_cache = self.raw_key_cache compressed_key_cache = self.compressed_key_cache.kv_cache - if self.use_fused_pre_indexer: + main_outputs: tuple[torch.Tensor, torch.Tensor] | None = None + if attn.use_fused_qsa_prepare: + if qkv is None or slot_mapping is None: + raise ValueError("fused QSA prepare requires qkv and slot_mapping") q = projected_q.new_empty( num_tokens, self.index_n_heads, self.index_head_dim, dtype=self.indexer_dtype, ) - qsa_pre_indexer( + main_kv_cache = attn.kv_cache.transpose(1, 2) + if attn.kv_cache_dtype in ("fp8", "fp8_e4m3"): + main_kv_cache = main_kv_cache.view(torch.float8_e4m3fn) + main_outputs = qsa_prepare( projected_q, raw_keys, positions, @@ -315,6 +333,14 @@ class QSAIndexer(nn.Module): if raw_key_state_cache.rope_position_cache is not None else None ), + main_qkv=qkv[:num_tokens], + main_q_norm_weight=attn.q_norm.weight, + main_k_norm_weight=attn.k_norm.weight, + main_eps=attn.q_norm.variance_epsilon, + main_kv_cache=main_kv_cache, + main_slot_mapping=slot_mapping[:num_tokens], + main_k_scale=attn._k_scale_float, + main_v_scale=attn._v_scale_float, ) else: # Unfused reference path @@ -385,7 +411,7 @@ class QSAIndexer(nn.Module): if self.skip_topk: if out is None: raise RuntimeError("QSA top-k reuse requires an output buffer") - return out + return out, main_outputs if out is None: out = torch.empty( @@ -448,7 +474,7 @@ class QSAIndexer(nn.Module): self.token_topk, out, ) - return out + return out, main_outputs __all__ = ["QSAIndexer", "apply_qsa_rope"] diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_prepare.py similarity index 75% rename from vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py rename to vllm/models/qwen4_exp/nvidia/ops/qsa_prepare.py index d996f3977a..6e97a89781 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_pre_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_prepare.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Fused QSA pre-indexer kernel for Qwen4Exp.""" +"""Fused QSA prepare kernel for Qwen4Exp.""" import torch @@ -80,6 +80,27 @@ def _norm_rope( return tl.reshape(result, (TILE_T, TILE_H, D)) +@triton.jit +def _to_dst_dtype(x, dst, scale): + """Round to BF16 like the unfused path, then scale for an FP8 destination.""" + out_ty = dst.dtype.element_ty + x = x.to(tl.bfloat16) + if out_ty == tl.float8e4nv: + x = x.to(tl.float32) / scale + return x.to(out_ty) + + +@triton.jit +def _store_rotated(dst, y, o1, o2, scale): + """Store a normalized head whose first ``2 * len(o1)`` dims are rotated.""" + HALF: tl.constexpr = o1.shape[0] + dims = tl.arange(0, y.shape[0]) + rot = tl.arange(0, HALF) + tl.store(dst + dims, _to_dst_dtype(y, dst, scale), mask=dims >= 2 * HALF) + tl.store(dst + rot, _to_dst_dtype(o1, dst, scale)) + tl.store(dst + HALF + rot, _to_dst_dtype(o2, dst, scale)) + + @triton.jit( do_not_specialize=[ "num_tokens", @@ -88,7 +109,7 @@ def _norm_rope( "num_k_work", ] ) -def _qsa_pre_indexer_kernel( +def _qsa_prepare_kernel( q_ptr, q_stride_token, k_ptr, @@ -132,8 +153,86 @@ def _qsa_pre_indexer_kernel( CACHE_HAS_ROPE_POS: tl.constexpr, MROPE_H: tl.constexpr, MROPE_W: tl.constexpr, + main_qkv_ptr, + main_q_norm_weight_ptr, + main_k_norm_weight_ptr, + main_eps, + main_q_out_ptr, + main_gate_out_ptr, + main_cache_ptr, + main_cache_stride_block, + main_cache_stride_token, + main_cache_stride_head, + main_slots_ptr, + main_k_scale, + main_v_scale, + MAIN_HQ: tl.constexpr, + MAIN_HK: tl.constexpr, + MAIN_D: tl.constexpr, + MAIN_PAGE_SIZE: tl.constexpr, ): pid = tl.program_id(0) + num_index_work = num_k_work + tl.cdiv(num_tokens, TILE_T_Q) * tl.cdiv(HQ, TILE_H_Q) + if pid >= num_index_work: + # Main attention: one program per (token, Q or KV head) after the + # indexer work. RoPE covers the first D // 2 dims, the width of the + # shared cos/sin table. + main_pid = pid - num_index_work + token = main_pid // (MAIN_HQ + MAIN_HK) + head = main_pid % (MAIN_HQ + MAIN_HK) + HALF: tl.constexpr = D // 4 + dims = tl.arange(0, MAIN_D) + rot = tl.arange(0, HALF) + row = main_qkv_ptr + token * (2 * (MAIN_HQ + MAIN_HK) * MAIN_D) + is_k = head >= MAIN_HQ + kv_head = head - MAIN_HQ + if is_k: + src = row + (2 * MAIN_HQ + kv_head) * MAIN_D + weight_ptr = main_k_norm_weight_ptr + else: + src = row + 2 * head * MAIN_D + weight_ptr = main_q_norm_weight_ptr + x = tl.load(src + dims).to(tl.float32) + inv_rms = tl.rsqrt(tl.sum(x * x, axis=0) / MAIN_D + main_eps) + w = tl.load(weight_ptr + dims).to(tl.float32) + 1.0 + y = x * inv_rms * w + x1 = tl.load(src + rot).to(tl.float32) + x2 = tl.load(src + HALF + rot).to(tl.float32) + w1 = tl.load(weight_ptr + rot).to(tl.float32) + 1.0 + w2 = tl.load(weight_ptr + HALF + rot).to(tl.float32) + 1.0 + x1 = (x1 * inv_rms * w1).to(tl.bfloat16).to(tl.float32) + x2 = (x2 * inv_rms * w2).to(tl.bfloat16).to(tl.float32) + pos = tl.load(pos_ptr + token * pos_stride_token).to(tl.int64) + if IS_2D_POSITIONS: + pos_h = tl.load(pos_ptr + pos_stride_axis + token * pos_stride_token) + pos_w = tl.load(pos_ptr + 2 * pos_stride_axis + token * pos_stride_token) + is_h = (rot % 3 == 1) & (rot < 3 * MROPE_H) + is_w = (rot % 3 == 2) & (rot < 3 * MROPE_W) + pos = tl.where( + is_h, pos_h.to(tl.int64), tl.where(is_w, pos_w.to(tl.int64), pos) + ) + cos = tl.load(cos_sin_ptr + pos * (D // 2) + rot).to(tl.float32) + sin = tl.load(cos_sin_ptr + pos * (D // 2) + HALF + rot).to(tl.float32) + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + if is_k: + slot = tl.load(main_slots_ptr + token).to(tl.int64) + if slot >= 0: + dst = ( + main_cache_ptr + + (slot // MAIN_PAGE_SIZE) * main_cache_stride_block + + (slot % MAIN_PAGE_SIZE) * main_cache_stride_token + + kv_head * main_cache_stride_head + ) + _store_rotated(dst, y, o1, o2, main_k_scale) + v = tl.load(src + MAIN_HK * MAIN_D + dims) + tl.store(dst + MAIN_D + dims, _to_dst_dtype(v, dst, main_v_scale)) + else: + out = (token * MAIN_HQ + head) * MAIN_D + _store_rotated(main_q_out_ptr + out, y, o1, o2, None) + gate = tl.load(src + MAIN_D + dims) + tl.store(main_gate_out_ptr + out + dims, gate) + return # K work occupies the first programs; the remaining programs tile Q. This # keeps both paths in one launch while leaving their register shapes # independent. @@ -400,7 +499,7 @@ def _qsa_pre_indexer_kernel( tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot) -def qsa_pre_indexer( +def qsa_prepare( q: torch.Tensor, k: torch.Tensor, positions: torch.Tensor, @@ -421,11 +520,28 @@ def qsa_pre_indexer( compress_ratio: int, mrope_section: tuple[int, int, int] | None, rope_pos_offset: int | None, -) -> None: - """Normalize Q, compress K, then update the circular raw state.""" + main_qkv: torch.Tensor, + main_q_norm_weight: torch.Tensor, + main_k_norm_weight: torch.Tensor, + main_eps: float, + main_kv_cache: torch.Tensor, + main_slot_mapping: torch.Tensor, + main_k_scale: float, + main_v_scale: float, +) -> tuple[torch.Tensor, torch.Tensor]: + """Normalize Q, compress K, then update the circular raw state. + + Also prepares the main attention (QK-norm/RoPE, gate copy, K/V cache write) + and returns its Q and gate. + """ num_tokens = q.shape[0] + main_head_dim = main_kv_cache.shape[-1] // 2 + num_main_kv_heads = main_kv_cache.shape[2] + num_main_q_heads = main_qkv.shape[1] // (2 * main_head_dim) - num_main_kv_heads + main_q_out = main_qkv.new_empty(num_tokens, num_main_q_heads, main_head_dim) + main_gate_out = torch.empty_like(main_q_out) if num_tokens == 0: - return + return main_q_out, main_gate_out num_q_heads, head_dim = q_out.shape[1:] assert cos_sin_cache.shape[-1] * 2 == head_dim assert q.shape == (num_tokens, num_q_heads * head_dim) @@ -450,6 +566,9 @@ def qsa_pre_indexer( pos_stride_axis, pos_stride_token = 0, positions.stride(0) section = mrope_section if mrope_section is not None else (0, 0, 0) assert len(section) == 3 + qkv_width = 2 * (num_main_q_heads + num_main_kv_heads) * main_head_dim + assert main_qkv.shape == (num_tokens, qkv_width) and main_qkv.is_contiguous() + assert main_slot_mapping.shape == (num_tokens,) if num_tokens <= 4096: TILE_T_Q, TILE_H_Q = 2, 2 @@ -457,7 +576,8 @@ def qsa_pre_indexer( TILE_T_Q, TILE_H_Q = 2, 4 num_k_work = k_work_metadata.shape[0] num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q) - _qsa_pre_indexer_kernel[(num_k_work + num_q_work,)]( + num_main_work = num_tokens * (num_main_q_heads + num_main_kv_heads) + _qsa_prepare_kernel[(num_k_work + num_q_work + num_main_work,)]( q, q.stride(0), k, @@ -501,8 +621,26 @@ def qsa_pre_indexer( CACHE_HAS_ROPE_POS=cache_has_rope_pos, MROPE_H=section[1], MROPE_W=section[2], + main_qkv_ptr=main_qkv, + main_q_norm_weight_ptr=main_q_norm_weight, + main_k_norm_weight_ptr=main_k_norm_weight, + main_eps=main_eps, + main_q_out_ptr=main_q_out, + main_gate_out_ptr=main_gate_out, + main_cache_ptr=main_kv_cache, + main_cache_stride_block=main_kv_cache.stride(0), + main_cache_stride_token=main_kv_cache.stride(1), + main_cache_stride_head=main_kv_cache.stride(2), + main_slots_ptr=main_slot_mapping, + main_k_scale=main_k_scale, + main_v_scale=main_v_scale, + MAIN_HQ=num_main_q_heads, + MAIN_HK=num_main_kv_heads, + MAIN_D=main_head_dim, + MAIN_PAGE_SIZE=main_kv_cache.shape[1], num_warps=1, ) + return main_q_out, main_gate_out -__all__ = ["qsa_pre_indexer"] +__all__ = ["qsa_prepare"] diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index b6da89a863..62aa6459fb 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -407,6 +407,12 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): quant_config=quant_config, prefix=f"{prefix}.indexer", ) + # One launch does the indexer prepare, the main QK-norm/RoPE/gate and + # the main K/V cache write (see QSAIndexer.forward); otherwise all of + # them take the separate kernels. + self.use_fused_qsa_prepare = ( + self.use_fused_qk_norm_rope_gate and self.indexer.use_fused_pre_indexer + ) max_tokens = vllm_config.scheduler_config.max_num_batched_tokens # PACKED selection buffer: the trailing column holds each row's # valid-entry count (written by the expand kernel) — never a token @@ -446,12 +452,15 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): self, projected_qk: torch.Tensor, positions: torch.Tensor, - query: torch.Tensor, - key: torch.Tensor, - value: torch.Tensor, + query: torch.Tensor | None, + key: torch.Tensor | None, + value: torch.Tensor | None, output: torch.Tensor, - output_gate: torch.Tensor, + output_gate: torch.Tensor | None, + qkv: torch.Tensor, ) -> None: + # query/key/value/output_gate are None when the fused prepare runs + # inside the indexer launch. metadata = get_forward_context().attn_metadata if isinstance(metadata, list): metadata = metadata[0] @@ -469,21 +478,29 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): ) if side_metadata.num_actual_tokens != num_tokens: raise RuntimeError("QSA main and side metadata token counts disagree") - selected = self.indexer( + selected, main_outputs = self.indexer( projected_qk, positions, self.topk_indices_buffer[:num_tokens], + attn=self, + qkv=qkv, + slot_mapping=main_metadata.slot_mapping, ) if selected.shape != (num_tokens, self.indexer.packed_output_width): raise RuntimeError("QSA indexer returned an invalid selection shape") impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl) - impl.do_kv_cache_update( - self, - key, - value, - self.kv_cache, - main_metadata.slot_mapping, - ) + if main_outputs is None: + assert key is not None and value is not None + impl.do_kv_cache_update( + self, + key, + value, + self.kv_cache, + main_metadata.slot_mapping, + ) + else: + query, output_gate = main_outputs + assert query is not None and output_gate is not None impl.forward_qsa( self, query, @@ -503,13 +520,17 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): hidden_states: torch.Tensor, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) - q, k, v, gate = self._project_qkv_gate(qkv, positions) - assert gate is not None num_tokens = hidden_states.shape[0] - query = q.view(num_tokens, self.num_heads, self.head_dim) - key = k.view(num_tokens, self.num_kv_heads, self.head_dim) - value = v.view(num_tokens, self.num_kv_heads, self.head_dim) - attn_output = torch.empty_like(query) + if not self.use_fused_qsa_prepare: + q, k, v, gate = self._project_qkv_gate(qkv, positions) + assert gate is not None + query = q.view(num_tokens, self.num_heads, self.head_dim) + key = k.view(num_tokens, self.num_kv_heads, self.head_dim) + value = v.view(num_tokens, self.num_kv_heads, self.head_dim) + else: + # Norm/RoPE/gate and the K/V cache write happen inside _run_qsa. + query = key = value = gate = None + attn_output = qkv.new_empty(num_tokens, self.num_heads, self.head_dim) # Keep the index projection outside the eager break. projected_qk, _ = self.indexer.index_qk_proj(hidden_states) self._run_qsa( @@ -520,6 +541,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase): value, attn_output, gate, + qkv, ) flat_output = attn_output.view(num_tokens, -1) output, _ = self.o_proj(flat_output) From 4a9d6f957203902ea6bff0b275c26876e5a47e46 Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Thu, 1 Oct 2026 14:43:38 +0800 Subject: [PATCH 183/248] [Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935) Signed-off-by: aoshen02 Signed-off-by: fh3 Co-authored-by: Codex Co-authored-by: fh3 Co-authored-by: Claude Fable 5.1 --- docs/training/sampling_mask.md | 4 +- .../server/src/routes/inference/generate.rs | 20 ++++++- .../src/routes/inference/generate/types.rs | 2 + .../test_generate_stream.py | 43 +++++++++++++- tests/v1/engine/test_output_processor.py | 57 ++++++++++++++++++- .../scale_out/token_in_token_out/serving.py | 5 ++ vllm/outputs.py | 6 ++ vllm/v1/engine/output_processor.py | 4 +- 8 files changed, 135 insertions(+), 6 deletions(-) diff --git a/docs/training/sampling_mask.md b/docs/training/sampling_mask.md index c8425e0245..7de2366800 100644 --- a/docs/training/sampling_mask.md +++ b/docs/training/sampling_mask.md @@ -53,6 +53,8 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint: } ``` +With `stream=true`, each chunk carries the masks of the tokens it emits. + ## Requirements | Requirement | Reason | @@ -145,5 +147,3 @@ Each position then carries the sampled token's logprob and the top-128 - **Engine-level flag:** `--return-sampling-mask` globally disables the FlashInfer fused sampler. All requests pay the cost of the PyTorch sampling path, even if they don't need the mask. -- **No streaming support:** The mask is returned only in the final response, - not in intermediate streaming chunks. diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 15caedaf17..0e682f7c45 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -231,6 +231,7 @@ async fn generate_chunk_stream( logprobs, finish_reason: finish_reason.map(|reason| reason.as_str().to_string()), token_ids, + sampling_mask: output.sampling_mask.map(|mask| mask.rows), }], usage: include_continuous_usage .then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)), @@ -331,6 +332,7 @@ fn collect_generate( logprobs, finish_reason: Some(finish_reason), token_ids: collected.token_ids, + sampling_mask: collected.sampling_mask.map(|mask| mask.rows), }], prompt_logprobs, prompt_token_id_logprobs: collected @@ -516,6 +518,7 @@ mod tests { use futures::{TryStreamExt as _, stream}; use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange}; use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; + use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_llm::GeneratePromptInfo; use super::*; @@ -657,6 +660,18 @@ mod tests { .expect("collect chunks") } + #[tokio::test] + async fn generate_chunk_stream_returns_sampling_mask() { + let mut output = stream_output(Some(&[11]), vec![33], Some(FinishReason::Length)); + output.sampling_mask = Some(SamplingMask { + rows: vec![vec![33, 44]], + }); + + let chunks = collect_chunks(vec![output], false, None).await; + + assert_eq!(chunks[0].choices[0].sampling_mask, Some(vec![vec![33, 44]])); + } + #[tokio::test] async fn generate_chunk_stream_omits_prompt_metadata_by_default() { let chunks = collect_chunks( @@ -792,7 +807,9 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, prompt_token_ids: vec![10, 20], - sampling_mask: None, + sampling_mask: Some(SamplingMask { + rows: vec![vec![30, 40]], + }), spec_decode_metrics: None, }; @@ -813,6 +830,7 @@ mod tests { assert!(response.prompt_token_ids.is_none()); assert!(response.mm_placeholders.is_none()); + assert_eq!(response.choices[0].sampling_mask, Some(vec![vec![30, 40]])); } #[test] diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 78fe085b83..b0b93d7080 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -64,6 +64,7 @@ pub(super) struct GenerateResponseChoice { pub logprobs: Option, pub finish_reason: Option, pub token_ids: Vec, + pub sampling_mask: Option>>, } /// Mirrors the Python vLLM `GenerateResponseStreamChoice` class. @@ -74,6 +75,7 @@ pub(super) struct GenerateResponseStreamChoice { pub logprobs: Option, pub finish_reason: Option, pub token_ids: Vec, + pub sampling_mask: Option>>, } /// Mirrors the Python vLLM `GenerateStreamResponse` class. diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index 28f8a505f8..53ff1dc598 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -34,7 +34,7 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItem, PlaceholderRange, ) -from vllm.outputs import CompletionOutput, RequestOutput +from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask from vllm.renderers import renderer_from_config from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams @@ -1154,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached(): # Zero cached tokens must be present, not omitted assert usage_chunk["usage"]["prompt_tokens_details"] is not None assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_mask", [False, True]) +async def test_stream_sampling_mask_matches_each_token_chunk(with_mask): + engine = _mock_engine() + + async def generate(*args, **kwargs): + for position, tokens in enumerate(([10], [20, 30])): + result = _make_request_output( + "req-mask", + token_ids=list(tokens), + finish_reason="length" if position else None, + finished=bool(position), + ) + if with_mask: + result.outputs[0].sampling_mask = SamplingMask( + [[token, token + 1] for token in tokens] + ) + yield result + + engine.generate = MagicMock(side_effect=generate) + serving = _build_serving_tokens(engine) + response = await serving.serve_tokens( + GenerateRequest( + model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True + ) + ) + chunks = _parse_sse_chunks([chunk async for chunk in response]) + choices = [ + choice + for chunk in chunks + if isinstance(chunk, dict) + for choice in chunk.get("choices", []) + ] + assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]] + for choice in choices: + expected = ( + [[token, token + 1] for token in choice["token_ids"]] if with_mask else None + ) + assert choice["sampling_mask"] == expected diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 53b8e0f552..40a6883aa9 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -18,7 +18,7 @@ from tests.v1.engine.utils import ( from vllm import PoolingParams from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs from vllm.lora.request import LoRARequest -from vllm.outputs import CompletionOutput, RequestOutput +from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.v1.engine import ( @@ -1575,3 +1575,58 @@ def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors): output_processor.abort_requests([request.request_id], internal=True) else: output_processor.abort_requests([request.external_req_id], internal=False) + + +@pytest.mark.parametrize("output_kind", list(RequestOutputKind)) +def test_sampling_masks_follow_output_kind(output_kind): + state = RequestState.__new__(RequestState) + state.detokenizer = MagicMock() + state.detokenizer.get_next_output_text.return_value = "" + state.logprobs_processor = MagicMock() + state.logprobs_processor.logprobs = None + state.logprobs_processor.cumulative_logprob = None + state.output_kind = output_kind + state.request_index = 0 + state.sampling_mask_chunks = [] + state.routed_experts_chunks = [] + state.spec_decode_metrics = None + + supports = [[10, 11], [20], [30, 31]] + masks = [] + for position, support in enumerate(supports): + state.sampling_mask_chunks.append(SamplingMaskLists(np.asarray(support))) + finish = FinishReason.LENGTH if position == len(supports) - 1 else None + mask = state._new_completion_output([support[0]], finish, None).sampling_mask + masks.append(mask.token_ids if mask is not None else None) + + if output_kind == RequestOutputKind.DELTA: + assert masks == [[support] for support in supports] + else: + assert masks == [None, None, supports] + + +def test_request_output_add_merges_delta_sampling_masks(): + def delta(token_ids, sampling_mask): + completion = CompletionOutput( + index=0, + text="", + token_ids=token_ids, + cumulative_logprob=None, + logprobs=None, + sampling_mask=sampling_mask, + ) + return RequestOutput( + request_id="request", + prompt=None, + prompt_token_ids=[1], + prompt_logprobs=None, + outputs=[completion], + finished=False, + ) + + merged = delta([], None) + merged.add(delta([10], SamplingMask([[10, 11]])), aggregate=True) + merged.add(delta([20], SamplingMask([[20]])), aggregate=True) + + assert merged.outputs[0].token_ids == [10, 20] + assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]] diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index ade103bbbc..249ed48093 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -515,6 +515,10 @@ class ServingTokens(GenerateBaseServing): else None ) + sampling_mask = None + if output.sampling_mask is not None: + sampling_mask = output.sampling_mask.token_ids + chunk = GenerateStreamResponse( request_id=request_id, choices=[ @@ -524,6 +528,7 @@ class ServingTokens(GenerateBaseServing): finish_reason=finish_reason, token_ids=as_list(delta_token_ids), routed_experts=routed_experts_b64, + sampling_mask=sampling_mask, ) ], ) diff --git a/vllm/outputs.py b/vllm/outputs.py index 049d795f78..8bbbbdf40c 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -197,6 +197,12 @@ class RequestOutput: if next_completion.logprobs: assert completion.logprobs is not None completion.logprobs.extend(next_completion.logprobs) # type: ignore[arg-type] + if next_completion.sampling_mask is not None: + if completion.sampling_mask is None: + completion.sampling_mask = SamplingMask([]) + completion.sampling_mask.token_ids.extend( + next_completion.sampling_mask.token_ids + ) completion.cumulative_logprob = ( next_completion.cumulative_logprob ) diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index e291b18806..89b3a9cf09 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -439,7 +439,7 @@ class RequestState: logprobs = logprobs[-num_new_tokens:] if num_new_tokens else logprobs[:0] sampling_mask = None - if finished and self.sampling_mask_chunks: + if (delta or finished) and self.sampling_mask_chunks: sampling_mask = SamplingMask( [ position @@ -447,6 +447,8 @@ class RequestState: for position in chunk.to_nested_list() ] ) + if delta: + self.sampling_mask_chunks.clear() # Concatenate routed experts on finish routed_experts = None From 87a4bf664f4b2344eced5a10add977b750d4b7d5 Mon Sep 17 00:00:00 2001 From: Luca Motz Date: Thu, 1 Oct 2026 08:53:06 +0200 Subject: [PATCH 184/248] [Perf][GDN] Slice pure spec-decode rows instead of a host-mask gather (#58763) Signed-off-by: Luca Motz Co-authored-by: Claude Opus 5.5 Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> --- .../v1/attention/test_gdn_metadata_builder.py | 18 ++++++++++++++++-- vllm/v1/attention/backends/gdn_attn.py | 16 +++++++++++----- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index 0256a13141..70a77f5929 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -65,6 +65,17 @@ GDN_BUILD_TEST_CASES = { expected_num_prefill_tokens=0, expected_num_spec_decodes=2, ), + # Padded (CUDA graph) sequences trail the spec decodes + "pure_spec_decode_with_padding": GDNBuildTestCase( + seq_lens=[50, 30, 16], + query_lens=[3, 3, 0], + num_decode_draft_tokens=[2, 2, -1], + num_speculative_tokens=2, + expected_num_decodes=0, + expected_num_prefills=0, + expected_num_prefill_tokens=0, + expected_num_spec_decodes=2, + ), # No speculative config at all — standard decode path "pure_regular_decode": GDNBuildTestCase( seq_lens=[40, 30, 20], @@ -143,8 +154,9 @@ def _create_gdn_builder( model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE, ) - if full_cuda_graph: - vllm_config.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE + vllm_config.compilation_config.cudagraph_mode = ( + CUDAGraphMode.FULL_AND_PIECEWISE if full_cuda_graph else CUDAGraphMode.NONE + ) if num_speculative_tokens > 0: vllm_config.speculative_config = SpeculativeConfig( method="ngram", @@ -197,6 +209,8 @@ def test_gdn_build_classification(test_case: GDNBuildTestCase): assert meta.num_prefills == test_case.expected_num_prefills assert meta.num_prefill_tokens == test_case.expected_num_prefill_tokens assert meta.num_spec_decodes == test_case.expected_num_spec_decodes + if meta.spec_state_indices_tensor is not None: + assert len(meta.spec_state_indices_tensor) == meta.num_spec_decodes @pytest.mark.parametrize("mamba_cache_mode", ["none", "align"]) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index eb616644f3..6b2c6b292b 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -361,9 +361,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] non_spec_token_indx = torch.empty( 0, dtype=torch.int32, device=query_start_loc.device ) - # Filter by spec_sequence_masks to exclude padded sequences + # Padded sequences trail the spec decodes, so slice them off + # rather than gather with the host mask (an H2D copy + a kernel). spec_state_indices_tensor = block_table_tensor[ - spec_sequence_masks_cpu, : self.num_spec + 1 + :num_spec_decodes, : self.num_spec + 1 ] non_spec_state_indices_tensor = None # Padded sequences are always at the back, so the first @@ -421,7 +422,10 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] ) assert num_accepted_tokens is not None - num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] + if num_prefills == 0 and num_decodes == 0: + num_accepted_tokens = num_accepted_tokens[:num_spec_decodes] + else: + num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu] chunk_indices: torch.Tensor | None = None chunk_offsets: torch.Tensor | None = None @@ -626,10 +630,12 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] non_spec_indices = blk_table[:, 0] if m.num_prefills > 0: prefill_indices = non_spec_indices[m.num_decodes :] + elif m.num_prefills == 0: + # Same as build(): padded sequences trail the spec decodes. + spec_indices = blk_table[: m.num_spec_decodes, : self.num_spec + 1] else: spec_indices = blk_table[masks, : self.num_spec + 1] - if m.num_prefills > 0: - non_spec_indices = prefill_indices = blk_table[~masks, 0] + non_spec_indices = prefill_indices = blk_table[~masks, 0] if self._stage_spec_decode( m.num_prefills, m.num_decodes, m.num_spec_decodes, m.num_spec_decode_tokens From aee8fe202a2fbd79a7a33cae70b310edc5960aa5 Mon Sep 17 00:00:00 2001 From: Mehdi Ghanimifard Date: Thu, 1 Oct 2026 09:02:52 +0200 Subject: [PATCH 185/248] [Bugfix][ROCm] Fix race on AITER MLA FP8 prefill scheduling metadata under async scheduling (#58887) Signed-off-by: Mehdi Ghanimifard Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung --- .../attention/backends/mla/rocm_aiter_mla.py | 60 +++++++++++++++---- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index f48dedb1d6..ab6c11d41a 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -948,6 +948,31 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): device=device, ) + # get_ps_metadata_v1 builds the plan on the host and writes its outputs with + # blocking hipMemcpy that is not ordered on the current stream. Writing the + # device buffers directly lets step N+1's build (async scheduling) overwrite a + # plan that step N's prefill kernels are still reading. Build into pinned host + # staging instead and copy on the current stream. Two staging slots, each + # reused only after its previous H2D copy has completed. + self._fp8_ps_device_outputs = ( + self.fp8_ps_work_indptr, + self.fp8_ps_work_info, + self.fp8_ps_reduce_indptr, + self.fp8_ps_reduce_final_map, + self.fp8_ps_reduce_partial_map, + ) + self._fp8_ps_staging = [ + tuple( + torch.empty(t.shape, dtype=t.dtype, device="cpu", pin_memory=True) + for t in self._fp8_ps_device_outputs + ) + for _ in range(2) + ] + # One event per slot, recorded after that slot's H2D copy. synchronize() + # on an event that has not been recorded yet returns immediately. + self._fp8_ps_staging_free = [torch.cuda.Event(), torch.cuda.Event()] + self._fp8_ps_slot = 0 + from vllm.platforms import current_platform from vllm.v1.worker.workspace import current_workspace_manager @@ -1043,6 +1068,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): kvlen_granularity = 128 block_size = 1 # non-paged: each "page" is one token + slot = self._fp8_ps_slot + self._fp8_ps_slot ^= 1 + staging_free = self._fp8_ps_staging_free[slot] + with gpu_sync_allowed(): + staging_free.synchronize() + ( + work_indptr_host, + work_info_host, + reduce_indptr_host, + reduce_final_map_host, + reduce_partial_map_host, + ) = self._fp8_ps_staging[slot] + + # work_metadata is not written by the host planner; it stays on device. get_ps_metadata_v1( qo_indptr_cpu, kv_indptr_cpu, @@ -1050,17 +1089,20 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): gqa_ratio, num_head_k, self.fp8_ps_work_metadata, - self.fp8_ps_work_indptr, - self.fp8_ps_work_info, - self.fp8_ps_reduce_indptr, - self.fp8_ps_reduce_final_map, - self.fp8_ps_reduce_partial_map, + work_indptr_host, + work_info_host, + reduce_indptr_host, + reduce_final_map_host, + reduce_partial_map_host, qhead_granularity=qhead_granularity, qlen_granularity=qlen_granularity, kvlen_granularity=kvlen_granularity, block_size=block_size, is_causal=True, ) + for dst, src in zip(self._fp8_ps_device_outputs, self._fp8_ps_staging[slot]): + dst.copy_(src, non_blocking=True) + staging_free.record() total_prefill_tokens = int(qo_indptr_cpu[-1].item()) kv_indices = torch.arange( @@ -1068,12 +1110,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): ) # The actual number of active partial tiles for this batch is the - # final value of reduce_indptr. Resolving it here (during metadata - # build) keeps it off the per-layer forward path where a sync would - # break CUDA Graph capture. Using the device-side reduce_indptr is - # acceptable since build is allowed to incur an occasional sync. - with gpu_sync_allowed(): - num_partial_tiles = int(self.fp8_ps_reduce_indptr[-1].item()) + # final value of reduce_indptr, read from the host plan (no GPU sync). + num_partial_tiles = int(reduce_indptr_host[-1]) # Attach PS metadata to the metadata object so forward_mha can read it. metadata.fp8_prefill_qo_indptr = qo_indptr From 4d6874d13756868453f78d5e717b2fbe9f8281d0 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:13:57 -0400 Subject: [PATCH 186/248] [GLM 5.3 Perf] Skip qlnorm calculation for MHA, 4.4~7.7% E2E TTFT Improvement (#58845) Signed-off-by: yewentao256 --- .../test_fused_deepseek_v32_norm_rope.py | 46 +++++++++++++++---- .../layers/test_mla_short_prefill_indexer.py | 8 +++- vllm/models/deepseek_v32/attention.py | 30 ++++++++++-- vllm/models/deepseek_v32/common/kernels.py | 1 + .../nvidia/ops/fused_q_cutedsl.py | 40 +++++++++++----- 5 files changed, 99 insertions(+), 26 deletions(-) diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index df9d45a99d..63bbe0c95f 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -24,6 +24,7 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses). """ +from functools import partial from typing import NamedTuple import pytest @@ -880,22 +881,36 @@ def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig): @pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS) @pytest.mark.parametrize("num_tokens", [1, 17, 512]) @pytest.mark.parametrize("has_indexer", [True, False]) -def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig): +@pytest.mark.parametrize("use_mha", [True, False]) +@pytest.mark.parametrize( + "index_interleave,cache_dtype", [(False, torch.float32), (True, torch.bfloat16)] +) +@pytest.mark.parametrize("capture", [False, True]) +def test_fused_q_bf16_query( + num_tokens: int, + has_indexer: bool, + use_mha: bool, + index_interleave: bool, + cache_dtype: torch.dtype, + capture: bool, + cfg: ModelConfig, +): """bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is - produced (bf16, unquantized); ql_nope is consumed directly by the caller.""" + produced; MHA does not need an absorbed ql_nope projection.""" torch.manual_seed(6) dev = "cuda" max_pos = 8192 pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos + # Attention takes q_pe as a strided slice of the full Q projection. q_pe = torch.randn( - num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16 - ) + num_tokens, cfg.num_heads, 4 * cfg.rope_dim, device=dev, dtype=torch.bfloat16 + )[..., -cfg.rope_dim :] ql_nope = torch.randn( num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16 ) q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32) - q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) + q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype) index_q = index_w = idx_cos_sin = None if has_indexer: @@ -909,24 +924,35 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig index_w = torch.randn( num_tokens, cfg.index_heads, device=dev, dtype=torch.float32 ) - idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev) + idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype) - iq_fp8, iw_out, q_pe_out = K.fused_q( + run = partial( + K.fused_q, pos, q_pe, q_cos_sin, index_q, idx_cos_sin, - ql_nope, + q_pe if use_mha else ql_nope, q_scale, index_w, cfg.index_head_dim**-0.5, cfg.index_heads**-0.5, has_indexer=has_indexer, - index_rope_interleave=False, + index_rope_interleave=index_interleave, quantize_mqa=False, ) + iq_fp8, iw_out, q_pe_out = run() + if capture: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + iq_fp8, iw_out, q_pe_out = run() + q_pe.mul_(0.5) + if index_q is not None: + index_q.mul_(1.5) + graph.replay() + # MQA query: only the RoPE'd q_pe, bf16, unquantized. assert q_pe_out.dtype == torch.bfloat16 assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim) @@ -945,7 +971,7 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig index_q.float(), pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads), idx_cos_sin, - interleave=False, + interleave=index_interleave, ) q_ref, scale_ref = ue8m0_quant(iq_ref) assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)") diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 77c52fb291..2e4d4d3d67 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -267,9 +267,11 @@ def test_skipped_k_cache_insert_accepts_no_k( @pytest.mark.parametrize("fp8_query", [False, True]) +@pytest.mark.parametrize("prepared_mha", [False, True]) def test_deepseek_v32_dispatches_selected_mha( monkeypatch: pytest.MonkeyPatch, fp8_query: bool, + prepared_mha: bool, ) -> None: attn_metadata = SimpleNamespace(num_actual_tokens=2) kv_cache = torch.empty(1) @@ -301,7 +303,11 @@ def test_deepseek_v32_dispatches_selected_mha( q_nope = torch.randn(2, 1, 2) q_pe = torch.randn(2, 1, 2) mqa_q = torch.randn(2, 1, 2) - mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1) + if fp8_query and not prepared_mha: + mqa_q = mqa_q.to(torch.float8_e4m3fn) + mha_q = torch.cat( + (q_nope, q_pe + 1 if fp8_query and not prepared_mha else mqa_q), dim=-1 + ) kv_c = torch.empty(2, 2) k_pe = torch.empty(2, 2) output = torch.empty(2, 2) diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 4e63076414..d352002009 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -7,7 +7,7 @@ import torch.nn as nn from transformers import DeepseekV2Config, DeepseekV3Config from vllm.compilation.breakable_cudagraph import eager_break_during_capture -from vllm.config import CacheConfig, VllmConfig +from vllm.config import CacheConfig, CUDAGraphMode, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed.parallel_state import get_tp_group from vllm.forward_context import get_forward_context @@ -283,6 +283,18 @@ class DeepseekV32Attention(MLAAttention): is_neox_style=not getattr(config, "indexer_rope_interleave", False), ) + def _should_prepare_mqa_query( + self, + attn_metadata: "MLACommonMetadata | None", + cudagraph_runtime_mode: CUDAGraphMode, + ) -> bool: + return ( + attn_metadata is None + or not self._use_sparse_mha(attn_metadata) + or cudagraph_runtime_mode == CUDAGraphMode.FULL + or torch.cuda.is_current_stream_capturing() + ) + def forward( # type: ignore[override] self, positions: torch.Tensor, @@ -321,6 +333,9 @@ class DeepseekV32Attention(MLAAttention): layer_attn_metadata, _, _, _ = get_attention_context(self.layer_name) self.impl.prepare_for_batch(layer_attn_metadata) + prepare_mqa_query = self._should_prepare_mqa_query( + layer_attn_metadata, forward_context.cudagraph_runtime_mode + ) if self.indexer is not None and not self.skip_topk: has_indexer = True indexer_k_norm_w = self.indexer.k_norm.weight @@ -404,7 +419,10 @@ class DeepseekV32Attention(MLAAttention): q = self.q_b_proj(q_c)[0].view(-1, self.num_local_heads, self.qk_head_dim) q_nope, q_pe = q.split([self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1) - ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1) + if prepare_mqa_query: + ql_nope = torch.bmm(q_nope.transpose(0, 1), self.W_UK_T).transpose(0, 1) + else: + ql_nope = q_nope if self.indexer is not None and not self.skip_topk: index_q = self.indexer.wq_b(q_c)[0] @@ -425,7 +443,7 @@ class DeepseekV32Attention(MLAAttention): indexer_n_head_scale, has_indexer=has_indexer, index_rope_interleave=self._index_rope_interleave, - quantize_mqa=self._fp8_query, + quantize_mqa=self._fp8_query and prepare_mqa_query, ) self._sparse_indexer_and_attn( @@ -535,7 +553,11 @@ class DeepseekV32Attention(MLAAttention): if self._use_sparse_mha(attn_metadata): assert kv_c is not None and k_pe is not None - mha_q_pe = self.rotary_emb(positions, q_pe)[0] if self._fp8_query else mqa_q + mha_q_pe = ( + self.rotary_emb(positions, q_pe)[0] + if mqa_q.dtype != q_pe.dtype + else mqa_q + ) mha_q = torch.cat((q_nope, mha_q_pe), dim=-1) self.forward_impl( mha_q, diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index e0259c46fa..255b1f5aaf 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -1052,6 +1052,7 @@ def fused_q( index_weights_out, has_indexer=has_indexer, index_rope_interleave=index_rope_interleave, + quantize_mqa=quantize_mqa, ) return index_q_fp8, index_weights_out, mqa_q diff --git a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py index dec1e6f034..e0c58a7e78 100644 --- a/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py +++ b/vllm/models/deepseek_v32/nvidia/ops/fused_q_cutedsl.py @@ -35,10 +35,12 @@ def is_fused_q_cutedsl_supported( ) -> bool: if not ( current_platform.has_device_capability(100) - and quantize_mqa - and q_pe.dtype == ql_nope.dtype == torch.bfloat16 + and q_pe.dtype == torch.bfloat16 and q_pe.shape[-1] == 64 - and ql_nope.shape[-1] == 512 + and ( + not quantize_mqa + or (ql_nope.dtype == torch.bfloat16 and ql_nope.shape[-1] == 512) + ) # One warp per head group; a non-multiple would trip the kernel's own # assert instead of falling back to Triton. and q_pe.shape[1] % 4 == 0 @@ -68,9 +70,10 @@ def fused_q_cutedsl( idx_weights_out: torch.Tensor, has_indexer: bool = True, index_rope_interleave: bool = True, + quantize_mqa: bool = True, ) -> None: _, num_heads, rope_dim = q_pe.shape - _, _, nope_dim = ql_nope.shape + nope_dim = ql_nope.shape[-1] if quantize_mqa else 0 _, num_idx_heads, idx_dim = idx_q.shape if has_indexer: @@ -98,8 +101,8 @@ def fused_q_cutedsl( positions, q_pe, rope_cache, - ql_nope, - q_scale.view(1), + ql_nope if quantize_mqa else None, + q_scale.view(1) if quantize_mqa else None, mqa_output, idx_q, idx_rope_cache, @@ -121,7 +124,7 @@ class FusedQKernel: index_rope_interleave: bool, ) -> None: assert rope_dim == 64 - assert nope_dim == 512 + assert nope_dim in (0, 512) assert idx_dim in (128, 0) self.rope_dim = rope_dim @@ -262,6 +265,17 @@ class FusedQKernel: cute.arch.griddepcontrol_wait() pos = positions[token_id] + # no need to do Q calculation for MHA + if cutlass.const_expr(self.nope_dim == 0): + q0, q1 = cvt.bf16x2_to_fp32x2( + cute.recast_tensor(q_pe, Uint32)[token_id, head_id, lane_id] + ) + cos = q_pe_rope_cache[pos, lane_id].to(Float32) + sin = q_pe_rope_cache[pos, 32 + lane_id].to(Float32) + result = cvt.fp32x2_to_bf16x2(q0 * cos - q1 * sin, q1 * cos + q0 * sin) + cute.arch.griddepcontrol_launch_dependents() + cute.recast_tensor(mqa_output, Uint32)[token_id, head_id, lane_id] = result + return inv_scale = 1.0 / q_scale[0] cp_op = cute.nvgpu.CopyUniversalOp() @@ -479,12 +493,16 @@ class FusedQKernel: BFloat16, (num_tokens, num_heads, rope_dim), divisibility=16 ) rope_cache = _make_fake_tensor(rope_type, (max_pos, rope_dim), divisibility=8) - ql_nope = _make_fake_tensor( - BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16 + ql_nope = ( + _make_fake_tensor( + BFloat16, (num_tokens, num_heads, nope_dim), divisibility=16 + ) + if nope_dim + else None ) - q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) + q_scale = _make_fake_tensor(Float32, (1,), divisibility=4) if nope_dim else None mqa_output = _make_fake_tensor( - Float8E4M3FN, + Float8E4M3FN if nope_dim else BFloat16, (num_tokens, num_heads, nope_dim + rope_dim), divisibility=16, ) From e12291d7332db897885fdc0e6ed61b28c969d743 Mon Sep 17 00:00:00 2001 From: Ganesh R Date: Thu, 1 Oct 2026 12:48:25 +0530 Subject: [PATCH 187/248] [CPU][Zen] Add DA8W4 (W4A8) int4 support for dense and MoE layers (#54024) Signed-off-by: R Signed-off-by: Ganesh R Co-authored-by: Cursor Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .buildkite/hardware_tests/cpu.yaml | 4 + tests/kernels/quantization/test_zen_da8w4.py | 661 ++++++++++++++++++ vllm/envs.py | 3 +- .../linear/mixed_precision/zentorch.py | 121 +++- .../layers/fused_moe/experts/cpu_moe.py | 26 + .../layers/fused_moe/experts/zentorch_moe.py | 204 ++++++ .../layers/fused_moe/oracle/int_wna16.py | 107 ++- .../compressed_tensors_moe_wna16.py | 68 +- vllm/model_executor/models/gpt_oss.py | 5 +- 9 files changed, 1184 insertions(+), 15 deletions(-) create mode 100644 tests/kernels/quantization/test_zen_da8w4.py create mode 100644 vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index b50a082727..d54197f836 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -24,6 +24,9 @@ steps: - tests/kernels/test_onednn.py - tests/kernels/test_awq_int4_to_int8.py - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py + - tests/kernels/quantization/test_zen_da8w4.py + - vllm/model_executor/kernels/linear/mixed_precision/zentorch.py + - vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py - tests/kernels/mamba/test_cpu_short_conv.py - tests/kernels/mamba/test_causal_conv1d.py @@ -46,6 +49,7 @@ steps: tests/kernels/test_onednn.py \ tests/kernels/test_awq_int4_to_int8.py \ tests/kernels/quantization/test_cpu_fp8_scaled_mm.py \ + tests/kernels/quantization/test_zen_da8w4.py \ tests/kernels/mamba/cpu/test_cpu_gdn_ops.py \ tests/kernels/mamba/test_causal_conv1d.py \ tests/kernels/mamba/test_mamba_ssm.py \ diff --git a/tests/kernels/quantization/test_zen_da8w4.py b/tests/kernels/quantization/test_zen_da8w4.py new file mode 100644 index 0000000000..924b7a865f --- /dev/null +++ b/tests/kernels/quantization/test_zen_da8w4.py @@ -0,0 +1,661 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs. + +The zentorch ops are mocked with reference implementations when zentorch is not +installed, so the layout and dispatch contracts are covered in CI. +""" + +import dataclasses + +import pytest +import torch +from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32 + +from tests.kernels.quant_utils import ref_dynamic_per_token_quant +from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( + MPLinearLayerConfig, +) +from vllm.model_executor.kernels.linear.mixed_precision.zentorch import ( + ZentorchWNA16LinearKernel, + _import_unpack_from_int32, +) +from vllm.model_executor.layers.fused_moe.oracle import int_wna16 +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend +from vllm.platforms import current_platform +from vllm.scalar_type import scalar_types + +if not current_platform.is_cpu(): + pytest.skip("skipping CPU-only tests", allow_module_level=True) + +GROUP_SIZE = 128 +IN_FEATURES = 512 +OUT_FEATURES = 128 + + +def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor: + """int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K].""" + words = packed.view(torch.int32) + out = torch.zeros(words.shape[0], in_features, dtype=torch.float32) + for i in range(8): + nibble = (words >> (4 * i)) & 0xF + out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float() + return out + + +def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor: + """Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8].""" + n, k = unpacked.shape + values = unpacked.to(torch.int32).reshape(n, k // 8, 8) + out = torch.zeros(n, k // 8, dtype=torch.int32) + for i in range(8): + out |= (values[:, :, i] & 0xF) << (4 * i) + return out + + +def _quantize_per_group( + weight: torch.Tensor, group_size: int +) -> tuple[torch.Tensor, torch.Tensor]: + """Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G]).""" + out_features, in_features = weight.shape + grouped = weight.float().reshape( + out_features, in_features // group_size, group_size + ) + scale = grouped.abs().amax(dim=-1) / 8.0 + quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8) + return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16) + + +def _dequantize_per_group( + quantized: torch.Tensor, scale: torch.Tensor, group_size: int +) -> torch.Tensor: + """Dequantize with [N, G] scales into a float32 [N, K] weight.""" + return quantized.float() * scale.float().repeat_interleave(group_size, dim=1) + + +def _dynamic_quant_matmul( + x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None +) -> torch.Tensor: + """DA8W4 reference: per-token int8 activation quant against a bf16 weight.""" + x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8) + out = (x_q.float() @ weight.t()) * scale + if bias is not None: + out = out + bias.float() + return out + + +@pytest.fixture +def mock_zentorch_ops(): + """Register reference zentorch ops when zentorch is not installed.""" + if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"): + yield None + return + + calls: dict[str, tuple] = {} + + def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""): + calls["dynamic_qlinear"] = (input, weight, weight_scales, bias) + in_features = weight.view(torch.int32).shape[1] * 8 + group_size = in_features // weight_scales.shape[0] + # weight_scales is [G, N]; expand to a [N, K] dequantized weight. + scales = weight_scales.float().t().repeat_interleave(group_size, dim=1) + weight_deq = _unpack_s4(weight, in_features) * scales + out = _dynamic_quant_matmul(input, weight_deq, bias) + return out.to(input.dtype) + + def _fused_moe( + output, + input, + w13, + w2, + w13_bias, + w2_bias, + topk_weights, + topk_id, + skip_weighted, + act, + w13_scales=None, + w2_scales=None, + zentorch_op_name="", + ): + calls["fused_moe"] = ( + input, + w13, + w2, + topk_weights, + topk_id, + skip_weighted, + act, + w13_scales, + w2_scales, + ) + output.zero_() + + lib_def = torch.library.Library("zentorch", "DEF") + lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor") + lib_def.define( + "zentorch_dynamic_qlinear(Tensor input, Tensor weight, " + "Tensor weight_scales, Tensor? bias=None, *, " + "str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor" + ) + lib_def.define( + "zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, " + "Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, " + "Tensor topk_id, bool skip_weighted, str act, " + "Tensor? w13_scales=None, Tensor? w2_scales=None, *, " + "str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()" + ) + + lib_impl = torch.library.Library("zentorch", "IMPL", "CPU") + lib_impl.impl("zentorch_woq_repack_weight", _repack_s4) + lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear) + lib_impl.impl("zentorch_fused_moe", _fused_moe) + + yield calls + + lib_impl._destroy() + lib_def._destroy() + + +def _make_config( + act_type: torch.dtype = torch.bfloat16, + weight_type=scalar_types.uint4b8, + zero_points: bool = False, + group_size: int = GROUP_SIZE, +) -> MPLinearLayerConfig: + return MPLinearLayerConfig( + full_weight_shape=(IN_FEATURES, OUT_FEATURES), + partition_weight_shape=(IN_FEATURES, OUT_FEATURES), + weight_type=weight_type, + act_type=act_type, + group_size=group_size, + zero_points=zero_points, + ) + + +def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]: + """Build a layer holding a CT-packed int4 weight, plus its dequantized form.""" + torch.manual_seed(0) + weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16) + quantized, scale = _quantize_per_group(weight, group_size) + + # compressed-tensors stores [N, K//8] int32, packed along the input dim. + packed = pack_to_int32(quantized, 4, packed_dim=1) + + layer = torch.nn.Module() + layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False) + layer.weight_packed.input_dim = 1 + layer.weight_packed.packed_dim = 1 + layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False) + layer.weight_zero_point = None + return layer, _dequantize_per_group(quantized, scale, group_size) + + +def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel: + return ZentorchWNA16LinearKernel( + config, "weight_packed", "weight_scale", "weight_zero_point" + ) + + +# --------------------------------------------------------------------------- +# Dense DA8W4 +# --------------------------------------------------------------------------- + + +def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + assert kernel._maybe_process_da8w4_weights(layer) + + +@pytest.mark.parametrize( + "config_kwargs,reason", + [ + ({"act_type": torch.float32}, "f32 activations are rejected"), + ( + {"weight_type": scalar_types.uint4, "zero_points": True}, + "asymmetric int4 is unsupported", + ), + ], +) +def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config(**config_kwargs)) + assert not kernel._maybe_process_da8w4_weights(layer), reason + + +def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "envs.VLLM_CPU_INT4_W4A8", + False, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + assert not kernel._maybe_process_da8w4_weights(layer) + + +def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + kernel.process_weights_after_loading(layer) + + assert layer._zentorch_da8w4 + assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4" + # Packed s4 holds two nibbles per byte, and scales transpose to {G, N}. + assert layer._zentorch_da8w4_packed.dtype == torch.int8 + assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2) + assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16 + assert layer._zentorch_da8w4_scale.shape == ( + IN_FEATURES // GROUP_SIZE, + OUT_FEATURES, + ) + assert layer._zentorch_da8w4_packed.is_contiguous() + assert layer._zentorch_da8w4_scale.is_contiguous() + # The checkpoint parameters are released once repacked. + assert layer.weight_packed.numel() == 0 + assert layer.weight_scale.numel() == 0 + + +def test_da8w4_apply_weights_matches_dequantized_reference( + mock_zentorch_ops, monkeypatch +): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + layer, weight_deq = _make_layer() + kernel = _make_kernel(_make_config()) + kernel.process_weights_after_loading(layer) + + x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16) + bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16) + out = kernel.apply_weights(layer, x, bias) + + expected = _dynamic_quant_matmul(x, weight_deq, bias) + assert out.dtype == torch.bfloat16 + torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2) + + +def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch): + """Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path.""" + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op", + lambda ops: "zentorch_dynamic_qlinear" not in ops, + ) + layer, _ = _make_layer() + kernel = _make_kernel(_make_config()) + assert not kernel._maybe_process_da8w4_weights(layer) + + +def test_can_implement_requires_zen_cpu(monkeypatch): + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: False, + ) + ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config()) + assert not ok + assert reason is not None + + +def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch): + """CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks.""" + from vllm.model_executor.kernels.linear.mixed_precision.cpu import ( + CPUWNA16LinearKernel, + ) + + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.cpu." + "current_platform.is_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op", + lambda ops: True, + ) + + unaligned = dataclasses.replace( + _make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16) + ) + assert unaligned.partition_weight_shape[1] % 32 != 0 + + ok, _ = CPUWNA16LinearKernel.can_implement(unaligned) + assert not ok, "the parent is expected to turn this shape away" + + ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned) + assert ok, reason + + +@pytest.mark.parametrize( + "group_size,expected_ok", + [ + (-1, True), # per-channel, one group spanning K + (GROUP_SIZE, True), + (IN_FEATURES, True), + (6, False), # even, so the parent's multiple-of-2 rule would allow it + (0, False), + ], +) +def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok): + """ZenDNN WOQ groups along K, so a group that straddles the end is unusable.""" + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch." + "current_platform.is_zen_cpu", + lambda: True, + ) + monkeypatch.setattr( + "vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op", + lambda ops: True, + ) + + ok, reason = ZentorchWNA16LinearKernel.can_implement( + _make_config(group_size=group_size) + ) + assert ok is expected_ok, reason + if not expected_ok: + assert reason is not None + assert "must divide input size" in reason + + +# --------------------------------------------------------------------------- +# MoE DA8W4 +# --------------------------------------------------------------------------- + +NUM_EXPERTS = 4 +HIDDEN = 256 +INTERMEDIATE = 128 + + +def _make_moe_weights(group_size: int = 32): + """Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8].""" + torch.manual_seed(0) + w13_q, w13_s, w2_q, w2_s = [], [], [], [] + for _ in range(NUM_EXPERTS): + w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16) + q13, s13 = _quantize_per_group(w13, group_size) + # CT is N-first and packs along the input dim: [N, K//8] per expert. + w13_q.append(pack_to_int32(q13, 4, packed_dim=1)) + w13_s.append(s13) + + w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16) + q2, s2 = _quantize_per_group(w2, group_size) + w2_q.append(pack_to_int32(q2, 4, packed_dim=1)) + w2_s.append(s2) + return ( + torch.stack(w13_q), + torch.stack(w2_q), + torch.stack(w13_s), + torch.stack(w2_s), + ) + + +def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch): + monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True) + backends = int_wna16._get_priority_backends() + assert backends[0] == WNA16MoEBackend.ZEN_CPU + # The generic CPU backend stays available as a fallback. + assert WNA16MoEBackend.CPU in backends + + +def test_zen_cpu_process_weights_layout(mock_zentorch_ops): + group_size = 32 + w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size) + assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8) + assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8) + + converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale) + w13_out, w2_out, w13_s_out, w2_s_out = converted[:4] + + # zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales. + assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8) + assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8) + assert w13_out.dtype == w2_out.dtype == torch.int32 + assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE) + assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN) + assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16 + # Symmetric checkpoints carry no zero points into the kernel. + assert converted[8] is None and converted[9] is None + + +def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops): + """The repacked weight must dequantize back to the checkpoint values.""" + group_size = 32 + w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size) + w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0] + + expected = _import_unpack_from_int32()( + w13, + 4, + torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]), + packed_dim=1, + ) + for expert in range(NUM_EXPERTS): + torch.testing.assert_close( + _unpack_s4(w13_out[expert], HIDDEN), + expected[expert].float(), + ) + + +@pytest.mark.parametrize( + "group_size,expected_ok", + [(32, True), (128, True), (-1, False), (2, False)], +) +def test_zen_cpu_group_size_gating(group_size, expected_ok): + quant_config = type("Args", (), {"group_size": group_size})() + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=False, + may_have_bias=False, + allow_tile_padding=True, + ) + assert (reason is None) == expected_ok + + +@pytest.mark.parametrize( + "may_have_zp,may_have_bias,expected_ok", + [(True, False, False), (False, True, True), (True, True, False)], +) +def test_zen_cpu_rejects_zero_points_but_takes_bias( + may_have_zp, may_have_bias, expected_ok +): + quant_config = type("Args", (), {"group_size": 128})() + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=may_have_zp, + may_have_bias=may_have_bias, + allow_tile_padding=True, + ) + assert (reason is None) == expected_ok + + +def test_zen_cpu_rejects_moe_wna16_layout(): + from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config + + quant_config = MoeWNA16Config( + linear_quant_method="gptq", + weight_bits=4, + group_size=128, + has_zp=False, + lm_head_quantized=False, + modules_to_not_convert=None, + full_config={}, + ) + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=False, + may_have_bias=False, + allow_tile_padding=True, + ) + assert reason is not None + + +def test_zen_cpu_disabled_by_env(monkeypatch): + monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False) + quant_config = type("Args", (), {"group_size": 128})() + reason = int_wna16._backend_incompatibility_reason( + WNA16MoEBackend.ZEN_CPU, + moe_config=None, + quant_config=quant_config, + may_have_zp=False, + may_have_bias=False, + allow_tile_padding=True, + ) + assert reason is not None + + +def test_zen_cpu_backend_maps_to_experts_class(): + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [ + ZentorchExpertsInt4 + ] + + +def test_zen_experts_support_predicates(): + from vllm.model_executor.kernels.linear.zentorch_utils import ( + _ZENTORCH_MOE_ACTIVATIONS, + ) + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt4Static, + kInt4Static32, + kInt8StaticChannelSym, + ) + + assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None) + assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None) + assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None) + for act in MoEActivation: + expected = act.value in _ZENTORCH_MOE_ACTIVATIONS + assert ZentorchExpertsInt4._supports_activation(act) == expected + assert ZentorchExpertsInt4.requires_interleaved_w13 + assert not ZentorchExpertsInt4._supports_no_act_and_mul() + assert ZentorchExpertsInt4._supports_parallel_config( + type("Par", (), {"use_ep": False})() + ) + assert not ZentorchExpertsInt4._supports_parallel_config( + type("Par", (), {"use_ep": True})() + ) + + +def test_zen_experts_take_custom_routing(): + """Models with their own router (gemma-4) reach select_experts through the + config captured off the layer, since apply() cannot be handed the callable.""" + from types import SimpleNamespace + + from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.config import RoutingMethodType + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + assert ZentorchExpertsInt4._supports_routing_method( + RoutingMethodType.Custom, None, None + ) + + def routing_fn(**kwargs): + raise AssertionError("not called") + + layer = _StubMoELayer( + torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN), + torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE), + MoEActivation.SILU, + ) + layer.renormalize = True + layer.custom_routing_function = routing_fn + + experts = SimpleNamespace(renormalize=False, custom_routing_function=None) + ZentorchExpertsInt4.process_weights_after_loading(experts, layer) + + assert experts.custom_routing_function is routing_fn + assert experts.renormalize is True + + +def _swigluoai_perm(activation, experts_cls=None): + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + swigluoai_w13_interleave_perm, + ) + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + return swigluoai_w13_interleave_perm( + experts_cls or ZentorchExpertsInt4, + activation, + 2 * INTERMEDIATE, + torch.device("cpu"), + ) + + +def test_swigluoai_perm_only_for_zen_swigluoai(): + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + + assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None + # Other activations keep the half-split layout the loader leaves. + assert _swigluoai_perm(MoEActivation.SILU) is None + # object stands in for any kernel that never set requires_interleaved_w13. + assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None + + +def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops): + """The permutation is a gather on w13's output-channel axis, which is dim 1 + for the N-first packed weights and their group scales.""" + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + + group_size = 32 + w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size) + bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16) + perm = _swigluoai_perm(MoEActivation.SWIGLUOAI) + + baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0] + permuted = int_wna16._process_weights_zen_cpu( + w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale + )[0] + + for expert in range(NUM_EXPERTS): + before = _unpack_s4(baseline[expert], HIDDEN) + after = _unpack_s4(permuted[expert], HIDDEN) + torch.testing.assert_close(after[0::2], before[:INTERMEDIATE]) + torch.testing.assert_close(after[1::2], before[INTERMEDIATE:]) + + torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE]) + torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:]) diff --git a/vllm/envs.py b/vllm/envs.py index 5daa4fd3e6..e63ed5cc02 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -912,7 +912,8 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ZENTORCH_WEIGHT_PREPACK": lambda: bool( int(os.getenv("VLLM_ZENTORCH_WEIGHT_PREPACK", "1")) ), - # (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ. + # (CPU backend only) whether to use SGLang INT4 W4A8 kernels for AWQ, and + # on Zen CPUs whether to serve int4 checkpoints as DA8W4 rather than W4A16. "VLLM_CPU_INT4_W4A8": lambda: bool(int(os.getenv("VLLM_CPU_INT4_W4A8", "1"))), # If the env var is set, Ray Compiled Graph uses the specified # channel type to communicate between workers belonging to diff --git a/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py b/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py index 90968ab2c2..c62a705076 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/zentorch.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Zentorch W4A16 GPTQ weight-only-quantized linear kernel for AMD Zen CPUs. +"""Zentorch int4 weight-quantized linear kernels for AMD Zen CPUs. Selected by ``choose_mp_linear_kernel`` ahead of the generic oneDNN-backed ``CPUWNA16LinearKernel``. When ``can_implement`` rejects a layer, the selector @@ -9,12 +9,13 @@ falls through to the next kernel in ``_POSSIBLE_KERNELS[PlatformEnum.CPU]``. import torch +from vllm import envs from vllm.logger import init_logger from vllm.model_executor.kernels.linear.zentorch_utils import has_zentorch_op from vllm.platforms import current_platform from vllm.scalar_type import scalar_types -from .cpu import CPUWNA16LinearKernel +from .cpu import _CPUWNA16_SUPPORTED_QUANT_TYPES, CPUWNA16LinearKernel from .MPLinearKernel import MPLinearLayerConfig logger = init_logger(__name__) @@ -34,17 +35,35 @@ def _import_unpack_from_int32(): class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): - """W4A16 GPTQ kernel backed by ``torch.ops.zentorch.zentorch_woq_linear``.""" + """Int4 kernel backed by ``zentorch_dynamic_qlinear`` (DA8W4) or + ``zentorch_woq_linear`` (W4A16).""" @classmethod def can_implement(cls, c: MPLinearLayerConfig) -> tuple[bool, str | None]: - ok, reason = super().can_implement(c) - if not ok: - return ok, reason - if not current_platform.is_zen_cpu(): return False, "ZentorchWNA16 requires an AMD Zen CPU." + # Not super(): its N/K multiple-of-32 rule is a oneDNN packing + # constraint the zentorch repack does not share. + if c.weight_type not in _CPUWNA16_SUPPORTED_QUANT_TYPES: + return ( + False, + f"Quant type ({c.weight_type}) not supported by " + "CPUWNA16, supported types are: " + f"{_CPUWNA16_SUPPORTED_QUANT_TYPES}", + ) + + # -1 is per-channel, one group spanning K. Anything else has to divide + # K, which is stricter than the parent's multiple-of-2 rule. + if c.group_size != -1: + in_features = c.partition_weight_shape[0] + if c.group_size <= 0 or in_features % c.group_size != 0: + return ( + False, + f"Group size ({c.group_size}) must divide input size " + f"({in_features})", + ) + if not has_zentorch_op(["zentorch_woq_repack_weight", "zentorch_woq_linear"]): return ( False, @@ -89,8 +108,81 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): num_groups = weight_scale.shape[1] return num_groups > 0 and in_features % num_groups == 0 + def _maybe_process_da8w4_weights(self, layer: torch.nn.Module) -> bool: + """Repack ``layer`` into the DA8W4 layout, or leave it untouched. + + Returns False without mutating the layer when it is not eligible, so + the caller falls through to the W4A16 paths. + """ + if not envs.VLLM_CPU_INT4_W4A8: + return False + + if not has_zentorch_op(["zentorch_dynamic_qlinear"]): + return False + + # DA8W4 is symmetric-only, and the kernel rejects f32 activations. + if self.config.zero_points or self.config.weight_type == scalar_types.uint4: + return False + if self.config.act_type != torch.bfloat16: + return False + + if not self._zentorch_woq_eligible(layer): + return False + + weight_q = getattr(layer, self.w_q_name) + weight_s = getattr(layer, self.w_s_name) + weight_packed = weight_q.data if hasattr(weight_q, "data") else weight_q + weight_scale = weight_s.data if hasattr(weight_s, "data") else weight_s + + bits = self.config.weight_type.mantissa + pack_factor = torch.iinfo(weight_packed.dtype).bits // bits + out_features, num_groups = weight_scale.shape[0], weight_scale.shape[1] + in_features = weight_packed.shape[1] * pack_factor + + # AOCL sym_quant requires K/G to be a multiple of 4; K must be even to + # pack two s4 values per byte. + if (in_features // num_groups) % 4 != 0 or in_features % 2 != 0: + return False + + if self.w_zp_name is not None: + setattr(layer, self.w_zp_name, None) + + unpack_from_int32 = _import_unpack_from_int32() + + weight_unpacked = unpack_from_int32( + weight_packed, + bits, + torch.Size([out_features, in_features]), + packed_dim=weight_q.packed_dim, + ) + # The WOQ repack packs 8 int4 per int32, which on little-endian is the + # same byte stream as s4 [N, K/2]; the kernel takes either view. + layer._zentorch_da8w4_packed = ( + torch.ops.zentorch.zentorch_woq_repack_weight.default( + weight_unpacked.to(torch.int8).contiguous() + ).view(torch.int8) + ) + # CT stores scales as [N, G]; DA8W4 wants per-group {G, N}. + layer._zentorch_da8w4_scale = weight_scale.t().to(torch.bfloat16).contiguous() + + for param_name in (self.w_q_name, self.w_s_name): + param = getattr(layer, param_name, None) + if param is not None and hasattr(param, "data"): + param.data = torch.empty(0) + + layer._zentorch_kind = "compressed_tensors_w4a8_da8w4" + layer._zentorch_da8w4 = True + layer._zentorch_processed_weights = True + logger.info_once( + "[zen_cpu] Using zentorch_dynamic_qlinear for DA8W4 (W4A8) " + "(weight_type=%s, group_size=%d)", + self.config.weight_type, + in_features // num_groups, + ) + return True + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - """Repack CT GPTQ weights into the zentorch WOQ layout. + """Repack CT GPTQ weights into the zentorch DA8W4 or WOQ layout. Falls back to ``CPUWNA16LinearKernel.process_weights_after_loading`` via ``super()`` when the layer doesn't satisfy @@ -101,6 +193,9 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): if getattr(layer, "_zentorch_processed_weights", False): return + if self._maybe_process_da8w4_weights(layer): + return + if not self._zentorch_woq_eligible(layer): logger.info_once( "[zen_cpu] ZentorchWNA16 fast path not eligible for this " @@ -186,6 +281,16 @@ class ZentorchWNA16LinearKernel(CPUWNA16LinearKernel): x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: + if getattr(layer, "_zentorch_da8w4", False): + # The kernel reads bias through a raw pointer, so it must be + # contiguous; the weight and scales already are. + return torch.ops.zentorch.zentorch_dynamic_qlinear.default( + x if x.dtype == torch.bfloat16 else x.to(torch.bfloat16), + layer._zentorch_da8w4_packed, + layer._zentorch_da8w4_scale, + bias.contiguous() if bias is not None else None, + ) + if getattr(layer, "_zentorch_processed_weights", False): return torch.ops.zentorch.zentorch_woq_linear.default( x, diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 3e7376fe90..9b66a1f853 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -51,6 +51,32 @@ from vllm.platforms import CpuArchEnum, current_platform from vllm.utils.math_utils import round_up logger = init_logger(__name__) +# =========================================================================== +# Weight layout +# =========================================================================== + + +def swigluoai_w13_interleave_perm( + experts_cls: type[mk.FusedMoEExperts], + activation: MoEActivation, + two_i: int, + device: torch.device, +) -> torch.Tensor | None: + """Index taking a half-split w13 ``[gate | up]`` to the interleaved order + ZenDNN's ``swiglu_oai_mul`` expects, or None when no reorder is needed. + Callers apply it along whichever axis holds their output channels.""" + if not getattr(experts_cls, "requires_interleaved_w13", False): + return None + if activation != MoEActivation.SWIGLUOAI: + return None + + i = two_i // 2 + return torch.stack( + [torch.arange(0, i, device=device), torch.arange(i, two_i, device=device)], + dim=1, + ).flatten() + + # =========================================================================== # Unquantized (BF16/FP16/FP32) MoE # =========================================================================== diff --git a/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py b/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py new file mode 100644 index 0000000000..0076b65966 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Zentorch fused MoE experts for AMD Zen CPUs.""" + +from collections.abc import Callable + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.model_executor.kernels.linear.zentorch_utils import ( + _ZENTORCH_MOE_ACTIVATIONS, + _moe_activation_to_str, + has_zentorch_op, +) +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, + RoutingMethodType, +) +from vllm.model_executor.layers.fused_moe.router.cpu_router import select_experts +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4Static, + kInt4Static32, +) + + +class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic): + """DA8W4 (W4A8) group-quantized monolithic MoE experts.""" + + # swiglu_oai_mul reads gate/up interleaved, not in the half-split order the + # weight loader leaves behind. + requires_interleaved_w13 = True + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + # Defaults hold until process_weights_after_loading sees the layer. + self.renormalize = moe_config.routing_method in ( + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ) + self.custom_routing_function: Callable | None = None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Capture the router config that the monolithic apply() signature + cannot carry; Custom routing has no other source for it.""" + self.renormalize = layer.renormalize + self.custom_routing_function = layer.custom_routing_function + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + return has_zentorch_op(["zentorch_fused_moe", "zentorch_woq_repack_weight"]) + + @staticmethod + def is_supported_config( + cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[bool, str | None]: + supported, reason = mk.FusedMoEExperts.is_supported_config( + cls, moe_config, weight_key, activation_key, activation_format + ) + if not supported: + return supported, reason + if moe_config.in_dtype != torch.bfloat16: + return False, "kernel requires bfloat16 activations" + # The grouped GEMM needs tokens spread over at least two experts, which + # top_k=1 cannot satisfy once decode drops to a single token. + if moe_config.experts_per_token < 2: + return False, "kernel requires experts_per_token >= 2" + return True, None + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return _moe_activation_to_str(activation) in _ZENTORCH_MOE_ACTIVATIONS + + @staticmethod + def _supports_parallel_config( + moe_parallel_config: FusedMoEParallelConfig, + ) -> bool: + # apply() passes select_experts' global ids straight to the kernel, so + # every expert has to be resident on the rank. + return not moe_parallel_config.use_ep + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) in [ + (kInt4Static, None), + (kInt4Static32, None), + ] + + @staticmethod + def _supports_routing_method( + routing_method: RoutingMethodType, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return routing_method in [ + RoutingMethodType.Default, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + RoutingMethodType.Custom, + ] + + @staticmethod + def _supports_router_logits_dtype( + router_logits_dtype: torch.dtype | None, + routing_method: RoutingMethodType, + ) -> bool: + return True + + def supports_expert_map(self) -> bool: + return False + + def apply( + self, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + apply_router_weight_on_input: bool, + # grouped topk + fused topk bias parameters + num_expert_group: int | None = None, + e_score_correction_bias: torch.Tensor | None = None, + routed_scaling_factor: float | None = None, + topk_group: int | None = None, + ) -> torch.Tensor: + if apply_router_weight_on_input: + raise NotImplementedError( + "ZentorchExpertsInt4 does not support " + "apply_router_weight_on_input=True." + ) + + if ( + self.moe_config.routing_method == RoutingMethodType.Custom + and self.custom_routing_function is None + ): + raise RuntimeError( + "ZentorchExpertsInt4 needs the model's custom_routing_function " + "for RoutingMethodType.Custom." + ) + + topk_weights, topk_ids = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + use_grouped_topk=num_expert_group is not None, + top_k=self.moe_config.experts_per_token, + renormalize=self.renormalize, + topk_group=topk_group, + num_expert_group=num_expert_group, + custom_routing_function=self.custom_routing_function, + scoring_func="softmax", + routed_scaling_factor=( + routed_scaling_factor if routed_scaling_factor is not None else 1.0 + ), + e_score_correction_bias=e_score_correction_bias, + ) + + output = torch.empty_like(hidden_states) + torch.ops.zentorch.zentorch_fused_moe( + output, + hidden_states, + w1, + w2, + self.w1_bias, + self.w2_bias, + topk_weights, + topk_ids, + False, # skip_weighted + _moe_activation_to_str(activation), + self.w1_scale, + self.w2_scale, + ) + return output + + +__all__ = ["ZentorchExpertsInt4"] diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py index 791dc56766..239fe1d55a 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -12,6 +12,7 @@ from compressed_tensors.quantization import ( import vllm._custom_ops as ops import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm import envs from vllm.config.kernel import MoEBackend from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.config import ( @@ -55,6 +56,7 @@ class WNA16MoEBackend(Enum): BATCHED_MARLIN = "BATCHED_MARLIN" HUMMING = "HUMMING" CPU = "CPU" + ZEN_CPU = "ZEN_CPU" FLASHINFER_TRTLLM = "FLASHINFER_TRTLLM" TRITON = "TRITON" XPU = "XPU" @@ -98,6 +100,12 @@ def backend_to_kernel_cls( ) return [CPUExpertsInt4] + elif backend == WNA16MoEBackend.ZEN_CPU: + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) + + return [ZentorchExpertsInt4] elif backend == WNA16MoEBackend.EMULATION: from vllm.model_executor.layers.fused_moe.experts.int4_emulation_moe import ( Int4EmulationTritonExperts, @@ -117,7 +125,7 @@ def backend_to_kernel_cls( def _get_priority_backends() -> list[WNA16MoEBackend]: """Get available backends in priority order based on platform and config.""" if current_platform.is_cpu(): - return [WNA16MoEBackend.CPU] + return [WNA16MoEBackend.ZEN_CPU, WNA16MoEBackend.CPU] if current_platform.is_xpu(): return [WNA16MoEBackend.XPU] @@ -160,6 +168,27 @@ def _backend_incompatibility_reason( from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config + if backend == WNA16MoEBackend.ZEN_CPU: + if not envs.VLLM_CPU_INT4_W4A8: + return "VLLM_CPU_INT4_W4A8=0 disables the DA8W4 path" + if may_have_zp: + return "zero points are not supported" + if isinstance(quant_config, MoeWNA16Config): + return "the MoeWNA16 weight layout is not supported" + if isinstance(quant_config, AutoGPTQConfig) and quant_config.desc_act: + return "GPTQ activation ordering is not supported" + if ( + isinstance(quant_config, QuantizationArgs) + and quant_config.actorder == "group" + ): + return "group activation ordering is not supported" + group_size = getattr(quant_config, "group_size", None) + if group_size is None or group_size <= 0: + return "DA8W4 requires group-quantized weights" + # AOCL sym_quant requires the group size to be a multiple of 4. + if group_size % 4 != 0: + return f"group size {group_size} is not a multiple of 4" + if backend == WNA16MoEBackend.TRITON: if may_have_bias: return "expert bias is not supported" @@ -403,10 +432,14 @@ def make_wna16_moe_kernel( from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( XPUExpertsWNA16, ) + from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import ( + ZentorchExpertsInt4, + ) # Currently, we only support TrtLlmMxint4ExpertsMonolithic, MarlinExperts, - # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, the Humming - # grouped/indexed experts, and Int4EmulationTritonExperts + # BatchedMarlinExperts, XPUExpertsWNA16, CPUExpertsInt4, + # ZentorchExpertsInt4, the Humming grouped/indexed experts, and + # Int4EmulationTritonExperts allowed_experts: tuple[type[mk.FusedMoEExperts], ...] = ( MarlinExperts, BatchedMarlinExperts, @@ -414,6 +447,7 @@ def make_wna16_moe_kernel( TrtLlmMxint4ExpertsMonolithic, XPUExpertsWNA16, CPUExpertsInt4, + ZentorchExpertsInt4, Int4EmulationTritonExperts, Rdna3WNA16Experts, ) @@ -1034,6 +1068,64 @@ def _process_weights_cpu( ) +def _zen_repack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor: + """Repack CT int4 ``[E, N, K//8]`` into zentorch s4 ``[E, N, K//8]``.""" + from vllm.model_executor.kernels.linear.mixed_precision.zentorch import ( + _import_unpack_from_int32, + ) + + num_experts, out_features, _ = packed.shape + # packed_dim indexes the 2D expert slice, which is N-first, so K is dim 1. + unpacked = _import_unpack_from_int32()( + packed, + 4, + torch.Size([num_experts, out_features, in_features]), + packed_dim=1, + ) + # CT is already N-first, which is what the repack consumes. + repack_op = torch.ops.zentorch.zentorch_woq_repack_weight.default + return torch.stack([repack_op(w) for w in unpacked]) + + +def _process_weights_zen_cpu( + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + w13_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, +) -> tuple[ + torch.Tensor, # w13_qweight + torch.Tensor, # w2_qweight + torch.Tensor, # w13_scales + torch.Tensor, # w2_scales + torch.Tensor | None, # w13_qzeros + torch.Tensor | None, # w2_qzeros + torch.Tensor | None, # w13_input_global_scale + torch.Tensor | None, # w2_input_global_scale + torch.Tensor | None, # w13_bias + torch.Tensor | None, # w2_bias +]: + """Zen CPU INT4 DA8W4 (W4A8) weight post-processing.""" + # CT WNA16 buffers are N-first, so the scales are [E, N, G]. + hidden_size = w2_scale.shape[1] + intermediate_size = w13_scale.shape[1] // 2 + + # The kernel reads per-group scales as [E, G, N], so N-first needs a swap. + return ( + _zen_repack_s4(w13.data, hidden_size), + _zen_repack_s4(w2.data, intermediate_size), + w13_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(), + w2_scale.data.transpose(1, 2).to(torch.bfloat16).contiguous(), + None, # w13_qzeros (symmetric) + None, # w2_qzeros (symmetric) + None, # w13_input_global_scale + None, # w2_input_global_scale + w13_bias, + w2_bias, + ) + + def _process_weights_xpu( layer: torch.nn.Module, quant_config: QuantizationConfig, @@ -1584,6 +1676,15 @@ def convert_to_wna16_moe_kernel_format( w13_bias, w2_bias, ) + elif backend == WNA16MoEBackend.ZEN_CPU: + return _process_weights_zen_cpu( + w13, + w2, + w13_scale, + w2_scale, + w13_bias, + w2_bias, + ) elif backend == WNA16MoEBackend.FLASHINFER_TRTLLM: return _process_weights_flashinfer( w13, diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py index c03d2bfcf4..8bbaaecd64 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py @@ -210,6 +210,27 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): layer.register_parameter("w2_weight_packed", w2_weight) set_weight_attrs(w2_weight, extra_weight_attrs) + # Per-expert biases, as gpt-oss carries. + if self.moe.has_bias: + w13_num_shards = 2 if self.moe.is_act_and_mul else 1 + w13_bias = torch.nn.Parameter( + torch.zeros( + num_experts, + w13_num_shards * intermediate_size_per_partition, + dtype=params_dtype, + ), + requires_grad=False, + ) + layer.register_parameter("w13_bias", w13_bias) + set_weight_attrs(w13_bias, extra_weight_attrs) + + w2_bias = torch.nn.Parameter( + torch.zeros(num_experts, hidden_size, dtype=params_dtype), + requires_grad=False, + ) + layer.register_parameter("w2_bias", w2_bias) + set_weight_attrs(w2_bias, extra_weight_attrs) + if self.strategy == "channel": num_groups_w2 = num_groups_w13 = 1 self.group_size = -1 @@ -350,7 +371,41 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): routing_tables=layer._expert_routing_tables(), ) + if self.wna16_backend == WNA16MoEBackend.ZEN_CPU: + # Monolithic experts route inside apply(), so like the unquantized + # CPU path they need the router config that only the layer carries. + self.moe_kernel.fused_experts.process_weights_after_loading(layer) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + swigluoai_w13_interleave_perm, + ) + + # Reorder before the conversion below, which repacks w13 for the kernel. + # The CT WNA16 buffers are N-first, so 2I is dim 1 of the packed weight + # [E, 2I, K/pack] and of its group scales [E, 2I, G], as for the bias. + perm = swigluoai_w13_interleave_perm( + self.experts_cls, + self.moe.activation, + layer.w13_weight_packed.size(1), + layer.w13_weight_packed.device, + ) + if perm is not None: + replace_parameter( + layer, + "w13_weight_packed", + layer.w13_weight_packed.data[:, perm].contiguous(), + ) + replace_parameter( + layer, + "w13_weight_scale", + layer.w13_weight_scale.data[:, perm].contiguous(), + ) + if self.moe.has_bias: + replace_parameter( + layer, "w13_bias", layer.w13_bias.data[:, perm].contiguous() + ) + # Process weights using the shared oracle infrastructure converted = convert_to_wna16_moe_kernel_format( backend=self.wna16_backend, @@ -363,6 +418,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): w2_scale=layer.w2_weight_scale, w13_qzeros=getattr(layer, "w13_weight_zero_point", None), w2_qzeros=getattr(layer, "w2_weight_zero_point", None), + w13_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), ) if converted is None: @@ -378,8 +435,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): w2_qzeros, w13_input_global_scale, w2_input_global_scale, - _, # w13_bias - _, # w2_bias + w13_bias, + w2_bias, ) = converted # Replace common parameters @@ -388,6 +445,11 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): replace_parameter(layer, "w13_weight_scale", w13_scales) replace_parameter(layer, "w2_weight_scale", w2_scales) + if w13_bias is not None: + replace_parameter(layer, "w13_bias", w13_bias) + if w2_bias is not None: + replace_parameter(layer, "w2_bias", w2_bias) + # CPU fused_experts_cpu requires zero points even for symmetric quant. # EMULATION bakes ZP into the dequantized bf16 weights — ZP is None. if ( @@ -437,6 +499,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): num_bits=self.num_bits, w1_zp=getattr(layer, "w13_weight_zero_point", None), w2_zp=getattr(layer, "w2_weight_zero_point", None), + w1_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), gemm1_clamp_limit=getattr(layer, "swiglu_limit", None), gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index 7c10aaa458..90286006a6 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -76,6 +76,8 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = { "w2_bias": "gpt_oss_w2", "w13_weight_scale": "gpt_oss_w13", "w2_weight_scale": "gpt_oss_w2", + "w13_weight_packed": "gpt_oss_w13", + "w2_weight_packed": "gpt_oss_w2", } @@ -263,7 +265,7 @@ class GptOssRoutedExperts(RoutedExperts): ) -> None: tp_rank = self.moe_config.moe_parallel_config.tp_rank is_w13 = shard_id == "gpt_oss_w13" - is_weight = weight_name.endswith("_weight") + is_weight = weight_name.endswith(("_weight", "_weight_packed")) is_partitioned_scale = weight_name.endswith("_weight_scale") if is_weight: @@ -347,6 +349,7 @@ class GptOssRoutedExperts(RoutedExperts): ) or quant_method_name in ( "CompressedTensorsW4A4Nvfp4MoEMethod", "Nvfp4OnlineMoEMethod", + "CompressedTensorsWNA16MoEMethod", ): self._load_packed_expert(expert_data, loaded_weight, weight_name, shard_id) else: From e12b7351099e23414de79cededdd3854633e986f Mon Sep 17 00:00:00 2001 From: jiangyunfan1 Date: Thu, 1 Oct 2026 15:38:29 +0800 Subject: [PATCH 188/248] [Test] Make Anthropic messages test compatible with SDK 1.x via extra_body (#57780) Signed-off-by: jiangyunfan1 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/entrypoints/anthropic/test_messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/entrypoints/anthropic/test_messages.py b/tests/entrypoints/anthropic/test_messages.py index 9f5cc2c451..9d7fa92104 100644 --- a/tests/entrypoints/anthropic/test_messages.py +++ b/tests/entrypoints/anthropic/test_messages.py @@ -224,7 +224,7 @@ async def test_anthropic_streaming_cache_usage(client: anthropic.AsyncAnthropic) request = dict( model="claude-3-7-sonnet-latest", max_tokens=1, - temperature=0.0, + extra_body={"temperature": 0.0}, messages=[ { "role": "user", From 5addde2a0afa1e27b6e0c478ba9b6995724f006d Mon Sep 17 00:00:00 2001 From: Ganesh R Date: Thu, 1 Oct 2026 13:21:53 +0530 Subject: [PATCH 189/248] [CPU][Zen] Pass f32 weight scales to the zentorch INT8 MoE (#59434) Signed-off-by: Ganesh R Co-authored-by: Cursor Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> --- tests/kernels/moe/test_zen_cpu_int8_moe.py | 5 +++-- .../layers/fused_moe/experts/cpu_moe.py | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/kernels/moe/test_zen_cpu_int8_moe.py b/tests/kernels/moe/test_zen_cpu_int8_moe.py index cdcfe3b165..83e051a1e3 100644 --- a/tests/kernels/moe/test_zen_cpu_int8_moe.py +++ b/tests/kernels/moe/test_zen_cpu_int8_moe.py @@ -224,8 +224,9 @@ def test_zen_int8_dispatch_contract( call = mock_zentorch_fused_moe["fused_moe"] assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE) assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE) - assert call["w13_scales"].dtype == torch.bfloat16 - assert call["w2_scales"].dtype == torch.bfloat16 + # f32 scales keep the layer on zentorch's fast routed MoE executor. + assert call["w13_scales"].dtype == torch.float32 + assert call["w2_scales"].dtype == torch.float32 assert call["act"] == act.value.lower() assert call["skip_weighted"] is False assert call["topk_weights"].dtype == torch.float32 diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py index 9b66a1f853..14b25ce30e 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_moe.py @@ -1502,15 +1502,17 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): "ZenCPUExpertsInt8 requires per-channel weight scales on the layer." ) num_experts = self.w1_scale.shape[0] - self._w1_scale_bf16 = ( + # ZenDNN's fast routed MoE kernel only accepts f32 expert scales; bf16 + # silently drops the layer onto the slower per-expert path. + self._w1_scale_f32 = ( self.w1_scale.detach() - .to(torch.bfloat16) + .to(torch.float32) .reshape(num_experts, -1) .contiguous() ) - self._w2_scale_bf16 = ( + self._w2_scale_f32 = ( self.w2_scale.detach() - .to(torch.bfloat16) + .to(torch.float32) .reshape(num_experts, -1) .contiguous() ) @@ -1637,8 +1639,8 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular): topk_ids.to(torch.int32).contiguous(), apply_router_weight_on_input, # skip_weighted str(activation.value).lower(), - self._w1_scale_bf16, - self._w2_scale_bf16, + self._w1_scale_f32, + self._w2_scale_f32, ) From e5e38ba9b7d18f9746d989e389a51a94b0f96f6f Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 1 Oct 2026 03:52:26 -0400 Subject: [PATCH 190/248] [Model Runner V2] Support randomized dummy inputs (#58411) Signed-off-by: Robert Shaw Co-authored-by: Robert Shaw Co-authored-by: Claude Opus 5.5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill --- .../distributed/elastic_ep/elastic_execute.py | 7 +++++- vllm/v1/worker/gpu/model_runner.py | 13 ++++++++-- vllm/v1/worker/gpu_model_runner.py | 7 ++++-- vllm/v1/worker/gpu_worker.py | 25 ++++++++++++++++--- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index e66bafd817..10b5e88d29 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -721,7 +721,12 @@ class ElasticEPScalingExecutor: self._suppress_eplb(), serving_state, ): - runner._dummy_run(runner.max_num_tokens, is_profile=True, skip_eplb=True) + runner._dummy_run( + runner.max_num_tokens, + is_profile=True, + skip_eplb=True, + randomize_inputs=self.worker.randomize_dummy_inputs, + ) self.worker.compile_or_warm_up_model() lock_workspace() diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 14ff11f448..b40fb4eac1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -805,6 +805,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): skip_eplb: bool = False, is_profile: bool = False, valid_dummy_state_slots: bool = False, + randomize_inputs: bool = False, **kwargs, ) -> tuple[torch.Tensor | None, torch.Tensor | None]: if skip_attn and not is_profile: @@ -878,6 +879,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): is_profile=is_profile, context_len=context_len, valid_dummy_state_slots=valid_dummy_state_slots, + randomize_inputs=randomize_inputs, ) self.kv_connector.set_disabled(False) @@ -968,7 +970,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.pooling_runner.dummy_pooler_run(hidden_states) @torch.inference_mode() - def profile_run(self) -> None: + def profile_run(self, randomize_inputs: bool = False) -> None: if self.supports_mm_inputs and self.is_first_pp_rank: mm_config = self.model_config.multimodal_config if mm_config is not None and not mm_config.skip_mm_profiling: @@ -981,7 +983,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) hidden_states, sample_hidden_states = self._dummy_run( - self.max_num_tokens, skip_attn=True, is_profile=True + self.max_num_tokens, + skip_attn=True, + is_profile=True, + randomize_inputs=randomize_inputs, ) # Only run sampler/pooler on last PP rank (non-last ranks return None). @@ -1696,6 +1701,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): is_profile: bool = False, context_len: int = 0, valid_dummy_state_slots: bool = False, + randomize_inputs: bool = False, ) -> ModelRunnerOutput | IntermediateTensors | None: if not dummy_run: # Update the request states. @@ -1814,6 +1820,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): # so MoE memory is measured and MoE kernels are exercised. is_padding=not is_profile, ) + if randomize_inputs: + # All-zero input_ids route every token to the same experts. + input_batch.input_ids.random_(0, self.vocab_size) if self.pcp_manager is not None: input_batch = self.pcp_manager.prepare_inputs_to_capture(input_batch) if skip_attn_for_dummy_run: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 3dc0524bab..4ccdec2f21 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -5735,6 +5735,9 @@ class GPUModelRunner( - during profile_run - during DP rank dummy run """ + # The worker also resolves this flag and passes randomize_inputs, but + # V1 has internal dummy runs (e.g. CUDA graph capture) that don't go + # through the worker, so keep the check here too. dp_size = self.vllm_config.parallel_config.data_parallel_size randomize_inputs = randomize_inputs or ( envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS and dp_size > 1 @@ -6420,7 +6423,7 @@ class GPUModelRunner( max_task = max(output_size.items(), key=lambda x: x[1])[0] return self._dummy_pooler_run_task(hidden_states, max_task) - def profile_run(self) -> None: + def profile_run(self, randomize_inputs: bool = False) -> None: # Profile with multimodal encoder & encoder cache. if self.supports_mm_inputs: mm_config = self.model_config.multimodal_config @@ -6481,7 +6484,7 @@ class GPUModelRunner( # Add `is_profile` here to pre-allocate communication buffers hidden_states, last_hidden_states = self._dummy_run( - self.max_num_tokens, is_profile=True + self.max_num_tokens, is_profile=True, randomize_inputs=randomize_inputs ) if get_pp_group().is_last_rank: if self.is_pooling_model: diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 247265b659..e9021bd142 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -613,7 +613,7 @@ class Worker(WorkerBase): if kv_cache_memory_bytes := self.cache_config.kv_cache_memory_bytes: # still need a profile run which compiles the model for # max_num_batched_tokens - self.model_runner.profile_run() + self.model_runner.profile_run(randomize_inputs=self.randomize_dummy_inputs) msg = ( f"Initial free memory {format_gib(self.init_snapshot.free_memory)} " @@ -650,7 +650,7 @@ class Worker(WorkerBase): # memory_profiling measures, restoring the original limit. self._scoped_allocator_max_split(max_split_size_mb=20), ): - self.model_runner.profile_run() + self.model_runner.profile_run(randomize_inputs=self.randomize_dummy_inputs) # Profile CUDA graph memory if graphs will be captured. # ROCm is included: #44825 moved the profiler to @@ -885,7 +885,12 @@ class Worker(WorkerBase): # We skip EPLB here since we don't want to record dummy metrics for size in sorted(warmup_sizes, reverse=True): logger.info("Compile and warming up model for size %d", size) - self.model_runner._dummy_run(size, skip_eplb=True, remove_lora=False) + self.model_runner._dummy_run( + size, + skip_eplb=True, + remove_lora=False, + randomize_inputs=self.randomize_dummy_inputs, + ) self.model_runner.maybe_remove_all_loras(self.model_runner.lora_config) # Warmup and tune the kernels used during model execution before @@ -1387,9 +1392,21 @@ class Worker(WorkerBase): # Recreate it so the next profile_prefix is honored. self.profiler = None + @property + def randomize_dummy_inputs(self) -> bool: + # Not cached: elastic EP rewrites parallel_config in place on reconfigure. + return ( + envs.VLLM_RANDOMIZE_DP_DUMMY_INPUTS + and self.parallel_config.data_parallel_size > 1 + ) + def execute_dummy_batch(self) -> None: num_tokens = getattr(self.model_runner, "uniform_decode_query_len", 1) - self.model_runner._dummy_run(num_tokens, uniform_decode=True) + self.model_runner._dummy_run( + num_tokens, + uniform_decode=True, + randomize_inputs=self.randomize_dummy_inputs, + ) def add_lora(self, lora_request: LoRARequest) -> bool: return self.model_runner.add_lora(lora_request) From c4fc3c7d322bbcd1b25307ccbd9770c8eebc5c09 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Thu, 1 Oct 2026 16:09:14 +0800 Subject: [PATCH 191/248] [Bugfix][Spec Decode] Per-module LM heads for multi-layer MTP on Model Runner V2 (#58921) Signed-off-by: zjy0516 Co-authored-by: chanh Co-authored-by: Claude Opus 5.5 --- vllm/model_executor/models/step3p5_mtp.py | 17 ++++++++++++++++- vllm/models/inkling/amd/mtp.py | 6 +++++- vllm/models/inkling/nvidia/mtp.py | 6 +++++- vllm/v1/worker/gpu/spec_decode/eagle/utils.py | 10 +++++++--- .../multi_module_mtp/speculator.py | 12 ++++++++++++ vllm/v1/worker/gpu/spec_decode/speculator.py | 19 +++++++++++++++---- 6 files changed, 60 insertions(+), 10 deletions(-) diff --git a/vllm/model_executor/models/step3p5_mtp.py b/vllm/model_executor/models/step3p5_mtp.py index d17b579c1a..9af8b26ce8 100644 --- a/vllm/model_executor/models/step3p5_mtp.py +++ b/vllm/model_executor/models/step3p5_mtp.py @@ -22,7 +22,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .step3p5 import Step3p5DecoderLayer -from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix +from .utils import ( + get_draft_quant_config, + get_spec_layer_idx_from_weight_name, + maybe_prefix, +) logger = init_logger(__name__) @@ -152,8 +156,19 @@ class Step3p5AMultiTokenPredictor(nn.Module): class Step3p5MTP(nn.Module): + # Each MTP layer's shared_head carries its own LM head weights. + has_own_lm_head = True + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + # The target config may be a multimodal wrapper (Step3.7); build the + # draft from its own text and quant configs. + assert vllm_config.speculative_config is not None + draft_quant_config = get_draft_quant_config(vllm_config) + vllm_config = vllm_config.with_hf_config( + vllm_config.speculative_config.draft_model_config.hf_config + ) + vllm_config.quant_config = draft_quant_config self.config = vllm_config.model_config.hf_config self.vllm_config = vllm_config self.model = Step3p5AMultiTokenPredictor( diff --git a/vllm/models/inkling/amd/mtp.py b/vllm/models/inkling/amd/mtp.py index f9cba37f3c..f2ed51c075 100644 --- a/vllm/models/inkling/amd/mtp.py +++ b/vllm/models/inkling/amd/mtp.py @@ -323,7 +323,11 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings): logits = logits[..., : self.logits_processor.org_vocab_size] return logits - def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor: + def get_top_tokens( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: """Greedy draft tokens via rank-local argmax + tiny (value, index) reduction — no full-vocab logits all-gather. The muP divisor is a positive scalar, so the argmax is invariant and the scaling is diff --git a/vllm/models/inkling/nvidia/mtp.py b/vllm/models/inkling/nvidia/mtp.py index f9cba37f3c..f2ed51c075 100644 --- a/vllm/models/inkling/nvidia/mtp.py +++ b/vllm/models/inkling/nvidia/mtp.py @@ -323,7 +323,11 @@ class InklingMTP(nn.Module, SupportsMultiModalEmbeddings): logits = logits[..., : self.logits_processor.org_vocab_size] return logits - def get_top_tokens(self, hidden_states: torch.Tensor) -> torch.Tensor: + def get_top_tokens( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: """Greedy draft tokens via rank-local argmax + tiny (value, index) reduction — no full-vocab logits all-gather. The muP divisor is a positive scalar, so the argmax is invariant and the scaling is diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py index 06bed9391e..65289bb18e 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -13,10 +13,14 @@ from vllm.v1.worker.gpu.spec_decode.utils import get_pp_safe_draft_load_config def _should_share(eagle: nn.Module, flag: str, draft, target) -> bool: - """Share when the draft has no own copy, or its copy matches the target.""" - if not getattr(eagle, flag, False) or draft is None: + """Share unless the draft declares its own copy that differs from the target. + + A draft that declares its own copy but has no top-level one (e.g. MTP + heads stored per layer) keeps it. + """ + if not getattr(eagle, flag, False): return True - if target is None: + if draft is None or target is None: return False # torch.equal on GPU allocates a bool mask the size of the input. # Use the faster GPU path when there is plenty of headroom; diff --git a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py index 928498edf1..c7026acdb8 100644 --- a/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/multi_module_mtp/speculator.py @@ -253,6 +253,17 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): ) return self.draft_tokens[:num_reqs] + # Each MTP module may carry its own LM head, selected by spec_step_idx. + def compute_draft_logits( + self, hidden_states: torch.Tensor, spec_step_idx: int + ) -> torch.Tensor: + return self.model.compute_logits(hidden_states, spec_step_idx=spec_step_idx) + + def get_draft_top_tokens( + self, hidden_states: torch.Tensor, spec_step_idx: int + ) -> torch.Tensor: + return self.model.get_top_tokens(hidden_states, spec_step_idx=spec_step_idx) + @torch.inference_mode() def _run_model( self, @@ -427,6 +438,7 @@ class MultiModuleMTPSpeculator(DraftModelSpeculator): self.seeds, self.current_draft_step, self.draft_logits, + spec_step_idx=step, ) self.draft_tokens[:num_reqs, step] = draft_tokens diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 17baddf0a8..20a3b29eab 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -403,6 +403,16 @@ class DraftModelSpeculator(BaseSpeculator): "(communication: O(2*tp_size) vs O(vocab_size))." ) + def compute_draft_logits( + self, hidden_states: torch.Tensor, spec_step_idx: int + ) -> torch.Tensor: + return self.model.compute_logits(hidden_states) + + def get_draft_top_tokens( + self, hidden_states: torch.Tensor, spec_step_idx: int + ) -> torch.Tensor: + return self.model.get_top_tokens(hidden_states) + def sample_draft( self, hidden_states: torch.Tensor, @@ -412,9 +422,13 @@ class DraftModelSpeculator(BaseSpeculator): seeds: torch.Tensor, draft_step: torch.Tensor, draft_logits: torch.Tensor | None, + spec_step_idx: int = 0, ) -> torch.Tensor: + if draft_logits is None and self.use_local_argmax_reduction: + return self.get_draft_top_tokens(hidden_states, spec_step_idx) + + logits = self.compute_draft_logits(hidden_states, spec_step_idx) if draft_logits is not None: - logits = self.model.compute_logits(hidden_states) sampler = ( gumbel_sample if self.draft_watermarker is None @@ -432,10 +446,7 @@ class DraftModelSpeculator(BaseSpeculator): logits_cache_col=draft_step, use_fp64=self.use_fp64_gumbel, ) - elif self.use_local_argmax_reduction: - return self.model.get_top_tokens(hidden_states) else: - logits = self.model.compute_logits(hidden_states) sampled = logits.argmax(dim=-1) self._maybe_predict_acceptance(logits, idx_mapping, draft_step) return sampled From 91fcec12350aa01cd151d866fd76a307df17d120 Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Thu, 1 Oct 2026 09:11:46 +0100 Subject: [PATCH 192/248] [Core][Logging] Add built-in JSON formatter (#58739) Add `--logging-config.formatter=json` to enable the python-json-logger formatter without having to use a complete Python logging dictConfig. Signed-off-by: Mark McLoughlin Co-authored-by: OpenAI Codex --- .buildkite/test_areas/misc.yaml | 4 +- examples/features/logging_configuration.md | 70 ++++++++++++++----- tests/entrypoints/launchers/test_cli_args.py | 3 + tests/test_logger.py | 31 ++++++++ ..._log_filter.py => test_uvicorn_logging.py} | 4 +- vllm/config/logging.py | 12 ++-- .../launchers/utils/server_utils.py | 15 ++-- vllm/logger.py | 20 ++++-- vllm/logging_utils/__init__.py | 8 +-- vllm/logging_utils/formatter.py | 4 ++ ...ccess_log_filter.py => uvicorn_logging.py} | 23 ++++-- 11 files changed, 148 insertions(+), 46 deletions(-) rename tests/{test_access_log_filter.py => test_uvicorn_logging.py} (99%) rename vllm/logging_utils/{access_log_filter.py => uvicorn_logging.py} (87%) diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 0c0e8c9a4d..dea2ca7a6f 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -530,7 +530,7 @@ steps: - vllm/v1/ - vllm/version.py - cmake/utils.cmake - - tests/test_access_log_filter.py + - tests/test_uvicorn_logging.py - tests/test_attention_backend_registry.py - tests/test_audio_media_size_precheck.py - tests/test_cmake_utils.py @@ -571,7 +571,7 @@ steps: - pip install cmake # test_cmake_utils.py shells out to cmake; the CPU image has none - python3 standalone_tests/lazy_imports.py - pytest -v -s device_allocator - - pytest -v -s test_access_log_filter.py + - pytest -v -s test_uvicorn_logging.py - pytest -v -s test_attention_backend_registry.py - pytest -v -s test_audio_media_size_precheck.py - pytest -v -s test_cmake_utils.py diff --git a/examples/features/logging_configuration.md b/examples/features/logging_configuration.md index f2e1e5b8a0..ee2d16aba0 100644 --- a/examples/features/logging_configuration.md +++ b/examples/features/logging_configuration.md @@ -6,6 +6,7 @@ robust and flexible configuration of the various loggers used by vLLM. For `vllm serve`, configure logging with CLI arguments: - Use the built-in configuration, optionally with `--log-level`. +- Emit structured JSON with `--logging-config.formatter json`. - Use a custom Python logging configuration file with `--logging-config.pylogging_config_file`. - Disable vLLM logging configuration with @@ -14,7 +15,7 @@ For `vllm serve`, configure logging with CLI arguments: ## CLI logging configuration `--logging-config` accepts a JSON object. Its fields are `log_level`, -`configure_logging`, and `pylogging_config_file`. For example: +`formatter`, `configure_logging`, and `pylogging_config_file`. For example: ```bash vllm serve mistralai/Mistral-7B-v0.1 \ @@ -28,6 +29,25 @@ vllm serve mistralai/Mistral-7B-v0.1 \ --logging-config.log_level DEBUG ``` +The built-in `text` formatter is selected by default. To emit one JSON object +per vLLM and Uvicorn log record, select the built-in `json` formatter: + +```bash +vllm serve mistralai/Mistral-7B-v0.1 \ + --logging-config.formatter json +``` + +The built-in JSON output includes `asctime`, `levelname`, `name`, +`processName`, `process`, `message`, and `vllm_process_name`. `processName` is +Python's process name, `vllm_process_name` identifies vLLM's logical process +(including worker ranks where applicable), and `process` is the +operating-system PID. + +The built-in profile formats records emitted through vLLM's Python loggers and +Uvicorn's server and access loggers when using `vllm serve`. It does not convert +direct writes to `stdout` or `stderr`. Use a custom configuration to format +other loggers. + `--log-level` is a shortcut for `--logging-config.log_level` and takes precedence if both are supplied. It sets the level of vLLM's built-in logging configuration. `--log-config-file` is deprecated and will be removed in @@ -41,11 +61,11 @@ The custom logging configuration file must be JSON following Python's [logging configuration dictionary schema](https://docs.python.org/3/library/logging.config.html#dictionary-schema-details). -!!! note "Custom configurations override `--log-level`" +!!! note "Custom configurations override built-in formatter settings" When `pylogging_config_file` is set, vLLM loads that JSON file and replaces - its built-in `dictConfig`; it does not merge the two. Therefore, - `--log-level` applies only when no custom configuration file is provided. - Set logger and handler levels in the custom file itself. vLLM applies the + its built-in `dictConfig`; it does not merge the two. Therefore, `log_level` + and `formatter` apply only when no custom configuration file is provided. + Set levels and formatters in the custom file itself. vLLM applies the resolved configuration in its child processes as well. ## Environment variables @@ -62,10 +82,11 @@ for those settings. ### Example 1: Customize vLLM root logger -For this example, we will customize the vLLM root logger to use +For more control over fields and handlers, configure the vLLM root logger and +Uvicorn's server and access loggers with [`python-json-logger`](https://github.com/nhairs/python-json-logger) -(which is part of the container image) to log to -STDOUT of the console in JSON format with a log level of `INFO`. +(which is part of the container image) to log to STDOUT in JSON format at +`INFO` level. To begin, first, create an appropriate JSON logging configuration file: @@ -92,8 +113,24 @@ To begin, first, create an appropriate JSON logging configuration file: "handlers": ["console"], "level": "INFO", "propagate": false + }, + "uvicorn": { + "handlers": ["console"], + "level": "INFO", + "propagate": false + }, + "uvicorn.error": { + "handlers": ["console"], + "level": "INFO", + "propagate": false + }, + "uvicorn.access": { + "handlers": ["console"], + "level": "INFO", + "propagate": false } }, + "disable_existing_loggers": false, "version": 1 } ``` @@ -105,21 +142,20 @@ vllm serve mistralai/Mistral-7B-v0.1 --max-model-len 2048 \ --logging-config.pylogging_config_file /path/to/logging_config.json ``` -Each vLLM log record is one JSON object and includes `vllm_process_name`, which -identifies vLLM's logical process (including worker ranks where applicable). -The standard `process` record attribute contains the operating-system PID. -vLLM avoids altering `stdout` or `stderr`, so no text is prepended to JSON -log records. +With this custom configuration, each vLLM and Uvicorn log record is one JSON +object. The example selects `vllm_process_name` and `process` explicitly; both +fields are also present in the built-in JSON output described above. vLLM +avoids altering `stdout` or `stderr`, so no text is prepended to JSON log +records. This applies to records emitted through the configured Python loggers. A JSON formatter cannot convert unrelated output into JSON, such as a third-party library writing directly to `stdout` or `stderr`; configure or route such output separately when a consumer requires every collected line to be JSON. -When serving, `VLLM_LOGGING_CONFIG_PATH` is also used as Uvicorn's logging -configuration. This example configures only `vllm`; configure `uvicorn`, -`uvicorn.error`, and `uvicorn.access` with a JSON handler when those server -logs must also be structured. +When serving, vLLM also passes this file to Uvicorn as its logging +configuration. The `uvicorn`, `uvicorn.error`, and `uvicorn.access` entries +format its server and access logs as JSON. ### Example 2: Silence a particular vLLM logger diff --git a/tests/entrypoints/launchers/test_cli_args.py b/tests/entrypoints/launchers/test_cli_args.py index de32e82ba4..97c1aa5fc7 100644 --- a/tests/entrypoints/launchers/test_cli_args.py +++ b/tests/entrypoints/launchers/test_cli_args.py @@ -76,6 +76,8 @@ def test_logging_config_cli_args(serve_parser): [ "--logging-config", '{"log_level":"WARNING","pylogging_config_file":"/tmp/json.json"}', + "--logging-config.formatter", + "json", "--log-level", "DEBUG", "--log-config-file", @@ -85,6 +87,7 @@ def test_logging_config_cli_args(serve_parser): config = AsyncEngineArgs.from_cli_args(args).create_logging_config() assert config.log_level == "DEBUG" + assert config.formatter == "json" assert config.pylogging_config_file == "/tmp/flat.json" diff --git a/tests/test_logger.py b/tests/test_logger.py index f194853fa4..2a2f9d34be 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -4,6 +4,7 @@ import enum import io import json import logging +import multiprocessing import os import sys import tempfile @@ -21,6 +22,7 @@ from vllm.config import LoggingConfig from vllm.logger import ( _DATE_FORMAT, _FORMAT, + _JSON_FORMAT, _configure_vllm_root_logger, _use_color, configure_logging, @@ -204,6 +206,35 @@ def test_configure_logging_preserves_application_record_factory( logging.setLogRecordFactory(original_factory) +def test_builtin_json_formatter(monkeypatch): + output = io.StringIO() + + try: + with monkeypatch.context() as context: + context.setattr(sys, "stdout", output) + context.setattr(vllm_logger, "_vllm_process_info", None) + _configure_vllm_root_logger(LoggingConfig(formatter="json")) + decorate_logs("Worker_DP0") + init_logger("vllm.structured_log_probe").info("structured log probe") + + log = json.loads(output.getvalue()) + formatter = logging.getLogger("vllm").handlers[0].formatter + finally: + _configure_vllm_root_logger(LoggingConfig()) + + assert log == { + "asctime": log["asctime"], + "levelname": "INFO", + "name": "vllm.structured_log_probe", + "processName": multiprocessing.current_process().name, + "process": os.getpid(), + "message": "structured log probe", + "vllm_process_name": "Worker_DP0", + } + assert formatter is not None + assert formatter._fmt == _JSON_FORMAT + + def test_use_color_force_color(monkeypatch): """FORCE_COLOR forces colored logs without a TTY, while NO_COLOR and an explicit VLLM_LOGGING_COLOR=0 take precedence over it.""" diff --git a/tests/test_access_log_filter.py b/tests/test_uvicorn_logging.py similarity index 99% rename from tests/test_access_log_filter.py rename to tests/test_uvicorn_logging.py index 4f03892e77..f331a91214 100644 --- a/tests/test_access_log_filter.py +++ b/tests/test_uvicorn_logging.py @@ -1,10 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the UvicornAccessLogFilter class.""" +"""Tests for Uvicorn logging configuration and access log filtering.""" import logging -from vllm.logging_utils.access_log_filter import ( +from vllm.logging_utils.uvicorn_logging import ( UvicornAccessLogFilter, create_uvicorn_log_config, ) diff --git a/vllm/config/logging.py b/vllm/config/logging.py index bb807d617d..e3a69778de 100644 --- a/vllm/config/logging.py +++ b/vllm/config/logging.py @@ -9,6 +9,7 @@ import vllm.envs as envs from vllm.config.utils import config LogLevel = Literal["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"] +LogFormatter = Literal["text", "json"] @config @@ -17,8 +18,8 @@ class LoggingConfig: Supply a JSON object with ``--logging-config`` or individual fields with dotted arguments such as ``--logging-config.log_level DEBUG`` and - ``--logging-config.pylogging_config_file logging.json``. ``--log-level`` - and legacy ``--log-config-file`` override their matching fields. Set + ``--logging-config.formatter json``. ``--log-level`` and legacy + ``--log-config-file`` override their matching fields. Set ``configure_logging`` to false to skip applying a ``dictConfig``. """ @@ -27,6 +28,9 @@ class LoggingConfig: ) """Log level used when no custom logging configuration is provided.""" + formatter: LogFormatter = "text" + """Formatter used by the built-in logging configuration.""" + configure_logging: bool = Field(default_factory=lambda: envs.VLLM_CONFIGURE_LOGGING) """Whether to apply a Python logging configuration. @@ -41,6 +45,6 @@ class LoggingConfig: """Path to a Python logging JSON file using [``dictConfig`` schema](https://docs.python.org/3/library/logging.config.html#configuration-file-format). - A custom ``dictConfig`` is authoritative over ``log_level`` for logger, - handler, and formatter settings. + A custom ``dictConfig`` is authoritative over ``log_level`` and + ``formatter`` for logger, handler, and formatter settings. """ diff --git a/vllm/entrypoints/launchers/utils/server_utils.py b/vllm/entrypoints/launchers/utils/server_utils.py index ada01706db..815374dec3 100644 --- a/vllm/entrypoints/launchers/utils/server_utils.py +++ b/vllm/entrypoints/launchers/utils/server_utils.py @@ -8,6 +8,7 @@ from contextlib import asynccontextmanager from fastapi import FastAPI from vllm import envs +from vllm.config.logging import LogFormatter from vllm.engine.protocol import EngineClient from vllm.logger import init_logger from vllm.utils.gc_utils import freeze_gc_heap @@ -33,8 +34,8 @@ def get_uvicorn_log_config(args: Namespace) -> dict | None: Priority: 1. If log_config_file is specified, use it - 2. If disable_access_log_for_endpoints is specified, create a config with - the access log filter + 2. If disable_access_log_for_endpoints or JSON formatting is specified, + create a config with the requested filter and format 3. Otherwise, return None (use uvicorn defaults) """ # First, try to load from file if specified @@ -48,19 +49,21 @@ def get_uvicorn_log_config(args: Namespace) -> dict | None: if log_config is not None: return log_config - # If endpoints to filter are specified, create a config with the filter - if args.disable_access_log_for_endpoints: + formatter: LogFormatter = ( + logging_config.formatter if logging_config is not None else "text" + ) + if args.disable_access_log_for_endpoints or formatter == "json": from vllm.logging_utils import create_uvicorn_log_config - # Parse comma-separated string into list excluded_paths = [ p.strip() - for p in args.disable_access_log_for_endpoints.split(",") + for p in (args.disable_access_log_for_endpoints or "").split(",") if p.strip() ] return create_uvicorn_log_config( excluded_paths=excluded_paths, log_level=args.uvicorn_log_level, + formatter=formatter, ) return None diff --git a/vllm/logger.py b/vllm/logger.py index 24692aa392..6a5e994601 100644 --- a/vllm/logger.py +++ b/vllm/logger.py @@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast import vllm.envs as envs from vllm.logging_utils import ColoredFormatter, NewLineFormatter +from vllm.logging_utils.formatter import JSON_FORMAT as _JSON_FORMAT if TYPE_CHECKING: from vllm.config.logging import LoggingConfig @@ -30,7 +31,6 @@ _FORMAT = ( "[%(fileinfo)s:%(lineno)d] %(message)s" ) _DATE_FORMAT = "%m-%d %H:%M:%S" - _vllm_process_info: tuple[str, int] | None = None @@ -211,10 +211,12 @@ def _configure_vllm_root_logger(config: "LoggingConfig | None" = None) -> None: configure_logging = envs.VLLM_CONFIGURE_LOGGING log_level = envs.VLLM_LOGGING_LEVEL log_config_file = envs.VLLM_LOGGING_CONFIG_PATH + log_formatter = "text" else: configure_logging = config.configure_logging log_level = config.log_level log_config_file = config.pylogging_config_file + log_formatter = config.formatter if not configure_logging and log_config_file: raise RuntimeError( @@ -233,12 +235,22 @@ def _configure_vllm_root_logger(config: "LoggingConfig | None" = None) -> None: # Refresh these values in case env vars have changed. vllm_handler["level"] = log_level vllm_handler["stream"] = envs.VLLM_LOGGING_STREAM - vllm_handler["formatter"] = "vllm_color" if _use_color() else "vllm" + for formatter in logging_config["formatters"].values(): + formatter["log_level"] = log_level + + if log_formatter == "json": + logging_config["formatters"] = { + "vllm_json": { + "class": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": _JSON_FORMAT, + } + } + vllm_handler["formatter"] = "vllm_json" + else: + vllm_handler["formatter"] = "vllm_color" if _use_color() else "vllm" vllm_loggers = logging_config["loggers"]["vllm"] vllm_loggers["level"] = log_level - for formatter in logging_config["formatters"].values(): - formatter["log_level"] = log_level if log_config_file: if not path.exists(log_config_file): diff --git a/vllm/logging_utils/__init__.py b/vllm/logging_utils/__init__.py index b83d499dbb..ddd18567f9 100644 --- a/vllm/logging_utils/__init__.py +++ b/vllm/logging_utils/__init__.py @@ -1,14 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.logging_utils.access_log_filter import ( - UvicornAccessLogFilter, - create_uvicorn_log_config, -) from vllm.logging_utils.formatter import ColoredFormatter, NewLineFormatter from vllm.logging_utils.lazy import lazy from vllm.logging_utils.log_time import logtime from vllm.logging_utils.torch_tensor import tensors_str_no_data +from vllm.logging_utils.uvicorn_logging import ( + UvicornAccessLogFilter, + create_uvicorn_log_config, +) __all__ = [ "NewLineFormatter", diff --git a/vllm/logging_utils/formatter.py b/vllm/logging_utils/formatter.py index 205bb36405..13ec3c6617 100644 --- a/vllm/logging_utils/formatter.py +++ b/vllm/logging_utils/formatter.py @@ -4,6 +4,10 @@ import logging from pathlib import Path +JSON_FORMAT = ( + "%(asctime)s %(levelname)s %(name)s %(processName)s %(process)d %(message)s" +) + class NewLineFormatter(logging.Formatter): """Adds logging prefix to newlines to align multi-line messages.""" diff --git a/vllm/logging_utils/access_log_filter.py b/vllm/logging_utils/uvicorn_logging.py similarity index 87% rename from vllm/logging_utils/access_log_filter.py rename to vllm/logging_utils/uvicorn_logging.py index 086449ead8..04a4549361 100644 --- a/vllm/logging_utils/access_log_filter.py +++ b/vllm/logging_utils/uvicorn_logging.py @@ -1,15 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Access log filter for uvicorn to exclude specific endpoints from logging. - -This module provides a logging filter that can be used to suppress access logs -for specific endpoints (e.g., /health, /metrics) to reduce log noise in -production environments. -""" +"""Uvicorn logging configuration and access log filtering.""" import logging +from typing import Literal from urllib.parse import urlparse +from vllm.logging_utils.formatter import JSON_FORMAT + class UvicornAccessLogFilter(logging.Filter): """A logging filter that excludes access logs for specified endpoint paths. @@ -70,8 +68,9 @@ class UvicornAccessLogFilter(logging.Filter): def create_uvicorn_log_config( excluded_paths: list[str] | None = None, log_level: str = "info", + formatter: Literal["text", "json"] = "text", ) -> dict: - """Create a uvicorn logging configuration with access log filtering. + """Create a uvicorn logging configuration with optional access filtering. This function generates a logging configuration dictionary that can be passed to uvicorn's `log_config` parameter. It sets up the access log @@ -80,6 +79,7 @@ def create_uvicorn_log_config( Args: excluded_paths: List of URL paths to exclude from access logs. log_level: The log level for uvicorn loggers. + formatter: Format used for uvicorn logs. Returns: A dictionary containing the logging configuration. @@ -140,4 +140,13 @@ def create_uvicorn_log_config( }, }, } + if formatter == "json": + json_formatter = { + "class": "pythonjsonlogger.jsonlogger.JsonFormatter", + "format": JSON_FORMAT, + } + config["formatters"] = { + "default": json_formatter.copy(), + "access": json_formatter.copy(), + } return config From 24fb5fc08c2bcae8575bd2e861f463d8cf919ffb Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 1 Oct 2026 16:16:38 +0800 Subject: [PATCH 193/248] [Rust Frontend] Add `hf` parser for response templates (#59005) Signed-off-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 --- rust/Cargo.lock | 4 + rust/Cargo.toml | 3 + rust/src/chat/src/backend/hf.rs | 56 +- rust/src/chat/src/lib.rs | 10 +- rust/src/chat/src/output/default/mod.rs | 150 ++- rust/src/chat/src/parser/unified.rs | 11 +- rust/src/chat/tests/roundtrip.rs | 10 + rust/src/parser/Cargo.toml | 4 + rust/src/parser/src/tool/mod.rs | 1 + rust/src/parser/src/tool/parameters.rs | 32 +- rust/src/parser/src/unified/combined.rs | 5 +- rust/src/parser/src/unified/hf/content.rs | 537 ++++++++++ .../src/unified/hf/fixtures/differential.json | 440 +++++++++ .../hf/fixtures/generate_differential.py | 264 +++++ rust/src/parser/src/unified/hf/mod.rs | 53 + rust/src/parser/src/unified/hf/parser.rs | 730 ++++++++++++++ rust/src/parser/src/unified/hf/pattern.rs | 403 ++++++++ rust/src/parser/src/unified/hf/spec.rs | 202 ++++ rust/src/parser/src/unified/hf/template.rs | 481 +++++++++ rust/src/parser/src/unified/hf/tests.rs | 933 ++++++++++++++++++ rust/src/parser/src/unified/hf/transform.rs | 263 +++++ rust/src/parser/src/unified/mod.rs | 11 +- rust/src/text/src/backend/hf/config.rs | 3 + 23 files changed, 4573 insertions(+), 33 deletions(-) create mode 100644 rust/src/parser/src/unified/hf/content.rs create mode 100644 rust/src/parser/src/unified/hf/fixtures/differential.json create mode 100644 rust/src/parser/src/unified/hf/fixtures/generate_differential.py create mode 100644 rust/src/parser/src/unified/hf/mod.rs create mode 100644 rust/src/parser/src/unified/hf/parser.rs create mode 100644 rust/src/parser/src/unified/hf/pattern.rs create mode 100644 rust/src/parser/src/unified/hf/spec.rs create mode 100644 rust/src/parser/src/unified/hf/template.rs create mode 100644 rust/src/parser/src/unified/hf/tests.rs create mode 100644 rust/src/parser/src/unified/hf/transform.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index bd5a63e81d..2ba9832b6b 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5971,12 +5971,16 @@ dependencies = [ "expect-test", "futures", "openai-protocol", + "regex-automata", + "regex-syntax", "serde", "serde_json", + "serde_path_to_error", "serde_with", "thiserror 2.0.18", "thiserror-ext", "tool-parser", + "tracing", "vllm-tokenizer", "winnow", "xgrammar-structural-tag", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a2e6d0c1ed..e9935d9e28 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -85,6 +85,8 @@ rand = "0.9.3" rand_distr = "0.5.1" rayon = "1.11.0" reasoning-parser = "1.2.2" +regex-automata = "0.4.14" +regex-syntax = "0.8.10" reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] } reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] } riptoken = { version = "0.3.0", default-features = false } @@ -96,6 +98,7 @@ serde = { version = "1.0.228", features = ["derive", "rc"] } serde-json-fmt = "0.1.0" serde_default = "0.2.0" serde_json = { version = "1.0.145", features = ["preserve_order"] } +serde_path_to_error = "0.1.20" serde_repr = "0.1.20" serde_tuple = "1.1.3" serde_with = "3.21.0" diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 07f0b2dce8..174115693a 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -3,8 +3,11 @@ use std::sync::Arc; -use tracing::info; -use vllm_text::backend::hf::{HfTextBackend, ResolvedModelFiles, load_model_config}; +use thiserror_ext::AsReport as _; +use tracing::{info, warn}; +use vllm_text::backend::hf::{ + HfTextBackend, ResolvedModelFiles, load_model_config, load_tokenizer_config, +}; use vllm_text::tokenizer::DynTokenizer; use vllm_text::{DynTextBackend, TextBackend as _}; @@ -17,6 +20,7 @@ use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo}; use crate::output::{ DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides, }; +use crate::parser::unified::{HfTemplateError, ResponseTemplate}; use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo}; use crate::renderer::{ DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer, @@ -32,6 +36,9 @@ pub struct HfChatBackend { tokenizer: DynTokenizer, chat_renderer: DynChatRenderer, multimodal_model_info: Option, + /// `response_template` from `tokenizer_config.json` for the `hf` parser, or + /// why it is unavailable. + response_template: std::result::Result, HfTemplateError>, } impl HfChatBackend { @@ -61,6 +68,7 @@ impl HfChatBackend { )? }; let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref()); + let response_template = load_response_template(&files)?; let renderer = options.renderer.resolve(model_type); let chat_renderer: DynChatRenderer = match renderer { @@ -105,10 +113,35 @@ impl HfChatBackend { tokenizer, chat_renderer, multimodal_model_info, + response_template, }) } } +/// Load and compile the checkpoint's `response_template`, if any. +/// +/// A missing or unusable template is kept as an error, so that only requests +/// selecting the `hf` parser fail. +fn load_response_template( + files: &ResolvedModelFiles, +) -> Result, HfTemplateError>> { + let Some(template) = + load_tokenizer_config(files.tokenizer_config_path.as_deref())?.response_template + else { + return Ok(Err(HfTemplateError::Missing)); + }; + Ok(match ResponseTemplate::from_json(&template) { + Ok(template) => { + info!("loaded response_template for the `hf` parser"); + Ok(Arc::new(template)) + } + Err(error) => { + warn!(error = %error.as_report(), "the `hf` parser cannot execute the model's response_template"); + Err(error) + } + }) +} + impl ChatBackend for HfChatBackend { fn chat_renderer(&self) -> DynChatRenderer { self.chat_renderer.clone() @@ -128,14 +161,17 @@ impl ChatBackend for HfChatBackend { return Ok(Box::new(HarmonyChatOutputProcessor::new(request)?)); } - Ok(Box::new(DefaultChatOutputProcessor::new( - request, - &self.model_id, - self.tokenizer.clone(), - options.tool_call_parser, - options.reasoning_parser, - options.tool_strict_level, - )?)) + Ok(Box::new( + DefaultChatOutputProcessor::with_response_template( + request, + &self.model_id, + self.tokenizer.clone(), + &self.response_template, + options.tool_call_parser, + options.reasoning_parser, + options.tool_strict_level, + )?, + )) } } diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 86b94b6b9c..f437745e39 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -412,6 +412,12 @@ mod tests { validate_parser_overrides(&selection, &selection).unwrap(); } + #[test] + fn validate_parser_overrides_accepts_explicit_hf() { + let selection = ParserSelection::Explicit("hf".to_string()); + validate_parser_overrides(&selection, &selection).unwrap(); + } + #[test] fn validate_parser_overrides_accepts_auto_and_none() { validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap(); @@ -425,7 +431,7 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string()); } #[test] @@ -436,6 +442,6 @@ mod tests { ) .unwrap_err(); - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string()); } } diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index ee067764cf..3f75aed63d 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -5,7 +5,7 @@ mod unified; -use std::sync::Once; +use std::sync::{Arc, Once}; use futures::StreamExt as _; use tracing::info; @@ -20,7 +20,9 @@ use crate::error::Result; use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; -use crate::parser::unified::UnifiedParserFactory; +use crate::parser::unified::{ + HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names, +}; use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool}; use crate::{Error, Result as ChatResult}; @@ -64,11 +66,48 @@ impl DefaultChatOutputProcessor { reasoning_parser: &ParserSelection, tool_strict_level: ToolStrictLevel, ) -> ChatResult { + Self::with_response_template( + request, + model_id, + tokenizer, + &Err(HfTemplateError::Missing), + tool_call_parser, + reasoning_parser, + tool_strict_level, + ) + } + + /// Like [`Self::new`], additionally providing the model's `response_template` + /// for the `hf` parser, or why it is unavailable. + pub fn with_response_template( + request: &mut ChatRequest, + model_id: &str, + tokenizer: DynTokenizer, + response_template: &std::result::Result, HfTemplateError>, + tool_call_parser: &ParserSelection, + reasoning_parser: &ParserSelection, + tool_strict_level: ToolStrictLevel, + ) -> ChatResult { + let tool_name = tool_call_parser.resolve_tool_name(model_id); + let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id); + // `hf` is opt-in: `Auto` never selects it, but a usable template is worth a hint. + let auto = [tool_call_parser, reasoning_parser] + .iter() + .any(|selection| matches!(selection, ParserSelection::Auto)); + if auto && tool_name.is_none() && reasoning_name.is_none() && response_template.is_ok() { + RESPONSE_TEMPLATE_HINT_ONCE.call_once(|| { + info!( + "the model ships a response_template; pass `--tool-call-parser hf \ + --reasoning-parser hf` to parse with it" + ); + }); + } let parser = if let Some(parser) = Self::resolve_optional_unified_parser( request.tools(), tokenizer.clone(), - tool_call_parser.resolve_tool_name(model_id), - reasoning_parser.resolve_reasoning_name(model_id), + response_template, + tool_name, + reasoning_name, )? { parser } else { @@ -141,6 +180,7 @@ impl DefaultChatOutputProcessor { fn resolve_optional_unified_parser( tools: &[ChatTool], tokenizer: DynTokenizer, + response_template: &std::result::Result, HfTemplateError>, tool_name: Option<&str>, reasoning_name: Option<&str>, ) -> ChatResult>> { @@ -157,7 +197,18 @@ impl DefaultChatOutputProcessor { }); } - let parser = factory.create(parser_name, tools, tokenizer)?; + // `hf` is built from the model's template, never by its registry constructor. + let parser = if parser_name == names::HF { + let template = + response_template.as_ref().map_err(|error| Error::ParserInitialization { + kind: "unified", + name: parser_name.to_string(), + error: Box::new(error.clone()), + })?; + Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer)) + } else { + factory.create(parser_name, tools, tokenizer)? + }; UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser")); Ok(Some(parser)) @@ -186,6 +237,7 @@ impl DefaultChatOutputProcessor { static TOOL_PARSER_LOG_ONCE: Once = Once::new(); static REASONING_PARSER_LOG_ONCE: Once = Once::new(); static UNIFIED_PARSER_LOG_ONCE: Once = Once::new(); +static RESPONSE_TEMPLATE_HINT_ONCE: Once = Once::new(); impl ChatOutputProcessor for DefaultChatOutputProcessor { fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { @@ -230,10 +282,12 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { mod tests { use std::sync::Arc; + use thiserror_ext::AsReport as _; use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; use crate::output::ChatOutputProcessor; + use crate::parser::unified::{HfTemplateError, ResponseTemplate}; use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext}; @@ -390,4 +444,90 @@ mod tests { expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=qwen3_xml, reasoning=gemma4"] .assert_eq(&format!("{error}")); } + + type TemplateResult = Result, HfTemplateError>; + + fn loaded_template() -> TemplateResult { + Ok(Arc::new( + ResponseTemplate::from_json(&serde_json::json!({ + "start_anchor": "<|turn>model\n", + "fields": { + "thinking": {"open": "<|channel>thought\n", "close": ""}, + "content": {"close": ""}, + }, + })) + .unwrap(), + )) + } + + fn invalid_template() -> TemplateResult { + let error = ResponseTemplate::from_json(&serde_json::json!({"fields": {"content": {}}})) + .unwrap_err(); + Err(error) + } + + /// Build a processor for `model_id`, returning whether it keeps special + /// tokens (only parsers that need them, such as `hf`, turn this on). + fn keeps_special_tokens( + model_id: &str, + template: &TemplateResult, + tool: &ParserSelection, + reasoning: &ParserSelection, + ) -> crate::Result { + let mut request = ChatRequest::for_test(); + DefaultChatOutputProcessor::with_response_template( + &mut request, + model_id, + tokenizer(), + template, + tool, + reasoning, + ToolStrictLevel::Auto, + )?; + Ok(!request.decode_options.skip_special_tokens) + } + + #[test] + fn explicit_hf_uses_the_model_response_template() { + let hf = ParserSelection::Explicit("hf".to_string()); + assert!(keeps_special_tokens("other-model", &loaded_template(), &hf, &hf).unwrap()); + } + + #[test] + fn explicit_hf_without_usable_template_fails() { + let hf = ParserSelection::Explicit("hf".to_string()); + let error = |template| { + keeps_special_tokens("other-model", &template, &hf, &hf) + .unwrap_err() + .to_report_string() + }; + expect_test::expect!["failed to initialize unified parser `hf`: the model's tokenizer_config.json provides no response_template"] + .assert_eq(&error(Err(HfTemplateError::Missing))); + expect_test::expect!["failed to initialize unified parser `hf`: invalid response_template: response_template must define 'start_anchor' or 'start_anchor_pattern'."] + .assert_eq(&error(invalid_template())); + } + + #[test] + fn auto_never_selects_hf() { + let auto = ParserSelection::Auto; + // Without a matching parser, parsing stays disabled even with a usable template. + assert!(!keeps_special_tokens("other-model", &loaded_template(), &auto, &auto).unwrap()); + // An unusable template is not consulted unless `hf` is selected. + assert!(!keeps_special_tokens("other-model", &invalid_template(), &auto, &auto).unwrap()); + keeps_special_tokens("google/gemma-4-27b-it", &invalid_template(), &auto, &auto).unwrap(); + } + + #[test] + fn hf_requires_matching_parser_selections() { + let error = keeps_special_tokens( + "other-model", + &loaded_template(), + &ParserSelection::Explicit("hf".to_string()), + &ParserSelection::None, + ) + .unwrap_err(); + + expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=hf, reasoning=none"] + .assert_eq(&format!("{error}")); + } } diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 52cabaf162..7db093c696 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -6,8 +6,8 @@ use std::sync::LazyLock; pub use vllm_parser::unified::{ - Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser, - KimiK3UnifiedParser, UnifiedParser, + Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, + InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser, }; use vllm_tokenizer::DynTokenizer; @@ -21,6 +21,11 @@ pub mod names { pub const HY_V4: &str = "hy_v4"; pub const INKLING: &str = "inkling"; pub const KIMI_K3: &str = "kimi_k3"; + + /// Executes the checkpoint's Hugging Face `response_template`. Registered for + /// name resolution only: it is built from the model's template, never by its + /// registry constructor. + pub const HF: &str = "hf"; } /// Constructor signature for one registered unified parser implementation. @@ -48,6 +53,8 @@ impl UnifiedParserFactory { factory.register_parser::(names::HY_V4); factory.register_parser::(names::INKLING); factory.register_parser::(names::KIMI_K3); + // Opt-in only: no model patterns map to it. + factory.register_parser::(names::HF); factory .register_pattern("gemma-4", names::GEMMA4) diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index de813e3045..b1a6505cf8 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -309,6 +309,15 @@ impl RoundtripCase { } } + /// Gemma4 through the checkpoint's `response_template` (`hf` parser). + fn gemma4_hf() -> Self { + Self { + tool_call_parser: ParserSelection::Explicit("hf".to_string()), + reasoning_parser: ParserSelection::Explicit("hf".to_string()), + ..Self::gemma4() + } + } + /// Kimi K2.5 tool-call format with `` reasoning tags. #[allow(dead_code)] fn kimi_k25() -> Self { @@ -459,6 +468,7 @@ roundtrip_tests! { step3p5 => [reasoning_and_content], nemotron_v3 => [reasoning_and_content], gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call + gemma4_hf => [tool_call_mix], kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history // K3 drops plain-assistant reasoning in history; tool-call turns keep it. kimi_k3 => [tool_call_mix], diff --git a/rust/src/parser/Cargo.toml b/rust/src/parser/Cargo.toml index 3086de03e9..aff4c795ab 100644 --- a/rust/src/parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -11,11 +11,15 @@ test-util = [] auto_impl.workspace = true easy-ext.workspace = true educe.workspace = true +regex-automata.workspace = true +regex-syntax.workspace = true serde.workspace = true serde_json.workspace = true +serde_path_to_error.workspace = true serde_with.workspace = true thiserror.workspace = true thiserror-ext.workspace = true +tracing.workspace = true vllm-tokenizer.workspace = true winnow.workspace = true xgrammar-structural-tag.workspace = true diff --git a/rust/src/parser/src/tool/mod.rs b/rust/src/parser/src/tool/mod.rs index 94f0b46711..7789752626 100644 --- a/rust/src/parser/src/tool/mod.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -34,6 +34,7 @@ pub use kimi_k2::KimiK2ToolParser; pub use mimo::MiMoToolParser; pub use minimax_m2::MinimaxM2ToolParser; pub use minimax_m3::MinimaxM3ToolParser; +pub(crate) use parameters::{ToolSchemas, convert_integer_text, convert_number_text}; pub use qwen_coder::Qwen3CoderToolParser; pub use seed_oss::SeedOssToolParser; use serde::{Deserialize, Serialize}; diff --git a/rust/src/parser/src/tool/parameters.rs b/rust/src/parser/src/tool/parameters.rs index 71601ca8af..19d700830b 100644 --- a/rust/src/parser/src/tool/parameters.rs +++ b/rust/src/parser/src/tool/parameters.rs @@ -9,7 +9,7 @@ use crate::tool::Tool; /// Normalized parameter schemas for all tools in one request. #[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(super) struct ToolSchemas { +pub(crate) struct ToolSchemas { tools: BTreeMap, } @@ -28,7 +28,7 @@ pub(super) struct ToolSchema { /// /// It can be either a raw text string, or a structured input with named child elements. #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) enum ParamInput { +pub(crate) enum ParamInput { Text(String), #[allow(dead_code)] Elements(Vec), @@ -42,7 +42,7 @@ impl From for ParamInput { /// One named structured parameter child. #[derive(Debug, Clone, PartialEq, Eq)] -pub(super) struct ParamElement { +pub(crate) struct ParamElement { pub name: String, pub value: ParamInput, } @@ -67,7 +67,7 @@ pub(super) enum JsonParamType { impl ToolSchemas { /// Normalize OpenAI-style tool parameter JSON schemas for one request. - pub(super) fn from_tools(tools: &[Tool]) -> Self { + pub(crate) fn from_tools(tools: &[Tool]) -> Self { let tools = tools .iter() .map(|tool| (tool.name.clone(), ToolSchema::from_schema(&tool.parameters))) @@ -98,7 +98,7 @@ impl ToolSchemas { } /// Convert one parameter value for one named tool. - pub(super) fn convert_param_with_schema

( + pub(crate) fn convert_param_with_schema

( &self, function_name: &str, name: &str, @@ -361,12 +361,8 @@ fn try_convert_value(param_type: &JsonParamType, input: &ParamInput) -> Option Option { match param_type { JsonParamType::String => Some(Value::String(value.to_string())), - JsonParamType::Integer => without_digit_separators(value.trim())? - .parse::() - .ok() - .map(Number::from) - .map(Value::Number), - JsonParamType::Number => try_convert_number(&without_digit_separators(value.trim())?), + JsonParamType::Integer => convert_integer_text(value), + JsonParamType::Number => convert_number_text(value), JsonParamType::Boolean => try_convert_boolean(value), JsonParamType::Object { .. } if value.trim().is_empty() => Some(Value::Object(Map::new())), JsonParamType::Array { .. } if value.trim().is_empty() => Some(Value::Array(Vec::new())), @@ -461,6 +457,20 @@ fn without_digit_separators(value: &str) -> Option> { valid.then(|| value.replace('_', "").into()) } +/// Convert raw text to a JSON integer, as for an `integer` parameter. +pub(crate) fn convert_integer_text(value: &str) -> Option { + without_digit_separators(value.trim())? + .parse::() + .ok() + .map(Number::from) + .map(Value::Number) +} + +/// Convert raw text to a JSON number, as for a `number` parameter. +pub(crate) fn convert_number_text(value: &str) -> Option { + try_convert_number(&without_digit_separators(value.trim())?) +} + /// Convert one raw string value to a JSON number. fn try_convert_number(value: &str) -> Option { serde_json::from_str::(value) diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index 48c77ef900..6f6849ec95 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -65,7 +65,10 @@ impl UnifiedParser for CombinedParser { where Self: Sized + 'static, { - Err(UnifiedParserError::CombinedParserConstructor) + Err(UnifiedParserError::NoNamedConstructor { + parser: "combined", + built_from: "split parser instances", + }) } fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { diff --git a/rust/src/parser/src/unified/hf/content.rs b/rust/src/parser/src/unified/hf/content.rs new file mode 100644 index 0000000000..d62ba9305e --- /dev/null +++ b/rust/src/parser/src/unified/hf/content.rs @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Content parsers: turn one region's captured text into a JSON value. +//! +//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and +//! parses it into a single key in the output message dictionary. + +use regex_automata::meta::Regex; +use regex_automata::util::syntax; +use serde::Deserialize; +use serde_json::{Map, Value}; +use thiserror_ext::AsReport as _; + +use super::spec::{ContentKind, deserialize}; +use super::{Result, invalid, unsupported}; +use crate::tool::{convert_integer_text, convert_number_text}; +use crate::unified::{self, parsing_failed}; + +// Sentinel characters for lax-JSON string pre-extraction — ASCII control chars +// that should never appear in real LLM output. +const LAX_OPEN: char = '\u{1}'; +const LAX_CLOSE: char = '\u{2}'; + +/// A configured content parser (`content` plus `content_args` of one field). +#[derive(Debug, Clone)] +pub(super) enum ContentParser { + Text(TextArgs), + /// Converted like an `integer` tool parameter. + Int, + /// Converted like a `number` tool parameter. + Float, + Bool(TextArgs), + Json(JsonArgs), + XmlInline(XmlInlineArgs), + KvLines(KvLinesArgs), +} + +/// The `strip` argument shared by parsers that go through `_text`. +#[derive(Debug, Clone, Copy, Deserialize)] +pub(super) struct TextArgs { + #[serde(default = "default_strip")] + pub strip: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub(super) struct JsonArgs { + #[serde(flatten)] + text: TextArgs, + /// Quote bare-identifier keys before parsing. + #[serde(default)] + unquoted_keys: bool, + /// Strings delimited by these custom markers are pre-extracted, then restored + /// as standard JSON strings. + #[serde(default)] + string_delims: Vec<(String, String)>, + /// Return stripped text if parsing fails. + #[serde(default)] + allow_non_json: bool, +} + +#[derive(Debug, Clone)] +pub(super) struct XmlInlineArgs { + tag_pattern: Regex, + /// Index of the required `key` group. + key_group: usize, + /// Index of the `value` group, if the pattern declares one. + value_group: Option, + value_parser: Option>, + /// Collect duplicate keys into a list. + merge_duplicates: bool, +} + +#[derive(Debug, Clone)] +pub(super) struct KvLinesArgs { + text: TextArgs, + line_sep: String, + kv_sep: String, + value_parser: Option>, +} + +/// `content_args` of `xml-inline`. +#[derive(Deserialize)] +struct XmlInlineSpec { + /// Must have named groups `key` and `value`. + tag_pattern: String, + value_parser: Option, + #[serde(default)] + merge_duplicates: bool, +} + +/// `content_args` of `kv-lines`. +#[derive(Deserialize)] +struct KvLinesSpec { + #[serde(flatten)] + text: TextArgs, + #[serde(default = "default_line_sep")] + line_sep: String, + #[serde(default = "default_kv_sep")] + kv_sep: String, + value_parser: Option, +} + +/// A nested `value_parser`: `{"name": ..., "args": {...}}`. +#[derive(Deserialize)] +struct ValueParserSpec { + #[serde(default)] + name: ContentKind, + #[serde(default)] + args: Map, +} + +impl ContentParser { + /// Build a parser from its kind and `content_args`. + /// + /// Unknown argument keys are ignored, as in Transformers. Argument types and the + /// `xml-inline` tag pattern are checked here rather than at parse time. + pub fn new(scope: &str, kind: ContentKind, args: &Map) -> Result { + let context = format!("{scope}: content_args"); + let args = Value::Object(args.clone()); + Ok(match kind { + ContentKind::Text => Self::Text(deserialize(&args, &context)?), + // Number conversion ignores `strip`, but a malformed value is still rejected. + ContentKind::Int => { + deserialize::(&args, &context)?; + Self::Int + } + ContentKind::Float => { + deserialize::(&args, &context)?; + Self::Float + } + ContentKind::Bool => Self::Bool(deserialize(&args, &context)?), + ContentKind::Json => { + let args: JsonArgs = deserialize(&args, &context)?; + if args + .string_delims + .iter() + .any(|(open, close)| open.is_empty() || close.is_empty()) + { + return Err(invalid!("{context}: string_delims cannot be empty strings")); + } + Self::Json(args) + } + ContentKind::XmlInline => { + let spec: XmlInlineSpec = deserialize(&args, &context)?; + let tag_pattern = compile_regex(scope, &spec.tag_pattern)?; + let group_info = tag_pattern.group_info(); + let Some(key_group) = group_info.to_index(Default::default(), "key") else { + return Err(invalid!( + "{scope}: xml-inline: tag_pattern must have a named group 'key'. Pattern: {}", + spec.tag_pattern + )); + }; + let value_group = group_info.to_index(Default::default(), "value"); + Self::XmlInline(XmlInlineArgs { + tag_pattern, + key_group, + value_group, + value_parser: ValueParserSpec::compile(scope, spec.value_parser)?, + merge_duplicates: spec.merge_duplicates, + }) + } + ContentKind::KvLines => { + let spec: KvLinesSpec = deserialize(&args, &context)?; + if spec.line_sep.is_empty() { + return Err(invalid!("{context}: line_sep cannot be empty")); + } + Self::KvLines(KvLinesArgs { + text: spec.text, + line_sep: spec.line_sep, + kv_sep: spec.kv_sep, + value_parser: ValueParserSpec::compile(scope, spec.value_parser)?, + }) + } + }) + } + + /// Parse one region body. + pub fn parse(&self, text: &str) -> unified::Result { + match self { + Self::Text(args) => Ok(Value::String(args.apply(text).to_string())), + // Numbers follow the tool-parameter conversion. + Self::Int => convert_integer_text(text) + .ok_or_else(|| parsing_failed!("int: invalid literal {text:?}")), + Self::Float => convert_number_text(text) + .ok_or_else(|| parsing_failed!("float: invalid or non-finite literal {text:?}")), + Self::Bool(args) => { + let text = args.apply(text).to_lowercase(); + Ok(Value::Bool(text == "true" || text == "1")) + } + Self::Json(args) => args.parse(text), + Self::XmlInline(args) => args.parse(text), + Self::KvLines(args) => args.parse(text), + } + } +} + +impl ValueParserSpec { + /// Build the nested parser, if any. + fn compile(scope: &str, spec: Option) -> Result>> { + spec.map(|spec| { + ContentParser::new(&format!("{scope}: value_parser"), spec.name, &spec.args) + .map(Box::new) + }) + .transpose() + } +} + +impl TextArgs { + /// Apply `_text`: strip surrounding whitespace unless disabled. + pub fn apply(self, text: &str) -> &str { + if self.strip { text.trim() } else { text } + } +} + +impl JsonArgs { + /// JSON parser with optional dialect knobs for LLM-emitted quirks. + fn parse(&self, text: &str) -> unified::Result { + if !self.string_delims.is_empty() && text.contains([LAX_OPEN, LAX_CLOSE]) { + return Err(parsing_failed!( + "json: input contains reserved sentinel characters (\\x01/\\x02); cannot parse safely." + )); + } + + let mut working = text.to_string(); + let mut captured: Vec = Vec::new(); + for (open, close) in &self.string_delims { + working = extract_delimited(&working, open, close, &mut captured); + } + + if self.unquoted_keys { + working = quote_bare_keys(&working); + } + + for (index, string) in captured.iter().enumerate() { + let quoted = serde_json::to_string(string).map_err(|error| { + parsing_failed!("json: failed to quote string: {}", error.as_report()) + })?; + working = working.replace(&format!("{LAX_OPEN}{index}{LAX_CLOSE}"), "ed); + } + + match serde_json::from_str(&working) { + Ok(value) => Ok(value), + Err(_) if self.allow_non_json => Ok(Value::String(self.text.apply(text).to_string())), + Err(error) if working == text => Err(parsing_failed!( + "json parser could not parse region as JSON.\nContent: {text:?}\nError: {}", + error.as_report() + )), + Err(error) => Err(parsing_failed!( + "json: could not parse after dialect transforms.\nOriginal: {text:?}\nTransformed: {working:?}\nError: {}", + error.as_report() + )), + } + } +} + +impl XmlInlineArgs { + /// Parse shallow XML-ish tags into a dict. + fn parse(&self, text: &str) -> unified::Result { + let mut out = Map::new(); + for captures in self.tag_pattern.captures_iter(text) { + let key = + captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else( + || parsing_failed!("xml-inline: named group 'key' did not participate"), + )?; + let raw = match self.value_group { + // The pattern has no `value` group. + None => Some(""), + Some(index) => captures.get_group(index).map(|span| &text[span.range()]), + }; + let value = match raw { + Some(raw) => sub_parse(raw, self.value_parser.as_deref())?, + None if self.value_parser.is_none() => Value::Null, + None => { + return Err(parsing_failed!( + "xml-inline: named group 'value' did not participate" + )); + } + }; + match out.get_mut(key) { + Some(existing) if self.merge_duplicates => match existing { + Value::Array(values) => values.push(value), + _ => { + let first = existing.take(); + *existing = Value::Array(vec![first, value]); + } + }, + _ => { + out.insert(key.to_string(), value); + } + } + } + Ok(Value::Object(out)) + } +} + +impl KvLinesArgs { + /// Parse line-delimited `keyvalue` pairs into a dict. + fn parse(&self, text: &str) -> unified::Result { + let mut out = Map::new(); + for line in text.split(self.line_sep.as_str()) { + let line = self.text.apply(line); + let Some((key, value)) = line.split_once(self.kv_sep.as_str()) else { + continue; + }; + let (key, value) = (self.text.apply(key), self.text.apply(value)); + out.insert( + key.to_string(), + sub_parse(value, self.value_parser.as_deref())?, + ); + } + Ok(Value::Object(out)) + } +} + +/// Parse `raw` with an optional nested value parser. +fn sub_parse(raw: &str, value_parser: Option<&ContentParser>) -> unified::Result { + match value_parser { + None => Ok(Value::String(raw.to_string())), + Some(parser) => parser.parse(raw), + } +} + +/// Replace every `open(.*?)close` span with an indexed sentinel, collecting the +/// inner strings. Equivalent to the non-overlapping, left-to-right `re.sub` with +/// `re.DOTALL` used by Transformers. +fn extract_delimited(text: &str, open: &str, close: &str, captured: &mut Vec) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find(open) { + let inner_start = start + open.len(); + // No close after this open means no close after any later open either. + let Some(inner_len) = rest[inner_start..].find(close) else { + break; + }; + out.push_str(&rest[..start]); + captured.push(rest[inner_start..inner_start + inner_len].to_string()); + out.push(LAX_OPEN); + out.push_str(&(captured.len() - 1).to_string()); + out.push(LAX_CLOSE); + rest = &rest[inner_start + inner_len + close.len()..]; + } + out.push_str(rest); + out +} + +/// Quote bare-identifier keys: Transformers' `(?<=[{,])(\w+):` → `"\1":`. +/// +/// The Rust regex engine has no lookbehind, so the preceding `{` or `,` is +/// matched and written back. Matches cannot overlap either way, so the rewrite is +/// equivalent. +fn quote_bare_keys(text: &str) -> String { + static BARE_KEY: std::sync::LazyLock = + std::sync::LazyLock::new(|| Regex::new(r"([{,])(\w+):").expect("valid regex")); + + let mut out = String::with_capacity(text.len()); + let mut last = 0; + for captures in BARE_KEY.captures_iter(text) { + let whole = captures.get_match().expect("match"); + let separator = captures.get_group(1).expect("group 1"); + let key = captures.get_group(2).expect("group 2"); + out.push_str(&text[last..whole.start()]); + out.push_str(&text[separator.range()]); + out.push('"'); + out.push_str(&text[key.range()]); + out.push_str("\":"); + last = whole.end(); + } + out.push_str(&text[last..]); + out +} + +/// Compile a Python-syntax regex with `re.DOTALL`, as Transformers does. +pub(super) fn compile_regex(scope: &str, pattern: &str) -> Result { + Regex::builder() + .syntax(syntax::Config::new().dot_matches_new_line(true)) + .build(pattern) + .map_err(|error| { + unsupported!( + "{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}", + error.as_report() + ) + }) +} + +fn default_strip() -> bool { + true +} + +fn default_line_sep() -> String { + "\n".to_string() +} + +fn default_kv_sep() -> String { + ":".to_string() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn kind(name: &str) -> ContentKind { + serde_json::from_value(json!(name)).unwrap() + } + + fn parser(name: &str, args: Value) -> ContentParser { + ContentParser::new("test", kind(name), args.as_object().unwrap()).unwrap() + } + + #[test] + fn scalar_parsers_convert_like_tool_parameters() { + let int = parser("int", json!({})); + assert_eq!(int.parse(" 42\n").unwrap(), json!(42)); + assert_eq!(int.parse("-1_000").unwrap(), json!(-1000)); + assert!(int.parse("1__0").is_err()); + assert!(int.parse("4.2").is_err()); + + let float = parser("float", json!({})); + assert_eq!(float.parse("1.5").unwrap(), json!(1.5)); + assert_eq!(float.parse(" 1e3 ").unwrap(), json!(1000.0)); + assert!(float.parse("inf").is_err(), "JSON cannot represent inf"); + + let boolean = parser("bool", json!({})); + assert_eq!(boolean.parse(" TRUE ").unwrap(), json!(true)); + assert_eq!(boolean.parse("1").unwrap(), json!(true)); + assert_eq!(boolean.parse("yes").unwrap(), json!(false)); + + let text = parser("text", json!({"strip": false})); + assert_eq!(text.parse(" a \n").unwrap(), json!(" a \n")); + assert_eq!( + parser("text", json!({})).parse(" a \n").unwrap(), + json!("a") + ); + } + + #[test] + fn json_dialects_match_gemma4_syntax() { + let json = parser( + "json", + json!({"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}), + ); + let value = json + .parse( + r#"{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],null_value:null,number_value:1,string_value:<|"|>a, b: {c}<|"|>,struct_value:{foo:<|"|>bar<|"|>}}"#, + ) + .unwrap(); + assert_eq!( + value, + json!({ + "bool_value": true, + "list_value": ["foo", "bar"], + "null_value": null, + "number_value": 1, + "string_value": "a, b: {c}", + "struct_value": {"foo": "bar"}, + }) + ); + assert!(json.parse("{a:\u{1}}").is_err()); + } + + #[test] + fn json_allow_non_json_returns_stripped_text() { + let json = parser("json", json!({"allow_non_json": true})); + assert_eq!(json.parse(" celsius \n").unwrap(), json!("celsius")); + assert_eq!(json.parse(" [1, 2] ").unwrap(), json!([1, 2])); + assert!(parser("json", json!({})).parse("celsius").is_err()); + } + + #[test] + fn xml_inline_parses_tags_with_value_parser_and_merging() { + let xml = parser( + "xml-inline", + json!({ + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "value_parser": {"name": "json", "args": {"allow_non_json": true}}, + "merge_duplicates": true, + }), + ); + let value = xml + .parse( + "\n[{\"city\": \"Paris\"}]\n\n\ + \ncelsius\n2", + ) + .unwrap(); + assert_eq!( + value, + json!({"locations": [{"city": "Paris"}], "unit": ["celsius", 2]}) + ); + } + + #[test] + fn kv_lines_split_and_strip() { + let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}})); + assert_eq!( + kv.parse(" a : 1 \n\nnot a pair\nb:2").unwrap(), + json!({"a": 1, "b": 2}) + ); + } + + #[test] + fn invalid_arguments_are_rejected_at_load() { + let error = |name: &str, args: Value| { + ContentParser::new("Field 'x'", kind(name), args.as_object().unwrap()).unwrap_err() + }; + expect_test::expect![[r#" + [ + Invalid { + message: "Field 'x': content_args.strip: invalid type: string \"no\", expected a boolean", + }, + Invalid { + message: "Field 'x': content_args.string_delims[0]: invalid length 1, expected a tuple of size 2", + }, + Invalid { + message: "Field 'x': content_args: missing field `tag_pattern`", + }, + Invalid { + message: "Field 'x': xml-inline: tag_pattern must have a named group 'key'. Pattern: <(?P\\w+)>", + }, + Invalid { + message: "Field 'x': content_args.value_parser.name: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`", + }, + ] + "#]].assert_debug_eq(&[ + error("text", json!({"strip": "no"})), + error("json", json!({"string_delims": [[""]]})), + error("xml-inline", json!({})), + error("xml-inline", json!({"tag_pattern": r"<(?P\w+)>"})), + error("kv-lines", json!({"value_parser": {"name": "yaml"}})), + ]); + assert!(matches!( + error("xml-inline", json!({"tag_pattern": r"(?<=a)b"})), + super::super::HfTemplateError::Unsupported { .. } + )); + } +} diff --git a/rust/src/parser/src/unified/hf/fixtures/differential.json b/rust/src/parser/src/unified/hf/fixtures/differential.json new file mode 100644 index 0000000000..9b24cb295f --- /dev/null +++ b/rust/src/parser/src/unified/hf/fixtures/differential.json @@ -0,0 +1,440 @@ +{ + "templates": { + "gemma4": { + "defaults": { + "role": "assistant" + }, + "start_anchor": [ + "<|turn>model\n", + "" + ], + "fields": { + "thinking": { + "open": "<|channel>thought\n", + "close": "", + "content": "text" + }, + "tool_calls": { + "open_pattern": "<\\|tool_call>call:(?P\\w+)", + "close": "", + "repeats": true, + "content": "json", + "content_args": { + "unquoted_keys": true, + "string_delims": [ + [ + "<|\"|>", + "<|\"|>" + ] + ] + }, + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}" + } + } + }, + "content": { + "close": [ + "", + "<|tool_response>", + "" + ], + "content": "text" + } + } + }, + "invoke": { + "defaults": { + "role": "assistant" + }, + "start_anchor": "<|begin|>assistant", + "fields": { + "reasoning_content": { + "open_pattern": "to=self<\\|msg\\|>", + "close": "<|pause|>", + "content": "text" + }, + "tool_calls": { + "open_pattern": "]*?\\bname=\"(?P[^\"]+)\">", + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": "]*?\\bname=\"(?P[^\"]+)\"[^>]*?>(?P.*?)", + "value_parser": { + "name": "json", + "args": { + "allow_non_json": true + } + } + }, + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}" + } + } + }, + "content": { + "open_pattern": "to=(?:user|note)<\\|msg\\|>", + "close": [ + "<|end|>", + "<|pause|>" + ], + "repeats": true, + "join": "", + "content": "text" + } + } + }, + "kv_tools": { + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": { + "open": "", + "close": "" + }, + "tool_calls": { + "open_pattern": "\\s*\\w+)>\\n", + "close": "", + "repeats": true, + "content": "kv-lines", + "transform": { + "type": "function", + "function": { + "name": "{name}", + "arguments": "{content}" + } + } + }, + "content": { + "close": "<|im_end|>" + } + } + } + }, + "cases": [ + { + "template": "gemma4", + "prefix": "", + "text": "Hello there.", + "tools": null, + "expected": { + "content": "Hello there." + } + }, + { + "template": "gemma4", + "prefix": "", + "text": " \n Hello\n\n", + "tools": null, + "expected": { + "content": "Hello" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|channel>thought\n plan\n\n\nAnswer: 天气很好 ", + "tools": null, + "expected": { + "reasoning": "plan", + "content": "Answer: 天气很好" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|channel>thought\nunterminated reasoning", + "tools": null, + "expected": { + "reasoning": "unterminated reasoning" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "Before <|tool_call>call:f{x:<|\"|>a, b<|\"|>,y:[1,2.5,-3],z:{w:true}} after", + "tools": null, + "expected": { + "content": "after", + "tool_calls": [ + { + "name": "f", + "arguments": { + "x": "a, b", + "y": [ + 1, + 2.5, + -3 + ], + "z": { + "w": true + } + } + } + ] + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|tool_call>call:f{}<|tool_call>call:g{n:1}", + "tools": null, + "expected": { + "tool_calls": [ + { + "name": "f", + "arguments": {} + }, + { + "name": "g", + "arguments": { + "n": 1 + } + } + ] + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|tool_call>call:f{a:<|\"|>unterminated", + "tools": null, + "expected": { + "error": "ValueError" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "text <|tool_call> not a call", + "tools": null, + "expected": { + "content": "text <|tool_call> not a call" + } + }, + { + "template": "gemma4", + "prefix": "", + "text": "<|tool_call>call:f{a:1}", + "tools": null, + "expected": { + "tool_calls": [ + { + "name": "f", + "arguments": { + "a": 1 + } + } + ] + } + }, + { + "template": "gemma4", + "prefix": "<|turn>user\nhi\n<|turn>model\n", + "text": "Hi!", + "tools": null, + "expected": { + "content": "Hi!" + } + }, + { + "template": "gemma4", + "prefix": "<|turn>model\n<|tool_call>call:f{}<|tool_response>response:f{}<|channel>thought\n", + "text": "Done.It is sunny.", + "tools": null, + "expected": { + "reasoning": "Done.", + "content": "It is sunny." + } + }, + { + "template": "gemma4", + "prefix": "<|turn>model\n<|channel>thought\n", + "text": "Plain answer", + "tools": null, + "expected": { + "content": "Plain answer" + } + }, + { + "template": "invoke", + "prefix": "", + "text": "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>", + "tools": null, + "expected": { + "reasoning": "think", + "content": "AB" + } + }, + { + "template": "invoke", + "prefix": "", + "text": "noise \"Paris\"3not json", + "tools": null, + "expected": {} + }, + { + "template": "invoke", + "prefix": "", + "text": "12", + "tools": null, + "expected": { + "tool_calls": [ + { + "name": "get", + "arguments": { + "k": 2 + } + } + ] + } + }, + { + "template": "invoke", + "prefix": "", + "text": "bare text only", + "tools": null, + "expected": {} + }, + { + "template": "invoke", + "prefix": "", + "text": "to=user<|msg|>unterminated content", + "tools": null, + "expected": { + "content": "unterminated content" + } + }, + { + "template": "invoke", + "prefix": "<|begin|>user<|end|><|begin|>assistant", + "text": "to=user<|msg|>hi<|end|>", + "tools": null, + "expected": { + "content": "hi" + } + }, + { + "template": "kv_tools", + "prefix": "", + "text": "\n\nhour: 7\nratio: 0.5\nenabled: 1\nlabel: 7\nmaybe: None\n", + "tools": [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "ratio": { + "type": "number" + }, + "enabled": { + "type": "boolean" + }, + "label": { + "type": "string" + }, + "maybe": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + } + } + } + } + ], + "expected": { + "tool_calls": [ + { + "name": "set_alarm", + "arguments": { + "hour": 7, + "ratio": 0.5, + "enabled": true, + "label": "7", + "maybe": null + } + } + ] + } + }, + { + "template": "kv_tools", + "prefix": "", + "text": "\n\nhour: seven\nratio: 1e3\nenabled: nope\n", + "tools": [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": { + "type": "integer" + }, + "ratio": { + "type": "number" + }, + "enabled": { + "type": "boolean" + }, + "label": { + "type": "string" + }, + "maybe": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + } + } + } + } + ], + "expected": { + "tool_calls": [ + { + "name": "set_alarm", + "arguments": { + "hour": "seven", + "ratio": 1000, + "enabled": "nope" + } + } + ] + } + }, + { + "template": "kv_tools", + "prefix": "<|im_start|>assistant\n\n", + "text": "\n reasoning \n\n\nanswer<|im_end|>", + "tools": null, + "expected": { + "reasoning": "reasoning", + "content": "answer" + } + } + ] +} diff --git a/rust/src/parser/src/unified/hf/fixtures/generate_differential.py b/rust/src/parser/src/unified/hf/fixtures/generate_differential.py new file mode 100644 index 0000000000..35df98e083 --- /dev/null +++ b/rust/src/parser/src/unified/hf/fixtures/generate_differential.py @@ -0,0 +1,264 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Regenerate `differential.json` from Transformers `parse_response`. + +Requires Transformers with `transformers.utils.chat_parsing` (>= 5.13); the +expected values in the checked-in file were produced at commit 6d43ab4008. + + python generate_differential.py > differential.json +""" + +import json + +from transformers.utils.chat_parsing import parse_response + +GEMMA4 = { + "defaults": {"role": "assistant"}, + "start_anchor": ["<|turn>model\n", ""], + "fields": { + "thinking": { + "open": "<|channel>thought\n", + "close": "", + "content": "text", + }, + "tool_calls": { + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "close": "", + "repeats": True, + "content": "json", + "content_args": { + "unquoted_keys": True, + "string_delims": [['<|"|>', '<|"|>']], + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + "content": { + "close": ["", "<|tool_response>", ""], + "content": "text", + }, + }, +} + +INVOKE = { + "defaults": {"role": "assistant"}, + "start_anchor": "<|begin|>assistant", + "fields": { + "reasoning_content": { + "open_pattern": r"to=self<\|msg\|>", + "close": "<|pause|>", + "content": "text", + }, + "tool_calls": { + "open_pattern": r']*?\bname="(?P[^"]+)">', + "close": "", + "repeats": True, + "content": "xml-inline", + "content_args": { + "tag_pattern": ( + r']*?\bname="(?P[^"]+)"[^>]*?>' + r"(?P.*?)" + ), + "value_parser": {"name": "json", "args": {"allow_non_json": True}}, + }, + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + "content": { + "open_pattern": r"to=(?:user|note)<\|msg\|>", + "close": ["<|end|>", "<|pause|>"], + "repeats": True, + "join": "", + "content": "text", + }, + }, +} + +KV_TOOLS = { + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": ""}, + "tool_calls": { + "open_pattern": r"\s*\w+)>\n", + "close": "", + "repeats": True, + "content": "kv-lines", + "transform": { + "type": "function", + "function": {"name": "{name}", "arguments": "{content}"}, + }, + }, + "content": {"close": "<|im_end|>"}, + }, +} + +ALARM_TOOLS = [ + { + "type": "function", + "function": { + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": {"type": "integer"}, + "ratio": {"type": "number"}, + "enabled": {"type": "boolean"}, + "label": {"type": "string"}, + "maybe": {"anyOf": [{"type": "integer"}, {"type": "null"}]}, + }, + }, + }, + } +] + +CASES = [ + ("gemma4", GEMMA4, "", "Hello there.", None), + ("gemma4", GEMMA4, "", " \n Hello\n\n", None), + ( + "gemma4", + GEMMA4, + "", + "<|channel>thought\n plan\n\n\nAnswer: 天气很好 ", + None, + ), + ("gemma4", GEMMA4, "", "<|channel>thought\nunterminated reasoning", None), + ( + "gemma4", + GEMMA4, + "", + 'Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}}' + " after", + None, + ), + ( + "gemma4", + GEMMA4, + "", + "<|tool_call>call:f{}<|tool_call>call:g{n:1}", + None, + ), + ("gemma4", GEMMA4, "", '<|tool_call>call:f{a:<|"|>unterminated', None), + ("gemma4", GEMMA4, "", "text <|tool_call> not a call", None), + ("gemma4", GEMMA4, "", "<|tool_call>call:f{a:1}", None), + ("gemma4", GEMMA4, "<|turn>user\nhi\n<|turn>model\n", "Hi!", None), + ( + "gemma4", + GEMMA4, + "<|turn>model\n<|tool_call>call:f{}<|tool_response>response:f{}<|channel>thought\n", + "Done.It is sunny.", + None, + ), + ( + "gemma4", + GEMMA4, + "<|turn>model\n<|channel>thought\n", + "Plain answer", + None, + ), + ( + "invoke", + INVOKE, + "", + "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>", + None, + ), + ( + "invoke", + INVOKE, + "", + 'noise "Paris"' + '3not json', + None, + ), + ( + "invoke", + INVOKE, + "", + '1' + '2', + None, + ), + ("invoke", INVOKE, "", "bare text only", None), + ("invoke", INVOKE, "", "to=user<|msg|>unterminated content", None), + ( + "invoke", + INVOKE, + "<|begin|>user<|end|><|begin|>assistant", + "to=user<|msg|>hi<|end|>", + None, + ), + ( + "kv_tools", + KV_TOOLS, + "", + "\n\nhour: 7\nratio: 0.5\nenabled: 1\n" + "label: 7\nmaybe: None\n", + ALARM_TOOLS, + ), + ( + "kv_tools", + KV_TOOLS, + "", + "\n\nhour: seven\nratio: 1e3\n" + "enabled: nope\n", + ALARM_TOOLS, + ), + ( + "kv_tools", + KV_TOOLS, + "<|im_start|>assistant\n\n", + "\n reasoning \n\n\nanswer<|im_end|>", + None, + ), +] + + +def normalize(message): + out = {} + reasoning = message.get("thinking", message.get("reasoning_content")) + if reasoning: + out["reasoning"] = reasoning + if message.get("content"): + out["content"] = message["content"] + calls = [ + {"name": call["function"]["name"], "arguments": call["function"]["arguments"]} + for call in message.get("tool_calls", []) + ] + if calls: + out["tool_calls"] = calls + return out + + +def main(): + templates = {} + cases = [] + for template_name, template, prefix, text, tools in CASES: + templates[template_name] = template + try: + expected = normalize( + parse_response(text, template, prefix=prefix, tools=tools) + ) + except Exception as error: # noqa: BLE001 + expected = {"error": type(error).__name__} + cases.append( + { + "template": template_name, + "prefix": prefix, + "text": text, + "tools": tools, + "expected": expected, + } + ) + print( + json.dumps( + {"templates": templates, "cases": cases}, indent=2, ensure_ascii=False + ) + ) + + +if __name__ == "__main__": + main() diff --git a/rust/src/parser/src/unified/hf/mod.rs b/rust/src/parser/src/unified/hf/mod.rs new file mode 100644 index 0000000000..568a8371d0 --- /dev/null +++ b/rust/src/parser/src/unified/hf/mod.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Unified parser driven by a checkpoint's Hugging Face `response_template`. +//! +//! The renderer executes a checkpoint's `chat_template`; this parser executes its +//! `response_template`, the declarative output-protocol description introduced by +//! Transformers `chat_parsing` (`ResponseParser`). +//! +//! The template language and its value semantics (content parsers, transforms, +//! tool-schema coercion) follow Transformers at +//! . +//! The streaming executor is native: it emits ordered [`UnifiedParserEvent`]s +//! instead of Transformers' region events and aggregated message dict. +//! +//! [`UnifiedParserEvent`]: super::UnifiedParserEvent + +mod content; +mod parser; +mod pattern; +mod spec; +mod template; +mod transform; + +#[cfg(test)] +mod tests; + +pub use parser::HfUnifiedParser; +pub use template::ResponseTemplate; +use thiserror::Error; +use thiserror_ext::Macro; + +/// Result alias for response-template operations. +pub type Result = std::result::Result; + +/// Errors produced while loading a `response_template`. +/// +/// Failures while parsing generated output are +/// [`UnifiedParserError::ParsingFailed`](super::UnifiedParserError::ParsingFailed), as for +/// the other unified parsers. +#[derive(Debug, Clone, Error, Macro)] +#[thiserror_ext(macro(path = "crate::unified::hf", mangle))] +pub enum HfTemplateError { + /// The template violates the Transformers `response_template` schema. + #[error("invalid response_template: {message}")] + Invalid { message: String }, + /// The template is valid but uses a feature this implementation does not support. + #[error("unsupported response_template: {message}")] + Unsupported { message: String }, + /// The model provides no template. + #[error("the model's tokenizer_config.json provides no response_template")] + Missing, +} diff --git a/rust/src/parser/src/unified/hf/parser.rs b/rust/src/parser/src/unified/hf/parser.rs new file mode 100644 index 0000000000..f523c0f31a --- /dev/null +++ b/rust/src/parser/src/unified/hf/parser.rs @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Streaming executor for a compiled [`ResponseTemplate`]. +//! +//! Follows `ResponseParser` in `response_parser.py`: a flat region state +//! machine that, outside explicit regions, watches every explicit open plus the +//! implicit region's close, and inside an explicit region watches only that +//! region's close. Instead of region events and an aggregated dict, it emits +//! ordered unified parser events: +//! +//! - text and reasoning stream as they arrive, with `strip` applied so that the +//! concatenated deltas equal Transformers' stripped value. For a field without +//! `repeats`, stripping spans all its occurrences: an interrupted field (text, +//! tool call, more text) streams every occurrence, where Transformers keeps only +//! the last; +//! - a tool call starts at its opener when the opener captures the function +//! name, and its arguments are emitted when the region closes. + +use std::mem::take; +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::{Map, Value}; +use thiserror_ext::AsReport as _; +use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution}; +use winnow::error::{ErrMode, ModalResult}; +use winnow::stream::{Partial, Stream}; + +use super::pattern::{Captures, Resolution}; +use super::template::{ + FieldName, Region, RegionKind, Repeat, ResponseTemplate, TextRegion, TextRole, ToolCallRegion, + Watch, WatchKind, WatchSet, +}; +use crate::tool::{Tool, ToolCallDelta, ToolSchemas}; +use crate::unified::{ + Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput, parsing_failed, +}; +use crate::utils::{incomplete, parse_buffered_event, safe_text_len_mul}; + +/// Prompt tail windows, in tokens, searched for the start anchor before +/// decoding the whole prompt. +const PROMPT_WINDOWS: [usize; 3] = [64, 512, 4096]; + +/// Unified parser executing a checkpoint's `response_template`. +/// +/// Original Python implementation: +/// +pub struct HfUnifiedParser { + template: Arc, + tool_schemas: ToolSchemas, + tokenizer: DynTokenizer, + buffer: DecodedText, + mode: HfMode, + /// Whether each region has closed an occurrence. + closed: Vec, + /// Strip state per region: per occurrence with `repeats`, per stream otherwise. + strip: Vec, + emitted_tool_count: usize, +} + +/// Parser mode: the region currently receiving text. +enum HfMode { + /// No implicit field is declared: text outside regions is discarded. + Discard, + /// The implicit region, opened lazily on its first byte. + Implicit(Occurrence), + /// An explicit region, opened by its open boundary. + Explicit(Occurrence), +} + +/// One occurrence of a region. +struct Occurrence { + region: usize, + /// Whether the occurrence has content to close (`_opened`). + opened: bool, + captures: Captures, + /// Opener text, returned by [`UnifiedParser::reset`]. + raw_open: String, + /// Buffered body of a tool-call region. + body: String, + /// Index of the tool call started at the opener, if any. + active_tool_index: Option, + /// `join` separator still to emit before this occurrence's content. + separator: Option, +} + +/// Strip-compatible streaming state of a text or reasoning occurrence. +#[derive(Default)] +struct StripState { + enabled: bool, + /// Whether a non-whitespace character has been seen. + started: bool, + /// Trailing whitespace held until more text arrives or the region closes. + held: DecodedText, +} + +/// One parsed event of the buffered input. +enum HfEvent { + /// Text routed into the current region. + Text, + /// A boundary of the current mode's watch list. + Boundary { watch: Watch, captures: Captures }, +} + +/// A committable boundary match at the current position. +struct BoundaryMatch { + watch: Watch, + len: usize, + captures: Captures, +} + +impl BoundaryMatch { + /// Preference order of `_scan`: longest first, opens before closes, then by + /// field name. + fn key(&self, template: &ResponseTemplate) -> (std::cmp::Reverse, bool, FieldName) { + ( + std::cmp::Reverse(self.len), + self.watch.kind == WatchKind::Close, + template.regions[self.watch.region].name, + ) + } +} + +/// How the event loop treats its input. +#[derive(Clone, Copy)] +enum Phase { + /// Prompt text after the start anchor: state transitions only, no events. + Prefill, + /// Generated text; more may follow. + Stream, + /// Generated text at the end of the stream. + Eof, +} + +impl Phase { + fn emits(self) -> bool { + !matches!(self, Self::Prefill) + } +} + +impl HfUnifiedParser { + /// Create a parser for one request. + pub fn new(template: Arc, tools: &[Tool], tokenizer: DynTokenizer) -> Self { + let mut parser = Self { + closed: vec![false; template.regions.len()], + strip: template.regions.iter().map(StripState::new).collect(), + mode: HfMode::Discard, + template, + tool_schemas: ToolSchemas::from_tools(tools), + tokenizer, + buffer: DecodedText::default(), + emitted_tool_count: 0, + }; + parser.reset_to_implicit(); + parser + } + + /// Clear all per-stream state. + fn reset_state(&mut self) { + self.buffer.clear(); + self.closed.fill(false); + self.strip = self.template.regions.iter().map(StripState::new).collect(); + self.emitted_tool_count = 0; + self.reset_to_implicit(); + } + + /// Return to the implicit region, or discard text when there is none. + fn reset_to_implicit(&mut self) { + self.mode = match self.template.implicit { + Some(region) => { + HfMode::Implicit(self.occurrence(region, false, Captures::new(), String::new())) + } + None => HfMode::Discard, + }; + } + + /// Start a new occurrence of `region`. + fn occurrence( + &mut self, + region: usize, + opened: bool, + captures: Captures, + raw_open: String, + ) -> Occurrence { + let spec = &self.template.regions[region]; + let mut separator = None; + if let RegionKind::Text(text) = &spec.kind { + if let Repeat::Join(join) = &text.repeat + && !join.is_empty() + && self.closed[region] + { + separator = Some(join.clone()); + } + if text.repeat != Repeat::Once { + self.strip[region] = StripState::new(spec); + } + } + Occurrence { + region, + opened, + captures, + raw_open, + body: String::new(), + active_tool_index: None, + separator, + } + } + + /// Parse buffered input until more is needed. + fn drive(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> { + let template = Arc::clone(&self.template); + loop { + let watch = match &self.mode { + HfMode::Explicit(occurrence) => &template.regions[occurrence.region].close_watch, + HfMode::Discard | HfMode::Implicit(_) => &template.idle_watch, + }; + let eof = matches!(phase, Phase::Eof); + let Some((step, consumed_len)) = parse_buffered_event(&self.buffer.text, |input| { + parse_next_event(input, &template, watch, eof) + })? + else { + return Ok(()); + }; + let piece = self.buffer.drain_prefix(consumed_len); + match step { + HfEvent::Text => self.route(piece, output, phase), + HfEvent::Boundary { watch, captures } => { + self.close_current(output, phase)?; + match watch.kind { + WatchKind::Open => { + self.open_explicit(watch.region, captures, piece.text, output, phase) + } + // The implicit region's close: start a fresh implicit occurrence. + WatchKind::Close => {} + } + } + } + } + } + + /// Route a text piece into the current region. + fn route(&mut self, piece: DecodedText, output: &mut UnifiedParserOutput, phase: Phase) { + let (occurrence, region) = match &mut self.mode { + HfMode::Discard => return, + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) => { + let region = &self.template.regions[occurrence.region]; + (occurrence, region) + } + }; + let RegionKind::Text(text) = ®ion.kind else { + if !piece.text.is_empty() { + occurrence.opened = true; + occurrence.body.push_str(&piece.text); + } + return; + }; + let reasoning = text.role == TextRole::Reasoning; + if piece.text.is_empty() { + // Zero-width tokens still count as reasoning. + if reasoning && phase.emits() { + output.push_reasoning(piece); + } + return; + } + occurrence.opened = true; + + let strip = &mut self.strip[occurrence.region]; + if !phase.emits() { + // Prompt text is never emitted; it only decides whether later + // leading whitespace is still stripped. + strip.started |= !piece.text.trim_start().is_empty(); + return; + } + let (visible, dropped) = strip.feed(piece); + if reasoning { + push_dropped_reasoning(output, dropped); + } + if visible.text.is_empty() { + return; + } + if let Some(separator) = occurrence.separator.take() { + push_text(output, text.role, DecodedText::unattributed(separator)); + } + push_text(output, text.role, visible); + } + + /// Open an explicit region. + fn open_explicit( + &mut self, + region: usize, + captures: Captures, + raw_open: String, + output: &mut UnifiedParserOutput, + phase: Phase, + ) { + let mut occurrence = self.occurrence(region, true, captures, raw_open); + let opener_name = match &self.template.regions[region].kind { + RegionKind::ToolCalls(ToolCallRegion { + name_group: Some(group), + .. + }) => occurrence.captures.get(group).and_then(Value::as_str).map(str::to_string), + _ => None, + }; + if let Some(name) = opener_name + && phase.emits() + { + let tool_index = self.allocate_tool_index(); + occurrence.active_tool_index = Some(tool_index); + output.push_call(ToolCallDelta { + tool_index, + name: Some(name), + arguments: String::new(), + }); + } + self.mode = HfMode::Explicit(occurrence); + } + + /// Close the current region and reset to the implicit region. + /// + /// Skipped (aside from the reset) when the current region never opened -- + /// avoids vacuous open/close pairs at every explicit boundary. + fn close_current(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> { + let template = Arc::clone(&self.template); + let mode = std::mem::replace(&mut self.mode, HfMode::Discard); + let occurrence = match mode { + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => { + Some(occurrence) + } + _ => None, + }; + // Without `repeats`, trailing whitespace stays held in case the field + // continues; it is dropped at the end of the stream. + let held = occurrence + .as_ref() + .filter(|occurrence| { + matches!(&template.regions[occurrence.region].kind, + RegionKind::Text(text) if text.repeat != Repeat::Once) + }) + .map(|occurrence| take(&mut self.strip[occurrence.region].held)); + if let Some(occurrence) = &occurrence { + self.closed[occurrence.region] = true; + } + self.reset_to_implicit(); + // Regions closed inside the prompt contribute nothing to the output. + let Some(occurrence) = occurrence.filter(|_| phase.emits()) else { + return Ok(()); + }; + + let region = &template.regions[occurrence.region]; + match ®ion.kind { + RegionKind::Text(text) => { + if text.role == TextRole::Reasoning + && let Some(held) = held + { + push_dropped_reasoning(output, held); + } + // An empty occurrence still joins: `previous + join + ""`. + if let Some(separator) = occurrence.separator { + push_text(output, text.role, DecodedText::unattributed(separator)); + } + } + RegionKind::ToolCalls(calls) => { + let parsed = calls.content.parse(&occurrence.body)?; + let value = match &calls.transform { + Some(transform) => { + transform.apply(region.name.as_str(), parsed, &occurrence.captures)? + } + None => parsed, + }; + self.emit_calls(value, occurrence.active_tool_index, output)?; + } + } + Ok(()) + } + + /// Emit the tool calls of a closed tool-call region. + // TODO: stream arguments incrementally instead of once at region close; the + // dialects (e.g. Gemma 4's unquoted keys and `<|"|>` strings, `xml-inline`) + // need incremental conversion rather than raw passthrough. + fn emit_calls( + &mut self, + value: Value, + started: Option, + output: &mut UnifiedParserOutput, + ) -> Result<()> { + let calls = match value { + Value::Array(calls) => calls, + call => vec![call], + }; + for (position, call) in calls.into_iter().enumerate() { + let (name, arguments) = call_parts(call, &self.tool_schemas)?; + let (tool_index, name) = match (position, started) { + (0, Some(tool_index)) => (tool_index, None), + _ => (self.allocate_tool_index(), Some(name)), + }; + output.push_call(ToolCallDelta { + tool_index, + name, + arguments, + }); + } + Ok(()) + } + + fn allocate_tool_index(&mut self) -> usize { + let tool_index = self.emitted_tool_count; + self.emitted_tool_count += 1; + tool_index + } + + /// Return the prompt text after the last start-anchor match, if any. + /// + /// Searches geometrically growing tail windows before decoding the whole + /// prompt. A decoded suffix can differ from the full decode only in its first + /// character, so a match is accepted only after that character unless the + /// window is the whole prompt. + fn prompt_remainder(&self, prompt_token_ids: &[u32]) -> Result> { + let len = prompt_token_ids.len(); + let windows = PROMPT_WINDOWS.into_iter().filter(|window| *window < len).chain([len]); + for window in windows { + let text = self.tokenizer.decode(&prompt_token_ids[len - window..], false).map_err( + |error| parsing_failed!("failed to decode prompt: {}", error.as_report()), + )?; + let whole = window == len; + if let Some((start, end)) = self.template.last_start_anchor(&text) { + let first_char = text.chars().next().map_or(0, char::len_utf8); + if whole || start >= first_char { + return Ok(Some(text[end..].to_string())); + } + } + } + Ok(None) + } +} + +impl StripState { + fn new(region: &Region) -> Self { + Self { + enabled: matches!( + region.kind, + RegionKind::Text(TextRegion { strip: true, .. }) + ), + ..Self::default() + } + } + + /// Split a piece into its visible part and whitespace dropped by `strip`. + /// + /// Leading whitespace is dropped until the first non-whitespace character; + /// trailing whitespace is held until more non-whitespace text arrives (then + /// released) or the region closes (then dropped). + fn feed(&mut self, mut piece: DecodedText) -> (DecodedText, DecodedText) { + if !self.enabled { + return (piece, DecodedText::default()); + } + let mut dropped = DecodedText::default(); + if !self.started { + let lead = piece.text.len() - piece.text.trim_start().len(); + if lead == piece.text.len() { + return (DecodedText::default(), piece); + } + dropped = piece.drain_prefix(lead); + self.started = true; + } + let body_len = piece.text.trim_end().len(); + if body_len == 0 { + self.held.append(piece); + return (DecodedText::default(), dropped); + } + let head = piece.drain_prefix(body_len); + let mut visible = take(&mut self.held); + visible.append(head); + self.held = piece; + (visible, dropped) + } +} + +/// Push visible text of a text region. +fn push_text(output: &mut UnifiedParserOutput, role: TextRole, piece: DecodedText) { + match role { + TextRole::Content => output.push_text(piece.text), + TextRole::Reasoning => output.push_reasoning(piece), + } +} + +/// Keep the tokens of stripped reasoning whitespace in the reasoning count. +fn push_dropped_reasoning(output: &mut UnifiedParserOutput, dropped: DecodedText) { + if dropped.attributions.is_empty() { + return; + } + output.push_reasoning(DecodedText { + text: String::new(), + attributions: dropped + .attributions + .into_iter() + .map(|attribution| TokenAttribution { + token_id: attribution.token_id, + anchor: TokenAnchor::ZeroWidth { byte_offset: 0 }, + }) + .collect(), + }); +} + +/// One tool call produced by a tool-call region, after its transform. +#[derive(Deserialize)] +#[serde(untagged)] +enum ToolCallValue { + /// The OpenAI shape `{"type": "function", "function": {...}}`. + Wrapped { function: FunctionValue }, + /// The bare `{"name", "arguments"}` shape. + Bare(FunctionValue), +} + +#[derive(Deserialize)] +struct FunctionValue { + name: String, + /// Missing or `null` arguments become `{}`. + #[serde(default)] + arguments: Option, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum ArgumentsValue { + /// An argument object, typed by the tool schema. + Object(Map), + /// Pre-serialized or non-JSON arguments (e.g. an `allow_non_json` + /// fallback), passed through unchanged. + Text(String), + /// Any other JSON value, serialized as is. + Other(Value), +} + +/// Extract the function name and serialized arguments of one tool-call value. +/// +/// String argument values are converted by the calling tool's parameter +/// schema, as for the other tool parsers. +fn call_parts(call: Value, tool_schemas: &ToolSchemas) -> Result<(String, String)> { + let (ToolCallValue::Wrapped { function } | ToolCallValue::Bare(function)) = + ToolCallValue::deserialize(&call).map_err(|_| { + parsing_failed!( + "tool call must be {{\"function\": {{\"name\": ..., \"arguments\": ...}}}} or \ + {{\"name\": ..., \"arguments\": ...}} with a string name, got {call}" + ) + })?; + let FunctionValue { name, arguments } = function; + let arguments = match arguments { + None => "{}".to_string(), + Some(ArgumentsValue::Text(arguments)) => arguments, + Some(ArgumentsValue::Object(mut arguments)) => { + convert_text_arguments(tool_schemas, &name, &mut arguments); + serialize_arguments(&arguments)? + } + Some(ArgumentsValue::Other(arguments)) => serialize_arguments(&arguments)?, + }; + Ok((name, arguments)) +} + +/// Convert the argument values a content parser left as raw text by the tool +/// schema: top-level strings, and string lists collected from duplicate keys +/// (`merge_duplicates`). +/// +/// Deeper values are the content parser's own JSON and are kept, as in +/// Transformers' `_coerce_tool_calls`. +fn convert_text_arguments( + tool_schemas: &ToolSchemas, + function_name: &str, + arguments: &mut Map, +) { + for (name, value) in arguments.iter_mut() { + let values = match value { + Value::Array(items) => items.as_mut_slice(), + value => std::slice::from_mut(value), + }; + for value in values { + if let Value::String(text) = value { + *value = tool_schemas.convert_param_with_schema(function_name, name, take(text)); + } + } + } +} + +/// Serialize tool-call arguments to JSON text. +fn serialize_arguments(arguments: &impl serde::Serialize) -> Result { + serde_json::to_string(arguments).map_err(|error| { + parsing_failed!("failed to serialize tool arguments: {}", error.as_report()) + }) +} + +/// Parse one event: safe text before the next marker, or a boundary at it. +fn parse_next_event( + input: &mut Partial<&str>, + template: &ResponseTemplate, + watch: &WatchSet, + eof: bool, +) -> ModalResult { + let markers: Vec<&str> = watch.markers.iter().map(String::as_str).collect(); + match safe_text_len_mul(input, &markers) { + Ok(_) => return Ok(HfEvent::Text), + // A marker, complete or partial, starts here (or the input is empty). + Err(ErrMode::Incomplete(_)) => {} + Err(error) => return Err(error), + } + + // Any pending boundary blocks the position; otherwise the preferred match + // wins. At the end of the stream, a partial marker matches nothing and is + // released as text one character at a time. + let text = **input; + let mut best: Option = None; + for &watch in &watch.boundaries { + match template.boundary(watch).resolve(text, eof) { + Resolution::Pending => return incomplete(), + Resolution::NoMatch => {} + Resolution::Matched { len, captures } => { + let candidate = BoundaryMatch { + watch, + len, + captures, + }; + if best.as_ref().is_none_or(|best| candidate.key(template) < best.key(template)) { + best = Some(candidate); + } + } + } + } + + match best { + Some(BoundaryMatch { + watch, + len, + captures, + }) => { + input.next_slice(len); + Ok(HfEvent::Boundary { watch, captures }) + } + None => { + // Not a boundary: the marker's first character is plain text. + let len = text.chars().next().map_or(0, char::len_utf8); + input.next_slice(len); + Ok(HfEvent::Text) + } + } +} + +impl UnifiedParser for HfUnifiedParser { + fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Err(UnifiedParserError::NoNamedConstructor { + parser: "hf", + built_from: "the checkpoint's response_template", + }) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + self.reset_state(); + if prompt_token_ids.is_empty() { + return Ok(()); + } + let Some(remainder) = self.prompt_remainder(prompt_token_ids)? else { + self.template.warn_missing_anchor_once(); + return Ok(()); + }; + // A partial boundary at the end of the prompt stays buffered, so the + // generated text can complete it. + self.buffer = DecodedText::unattributed(remainder); + self.drive(&mut UnifiedParserOutput::default(), Phase::Prefill) + } + + fn preserve_special_tokens(&self) -> bool { + // Templates spell boundaries with special tokens and expect decoding + // with them kept. + true + } + + // TODO: derive a structural-tag output grammar from literal-bounded templates. + + fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> { + self.buffer.append(delta); + self.drive(output, Phase::Stream) + } + + fn finish(&mut self) -> Result { + let mut output = UnifiedParserOutput::default(); + self.drive(&mut output, Phase::Eof)?; + // Trailing zero-width tokens carry no text for the step loop to consume. + let rest = self.buffer.take(); + self.route(rest, &mut output, Phase::Eof); + self.close_current(&mut output, Phase::Eof)?; + for (region, strip) in self.template.regions.iter().zip(&mut self.strip) { + let held = take(&mut strip.held); + if matches!(®ion.kind, RegionKind::Text(text) if text.role == TextRole::Reasoning) { + push_dropped_reasoning(&mut output, held); + } + } + + let missing: Vec<_> = self + .template + .regions + .iter() + .zip(&self.closed) + .filter(|(region, closed)| !region.optional && !**closed) + .map(|(region, _)| region.name.as_str()) + .collect(); + if !missing.is_empty() { + return Err(parsing_failed!( + "Required response_template fields missing from parsed output: {missing:?}" + )); + } + + self.reset_state(); + Ok(output) + } + + fn reset(&mut self) -> String { + let mut raw = String::new(); + if let HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) = + std::mem::replace(&mut self.mode, HfMode::Discard) + { + raw.push_str(&occurrence.raw_open); + raw.push_str(&occurrence.body); + raw.push_str(&self.strip[occurrence.region].held.text); + } + raw.push_str(&self.buffer.take().text); + self.reset_state(); + raw + } +} diff --git a/rust/src/parser/src/unified/hf/pattern.rs b/rust/src/parser/src/unified/hf/pattern.rs new file mode 100644 index 0000000000..143545ca12 --- /dev/null +++ b/rust/src/parser/src/unified/hf/pattern.rs @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Streaming resolution of `open_pattern` / `close_pattern` boundaries. +//! +//! Transformers relies on the Python `regex` module's `partial=True` search to +//! decide whether a match at the end of the buffer is final, could still appear, +//! or is impossible. The Rust `regex` crate has no partial-match API, so this +//! module answers that question by stepping a `regex-automata` DFA, and uses a +//! `meta::Regex` for the actual match span and captures. + +use regex_automata::dfa::{Automaton, StartKind, dense}; +use regex_automata::meta::Regex; +use regex_automata::nfa::thompson::{self, WhichCaptures}; +use regex_automata::util::primitives::StateID; +use regex_automata::{Anchored, Input, MatchKind, PatternID}; +use regex_syntax::hir::literal::{ExtractKind, Extractor}; +use regex_syntax::hir::{Hir, HirKind}; +use serde_json::{Map, Value}; +use thiserror_ext::AsReport as _; + +use super::{Result, unsupported}; + +/// Upper bound for the memory of one boundary DFA. +const DFA_SIZE_LIMIT: usize = 16 << 20; + +/// Named capture groups of a boundary match, as string values: the scope a +/// transform template reads them from. +pub(super) type Captures = Map; + +/// Outcome of resolving a boundary at a candidate position. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Resolution { + /// The boundary matches `len` bytes; more input cannot change the match. + Matched { len: usize, captures: Captures }, + /// More input could still produce or change a match. + Pending, + /// No match starts here, whatever input follows. + NoMatch, +} + +/// A regex boundary that can be resolved against a growing buffer. +#[derive(Debug, Clone)] +pub(super) struct StreamingPattern { + /// Finite set of non-empty literals, one of which starts every match. + prefixes: Vec, + /// Anchored leftmost-first DFA of the pattern: decides whether the + /// leftmost-first match is final. Unicode word boundaries are handled + /// heuristically, quitting on non-ASCII bytes. + first: dense::DFA>, + /// Anchored DFA with every look-around assertion removed and + /// `MatchKind::All`, used when `first` quits. Relaxing an assertion only + /// enlarges the language, so once this DFA is dead no continuation can + /// produce a later-ending match of the real pattern. `MatchKind::All` is + /// required for that argument: leftmost-first pruning could discard the + /// thread the real pattern takes when an assertion fails (`a\b|ab` on `ab`). + fallback: Option>>, + /// The pattern itself, for the match span and named captures. + exact: Regex, +} + +/// Result of walking a DFA over the available bytes. +enum Walk { + /// No continuation can change the result. + Final, + /// More input could still change the result. + Alive, + /// The DFA gave up (non-ASCII byte near a Unicode word boundary). + Quit, +} + +impl StreamingPattern { + /// Compile a boundary pattern in Python `regex` syntax, with `re.DOTALL`. + /// + /// Streaming resolution only starts at occurrences of the pattern's literal + /// prefixes, so every match must start with one of a finite set of non-empty + /// literals. + pub fn new(scope: &str, pattern: &str) -> Result { + let hir = regex_syntax::ParserBuilder::new() + .dot_matches_new_line(true) + .build() + .parse(pattern) + .map_err(|error| { + unsupported!( + "{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}", + error.as_report() + ) + })?; + + let prefixes = literal_prefixes(&hir).ok_or_else(|| { + unsupported!( + "{scope}: pattern {pattern:?} must start with a literal prefix for streaming \ + matching (e.g. `` in `(?P\\w+)`)" + ) + })?; + + let first = build_dfa(scope, pattern, &hir, MatchKind::LeftmostFirst)?; + let fallback = hir + .properties() + .look_set() + .contains_word_unicode() + .then(|| build_dfa(scope, pattern, &relax_assertions(&hir), MatchKind::All)) + .transpose()?; + let exact = Regex::builder().build_from_hir(&hir).map_err(|error| { + unsupported!( + "{scope}: failed to compile regex {pattern:?}: {}", + error.as_report() + ) + })?; + + Ok(Self { + prefixes, + first, + fallback, + exact, + }) + } + + /// Literals, one of which starts every match. + pub fn prefixes(&self) -> &[String] { + &self.prefixes + } + + /// Named capture groups of the pattern. + pub fn group_names(&self) -> impl Iterator { + self.exact.group_info().pattern_names(PatternID::ZERO).flatten() + } + + /// Resolve a match starting at the beginning of `text`. + /// + /// `eof` means no more input will arrive. + pub fn resolve(&self, text: &str, eof: bool) -> Resolution { + let walk = match walk(&self.first, text) { + Walk::Quit => walk( + self.fallback.as_ref().expect("quits imply a fallback"), + text, + ), + walk => walk, + }; + if matches!(walk, Walk::Alive) && !eof { + return Resolution::Pending; + } + + let input = Input::new(text).anchored(Anchored::Yes); + let mut captures = self.exact.create_captures(); + self.exact.search_captures(&input, &mut captures); + let Some(found) = captures.get_match() else { + return Resolution::NoMatch; + }; + let names = self.exact.group_info().pattern_names(PatternID::ZERO); + let captures = names + .enumerate() + .filter_map(|(index, name)| { + let span = captures.get_group(index)?; + Some(( + name?.to_string(), + Value::String(text[span.range()].to_string()), + )) + }) + .collect(); + Resolution::Matched { + len: found.end(), + captures, + } + } +} + +/// Walk `dfa` from an anchored start over `text`. +/// +/// The walk is final once the DFA is dead, or once it is in a match state whose +/// transitions are all dead: that removes the one byte of holdback that delayed +/// DFA match reporting would otherwise add. A quit transition counts as live. +fn walk(dfa: &dense::DFA>, text: &str) -> Walk { + let input = Input::new(text).anchored(Anchored::Yes); + let Ok(mut state) = dfa.start_state_forward(&input) else { + return Walk::Quit; + }; + for byte in text.bytes() { + state = dfa.next_state(state, byte); + if dfa.is_quit_state(state) { + return Walk::Quit; + } + if dfa.is_dead_state(state) || is_terminal_match(dfa, state) { + return Walk::Final; + } + } + Walk::Alive +} + +/// Return whether `state` reports a match and cannot continue. +fn is_terminal_match(dfa: &dense::DFA>, state: StateID) -> bool { + dfa.is_match_state(state) + && (0..=u8::MAX).all(|byte| dfa.is_dead_state(dfa.next_state(state, byte))) +} + +/// Build an anchored dense DFA for `hir`. +fn build_dfa( + scope: &str, + pattern: &str, + hir: &Hir, + kind: MatchKind, +) -> Result>> { + let nfa = thompson::Compiler::new() + .configure(thompson::Config::new().which_captures(WhichCaptures::None)) + .build_from_hir(hir) + .map_err(|error| { + unsupported!( + "{scope}: failed to compile regex {pattern:?}: {}", + error.as_report() + ) + })?; + dense::Builder::new() + .configure( + dense::Config::new() + .match_kind(kind) + .start_kind(StartKind::Anchored) + .unicode_word_boundary(true) + .dfa_size_limit(Some(DFA_SIZE_LIMIT)) + .determinize_size_limit(Some(DFA_SIZE_LIMIT)), + ) + .build_from_nfa(&nfa) + .map_err(|error| { + unsupported!( + "{scope}: regex {pattern:?} cannot be matched incrementally: {}", + error.as_report() + ) + }) +} + +/// Replace every look-around assertion with the empty regex. +fn relax_assertions(hir: &Hir) -> Hir { + match hir.kind() { + HirKind::Look(_) => Hir::empty(), + HirKind::Repetition(repetition) => { + let mut repetition = repetition.clone(); + repetition.sub = Box::new(relax_assertions(&repetition.sub)); + Hir::repetition(repetition) + } + HirKind::Capture(capture) => { + let mut capture = capture.clone(); + capture.sub = Box::new(relax_assertions(&capture.sub)); + Hir::capture(capture) + } + HirKind::Concat(subs) => Hir::concat(subs.iter().map(relax_assertions).collect()), + HirKind::Alternation(subs) => Hir::alternation(subs.iter().map(relax_assertions).collect()), + HirKind::Empty | HirKind::Literal(_) | HirKind::Class(_) => hir.clone(), + } +} + +/// Extract the finite set of non-empty literal prefixes starting every match. +fn literal_prefixes(hir: &Hir) -> Option> { + let seq = Extractor::new().kind(ExtractKind::Prefix).extract(hir); + let mut prefixes: Vec = Vec::new(); + for literal in seq.literals()? { + let bytes = literal.as_bytes(); + // Truncated inexact literals may end inside a UTF-8 sequence; any valid + // prefix of a prefix is still a prefix. + let valid = match std::str::from_utf8(bytes) { + Ok(text) => text, + Err(error) => std::str::from_utf8(&bytes[..error.valid_up_to()]).ok()?, + }; + if valid.is_empty() { + return None; + } + if !prefixes.iter().any(|prefix| prefix == valid) { + prefixes.push(valid.to_string()); + } + } + (!prefixes.is_empty()).then_some(prefixes) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MUSE_INVOKE: &str = r#"]*?\bname="(?P[^"]+)">"#; + const GEMMA4_CALL: &str = r"<\|tool_call>call:(?P\w+)"; + const GPT_OSS_CALL: &str = + r"<\|channel\|>commentary to=functions\.(?P\w+).*?<\|message\|>"; + + fn pattern(source: &str) -> StreamingPattern { + StreamingPattern::new("test", source).unwrap() + } + + fn matched(len: usize, name: &str) -> Resolution { + Resolution::Matched { + len, + captures: Captures::from_iter([("name".to_string(), name.into())]), + } + } + + #[test] + fn extracts_literal_prefixes() { + assert_eq!(pattern(GEMMA4_CALL).prefixes(), ["<|tool_call>call:"]); + assert_eq!(pattern(MUSE_INVOKE).prefixes(), ["").prefixes(), + ["to=user<|message|>", "to=commentary<|message|>"] + ); + assert_eq!( + pattern(r"\n?").prefixes(), + ["\n", ""] + ); + for source in [r"(?:^|\s*)", r"(?:<\|m\|>)?[^<]*<\|c\|>", r".*x"] { + let result = StreamingPattern::new("test", source); + assert!( + matches!( + result, + Err(super::super::HfTemplateError::Unsupported { .. }) + ), + "{source}" + ); + } + } + + #[test] + fn word_boundaries_follow_unicode_semantics() { + let muse = pattern(MUSE_INVOKE); + assert_eq!( + muse.resolve(r#"rest"#, false), + matched(24, "get") + ); + let unicode_attr = r#"rest"#; + assert_eq!( + muse.resolve(unicode_attr, false), + matched(unicode_attr.len() - 4, "获取") + ); + assert_eq!( + muse.resolve(r#"r"#, false), + Resolution::NoMatch + ); + assert_eq!( + muse.resolve(r#"r"#, false), + Resolution::NoMatch + ); + assert_eq!( + muse.resolve(r#"r"#, false), + matched(25, "x") + ); + assert_eq!( + muse.resolve(r#"r"#, false), + Resolution::NoMatch + ); + assert_eq!( + muse.resolve(" 正文", false), + Resolution::NoMatch + ); + + let alternation = pattern(r"a\b|ab"); + assert_eq!( + alternation.resolve("ab!", true), + Resolution::Matched { + len: 2, + captures: Captures::new() + } + ); + } + + #[test] + fn greedy_and_lazy_tails_commit_like_transformers() { + let gemma4 = pattern(GEMMA4_CALL); + assert_eq!( + gemma4.resolve("<|tool_call>call:get_wea", false), + Resolution::Pending + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:get_weather{", false), + matched(28, "get_weather") + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:天气{", false), + matched(23, "天气") + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:{", false), + Resolution::NoMatch + ); + assert_eq!( + gemma4.resolve("<|tool_call>call:get", true), + matched(20, "get") + ); + + let gpt_oss = pattern(GPT_OSS_CALL); + let opener = "<|channel|>commentary to=functions.get_weather <|constrain|>json<|message|>"; + assert_eq!(gpt_oss.resolve(opener, false), Resolution::Pending); + assert_eq!( + gpt_oss.resolve(&format!("{opener}{{\"a\""), false), + matched(opener.len(), "get_weather") + ); + } +} diff --git a/rust/src/parser/src/unified/hf/spec.rs b/rust/src/parser/src/unified/hf/spec.rs new file mode 100644 index 0000000000..20738b95c7 --- /dev/null +++ b/rust/src/parser/src/unified/hf/spec.rs @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Serde mirror of the `response_template` JSON. +//! +//! Key sets and defaults follow `response_templates.py`; semantic validation +//! happens when compiling into [`super::ResponseTemplate`]. + +use serde::Deserialize; +use serde::de::DeserializeOwned; +use serde_json::{Map, Value}; +use serde_with::serde_as; +use thiserror_ext::AsReport as _; + +use super::{Result, invalid}; + +/// Top-level `response_template` object. +#[serde_as] +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TemplateSpec { + #[serde(default = "default_version")] + version: u64, + /// Constant keys of the Transformers output dict (e.g. `role`). Unused by the + /// event-based executor, but validated as part of the schema. + #[serde(default, rename = "defaults")] + _defaults: Map, + /// Field specs in declaration order. + #[serde_as(as = "serde_with::Map<_, _>")] + pub fields: Vec<(String, FieldSpec)>, + start_anchor: Option, + start_anchor_pattern: Option, +} + +/// One entry of `fields`. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FieldSpec { + open: Option, + open_pattern: Option, + close: Option, + close_pattern: Option, + #[serde(default)] + pub content: ContentKind, + /// Arguments of the `content` parser; their keys depend on the parser. + #[serde(default)] + pub content_args: Map, + #[serde(default)] + pub repeats: bool, + pub join: Option, + #[serde(default = "default_optional")] + pub optional: bool, + /// A free-form JSON template, compiled into a [`super::transform::Transform`]. + pub transform: Option, + #[serde(default)] + pub transform_each: bool, +} + +/// Name of a content parser. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub(super) enum ContentKind { + #[default] + Text, + Int, + Float, + Bool, + Json, + XmlInline, + KvLines, +} + +/// A literal anchor: one string, or a list of alternatives. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum Literals { + One(String), + Many(Vec), +} + +/// An anchor after resolving its literal-or-pattern form. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum AnchorSpec<'a> { + /// Deduplicated literals in first-seen order. + Literals(Vec), + /// A regex in Python `regex` syntax. + Pattern(&'a str), +} + +impl TemplateSpec { + /// Parse the top-level template object. + pub fn from_value(value: &Value) -> Result { + let spec: Self = deserialize(value, "response_template")?; + if spec.version != 1 { + return Err(invalid!( + "unsupported response_template version: {}", + spec.version + )); + } + if spec.fields.is_empty() { + return Err(invalid!( + "response_template.fields must be a non-empty dict" + )); + } + Ok(spec) + } + + /// The required start anchor. + pub fn start_anchor(&self) -> Result> { + anchor_spec( + "response_template", + (self.start_anchor.as_ref(), "start_anchor"), + (self.start_anchor_pattern.as_deref(), "start_anchor_pattern"), + )? + .ok_or_else(|| { + invalid!("response_template must define 'start_anchor' or 'start_anchor_pattern'.") + }) + } +} + +impl FieldSpec { + /// The open anchor; `None` for the implicit field. + pub fn open(&self, scope: &str) -> Result>> { + anchor_spec( + scope, + (self.open.as_ref(), "open"), + (self.open_pattern.as_deref(), "open_pattern"), + ) + } + + /// The close anchor; `None` when the field runs to the end of the stream. + pub fn close(&self, scope: &str) -> Result>> { + anchor_spec( + scope, + (self.close.as_ref(), "close"), + (self.close_pattern.as_deref(), "close_pattern"), + ) + } +} + +/// Deserialize `value`, reporting `context` and the JSON path of any error. +pub(super) fn deserialize(value: &Value, context: &str) -> Result { + serde_path_to_error::deserialize(value).map_err(|error| { + let path = error.path().to_string(); + let error = error.into_inner(); + if path == "." { + invalid!("{context}: {}", error.as_report()) + } else { + invalid!("{context}.{path}: {}", error.as_report()) + } + }) +} + +/// Resolve an anchor given in literal form or regex form. +/// +/// Mirrors `_compile_anchor`: both forms at once is an error, literal lists must be +/// non-empty and contain no empty strings, and duplicates are dropped in first-seen +/// order. +fn anchor_spec<'a>( + scope: &str, + (literals, literal_key): (Option<&Literals>, &str), + (pattern, pattern_key): (Option<&'a str>, &str), +) -> Result>> { + match (literals, pattern) { + (Some(_), Some(_)) => Err(invalid!( + "{scope}: cannot specify both '{literal_key}' and '{pattern_key}'" + )), + (Some(literals), None) => { + let raw = match literals { + Literals::One(literal) => std::slice::from_ref(literal), + Literals::Many(literals) => literals.as_slice(), + }; + if raw.is_empty() { + return Err(invalid!( + "{scope}: '{literal_key}' list must contain at least one literal" + )); + } + if raw.iter().any(String::is_empty) { + return Err(invalid!( + "{scope}: '{literal_key}' literals cannot be empty strings" + )); + } + let mut deduped: Vec = Vec::with_capacity(raw.len()); + for literal in raw { + if !deduped.contains(literal) { + deduped.push(literal.clone()); + } + } + Ok(Some(AnchorSpec::Literals(deduped))) + } + (None, Some(pattern)) => Ok(Some(AnchorSpec::Pattern(pattern))), + (None, None) => Ok(None), + } +} + +fn default_version() -> u64 { + 1 +} + +fn default_optional() -> bool { + true +} diff --git a/rust/src/parser/src/unified/hf/template.rs b/rust/src/parser/src/unified/hf/template.rs new file mode 100644 index 0000000000..4889f6f9b5 --- /dev/null +++ b/rust/src/parser/src/unified/hf/template.rs @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Compiled `response_template`. +//! +//! Validation follows `response_templates.py`; the compiled form additionally +//! maps fields to parser roles and precomputes each state's boundary markers. + +use std::fmt; +use std::sync::atomic::{AtomicBool, Ordering}; + +use regex_automata::meta::Regex; +use serde_json::Value; + +use super::content::{ContentParser, TextArgs, compile_regex}; +use super::pattern::{Captures, Resolution, StreamingPattern}; +use super::spec::{AnchorSpec, FieldSpec, TemplateSpec}; +use super::transform::{FieldTransform, Transform}; +use super::{Result, invalid, unsupported}; + +/// A compiled `response_template`, shared by all requests of one model. +#[derive(Debug)] +pub struct ResponseTemplate { + /// Matches the start of the current assistant turn in the prompt. + start_anchor: Regex, + pub(super) regions: Vec, + /// The field without an opener: the implicit-open / leftover sink. + pub(super) implicit: Option, + /// Boundaries watched outside explicit regions: every explicit open plus the + /// implicit region's close. + pub(super) idle_watch: WatchSet, + warned_missing_anchor: AtomicBool, +} + +/// One compiled field. +#[derive(Debug)] +pub(super) struct Region { + pub name: FieldName, + /// `None`: the implicit region. + pub open: Option, + /// `None`: the region runs to the end of the stream. + pub close: Option, + /// Boundaries watched inside this region: its close, if any. + pub close_watch: WatchSet, + pub optional: bool, + pub kind: RegionKind, +} + +/// A field this parser can report, in the lexical order of its name, which +/// Transformers uses to break ties between boundaries. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum FieldName { + Content, + ReasoningContent, + Thinking, + ToolCalls, +} + +/// How a region's content is reported. +#[derive(Debug)] +pub(super) enum RegionKind { + /// Streamed as it arrives. + Text(TextRegion), + /// Buffered, and parsed into tool calls at close. + ToolCalls(ToolCallRegion), +} + +#[derive(Debug)] +pub(super) struct TextRegion { + pub role: TextRole, + /// Whether the value is stripped (`content_args.strip`). + pub strip: bool, + pub repeat: Repeat, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum TextRole { + Content, + Reasoning, +} + +/// How repeated occurrences of a text field combine. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum Repeat { + /// No `repeats`: occurrences stream as one value. + Once, + /// `repeats` without `join`: each occurrence is its own value. + Each, + /// `repeats` with `join`: occurrences are joined by the separator. + Join(String), +} + +#[derive(Debug)] +pub(super) struct ToolCallRegion { + pub content: ContentParser, + pub transform: Option, + /// Opener capture group that provides the function name, allowing the tool + /// call to start before its arguments are complete. + pub name_group: Option, +} + +/// A region delimiter. +#[derive(Debug)] +pub(super) enum Boundary { + /// Literal alternatives, longest first. + Literals(Vec), + Pattern(Box), +} + +/// One boundary the executor watches for in a given state. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct Watch { + pub region: usize, + pub kind: WatchKind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum WatchKind { + Open, + Close, +} + +/// The boundaries watched in one parser mode. +#[derive(Debug, Default)] +pub(super) struct WatchSet { + pub boundaries: Vec, + /// Literals, deduplicated, whose occurrences are the only positions any of + /// the boundaries can match at. + pub markers: Vec, +} + +impl ResponseTemplate { + /// Compile the `response_template` object from `tokenizer_config.json`. + pub fn from_json(value: &Value) -> Result { + let spec = TemplateSpec::from_value(value)?; + + let regions = spec + .fields + .iter() + .enumerate() + .map(|(index, (name, field))| Region::compile(index, name, field)) + .collect::>>()?; + + // A field without an open anchor is the implicit-open / leftover sink; at most one is allowed. + let implicit_fields: Vec<_> = + regions.iter().enumerate().filter(|(_, region)| region.open.is_none()).collect(); + if implicit_fields.len() > 1 { + let names: Vec<_> = + implicit_fields.iter().map(|(_, region)| region.name.as_str()).collect(); + return Err(invalid!( + "At most one field may omit 'open'/'open_pattern' (that field becomes the \ + implicit-open / leftover sink). Found: {}", + names.join(", ") + )); + } + let implicit = implicit_fields.first().map(|(index, _)| *index); + + let start_anchor = match spec.start_anchor()? { + AnchorSpec::Literals(mut literals) => { + // Sort longest-first so alternation prefers the longer alternative when both could match. + literals.sort_by_key(|literal| std::cmp::Reverse(literal.len())); + let alternation: Vec<_> = + literals.iter().map(|literal| regex_syntax::escape(literal)).collect(); + compile_regex("response_template", &alternation.join("|"))? + } + AnchorSpec::Pattern(pattern) => compile_regex("response_template", pattern)?, + }; + + let opens = regions.iter().enumerate().filter_map(|(region, spec)| { + let watch = Watch { + region, + kind: WatchKind::Open, + }; + Some((watch, spec.open.as_ref()?)) + }); + let implicit_close = implicit.and_then(|region| { + let watch = Watch { + region, + kind: WatchKind::Close, + }; + Some((watch, regions[region].close.as_ref()?)) + }); + let idle_watch = WatchSet::new(opens.chain(implicit_close)); + + Ok(Self { + start_anchor, + regions, + implicit, + idle_watch, + warned_missing_anchor: AtomicBool::new(false), + }) + } + + /// Return the start and end offsets of the last start-anchor match in + /// `prompt`. + pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> { + self.start_anchor + .find_iter(prompt) + .last() + .map(|found| (found.start(), found.end())) + } + + /// Log once per template that the prompt carried no start anchor. + pub(super) fn warn_missing_anchor_once(&self) { + if !self.warned_missing_anchor.swap(true, Ordering::Relaxed) { + tracing::warn!( + "response_template start anchor not found in the prompt; parsing starts from the \ + template's initial state (the chat template may not match the checkpoint's \ + response_template)" + ); + } + } + + /// The boundary of a watch entry. + pub(super) fn boundary(&self, watch: Watch) -> &Boundary { + let region = &self.regions[watch.region]; + match watch.kind { + WatchKind::Open => region.open.as_ref(), + WatchKind::Close => region.close.as_ref(), + } + .expect("watched boundaries exist") + } +} + +impl Region { + /// Validate the spec of the field at `index` and compile it. + fn compile(index: usize, name: &str, field: &FieldSpec) -> Result { + let scope = format!("Field '{name}'"); + let open = field.open(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?; + let close = field.close(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?; + + let content = ContentParser::new(&scope, field.content, &field.content_args)?; + let repeat = match (field.repeats, &field.join) { + (false, None) => Repeat::Once, + (true, None) => Repeat::Each, + (true, Some(join)) => Repeat::Join(join.clone()), + (false, Some(_)) => return Err(invalid!("{scope}: 'join' requires 'repeats': true")), + }; + let transform = match (&field.transform, field.transform_each) { + (Some(template), each) => Some(FieldTransform { + template: Transform::compile(&scope, template)?, + each, + }), + (None, false) => None, + (None, true) => { + return Err(invalid!( + "{scope}: transform_each is set but no transform was provided" + )); + } + }; + + let captured: Vec<&str> = + [&open, &close].into_iter().flatten().flat_map(Boundary::group_names).collect(); + match &transform { + None if !captured.is_empty() => { + // Named captures only reach the output through a transform, so flag any that would be silently dropped. + return Err(invalid!( + "{scope}: open_pattern/close_pattern declares named group(s) {captured:?}, but \ + the field has no 'transform'. Named captures are only surfaced through a \ + 'transform' template (where they appear alongside 'content'). Either add a \ + 'transform' that uses the captures, or remove the named groups from the pattern." + )); + } + // Without `transform_each`, the scope is the opener's captures plus `content`; + // any other placeholder fails on every match. + Some(FieldTransform { + template, + each: false, + }) => { + let open_groups: Vec<&str> = open.iter().flat_map(Boundary::group_names).collect(); + if let Some(root) = template + .placeholder_roots() + .find(|root| *root != "content" && !open_groups.contains(root)) + { + return Err(invalid!( + "{scope}: transform placeholder '{{{root}}}' is neither 'content' nor a \ + named group of open_pattern" + )); + } + } + _ => {} + } + + let name = FieldName::parse(&scope, name)?; + let kind = match name.text_role() { + Some(role) => { + let strip = match (content, &transform) { + (ContentParser::Text(TextArgs { strip }), None) => strip, + _ => { + return Err(unsupported!( + "{scope}: text and reasoning fields must use the 'text' content \ + parser without a transform" + )); + } + }; + RegionKind::Text(TextRegion { + role, + strip, + repeat, + }) + } + None => { + if matches!(repeat, Repeat::Join(_)) { + return Err(unsupported!( + "{scope}: 'join' requires each match to parse to a string" + )); + } + let name_group = name_group(transform.as_ref(), open.as_ref()); + RegionKind::ToolCalls(ToolCallRegion { + content, + transform, + name_group, + }) + } + }; + + let close_watch = WatchSet::new(close.as_ref().map(|close| { + let watch = Watch { + region: index, + kind: WatchKind::Close, + }; + (watch, close) + })); + + Ok(Self { + name, + open, + close, + close_watch, + optional: field.optional, + kind, + }) + } +} + +/// The opener capture group that a non-`transform_each` transform uses as the +/// function name, if any. +fn name_group(transform: Option<&FieldTransform>, open: Option<&Boundary>) -> Option { + let ( + FieldTransform { + template, + each: false, + }, + open, + ) = (transform?, open?) + else { + return None; + }; + let path = template + .placeholder_at(&["function", "name"]) + .or_else(|| template.placeholder_at(&["name"]))?; + match path { + [root] if open.group_names().any(|group| group == root) => Some(root.clone()), + _ => None, + } +} + +impl FieldName { + /// Map a field name to a reportable field. + fn parse(scope: &str, name: &str) -> Result { + Ok(match name { + "content" => Self::Content, + "reasoning_content" => Self::ReasoningContent, + "thinking" => Self::Thinking, + "tool_calls" => Self::ToolCalls, + _ => { + return Err(unsupported!( + "{scope}: only 'content', 'reasoning_content'/'thinking', and 'tool_calls' \ + fields can be reported" + )); + } + }) + } + + /// The role of a text field; `None` for `tool_calls`. + fn text_role(self) -> Option { + match self { + Self::Content => Some(TextRole::Content), + Self::ReasoningContent | Self::Thinking => Some(TextRole::Reasoning), + Self::ToolCalls => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Content => "content", + Self::ReasoningContent => "reasoning_content", + Self::Thinking => "thinking", + Self::ToolCalls => "tool_calls", + } + } +} + +impl fmt::Display for FieldName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Boundary { + /// Compile an anchor into a boundary. + fn compile(scope: &str, spec: AnchorSpec<'_>) -> Result { + Ok(match spec { + AnchorSpec::Literals(mut literals) => { + // Sort longest-first so the longer alternative wins when both match. + literals.sort_by_key(|literal| std::cmp::Reverse(literal.len())); + Self::Literals(literals) + } + AnchorSpec::Pattern(pattern) => { + Self::Pattern(Box::new(StreamingPattern::new(scope, pattern)?)) + } + }) + } + + /// Literals whose occurrences are the only positions this boundary can match at. + pub fn markers(&self) -> &[String] { + match self { + Self::Literals(literals) => literals, + Self::Pattern(pattern) => pattern.prefixes(), + } + } + + /// Named capture groups. + pub fn group_names(&self) -> impl Iterator { + match self { + Self::Literals(_) => None, + Self::Pattern(pattern) => Some(pattern.group_names()), + } + .into_iter() + .flatten() + } + + /// Resolve this boundary at the start of `text`. + pub fn resolve(&self, text: &str, eof: bool) -> Resolution { + let markers = self.markers(); + // A marker the available text is a proper prefix of: more input decides. + let partial = !eof + && markers + .iter() + .any(|marker| marker.len() > text.len() && marker.starts_with(text)); + match self { + Self::Literals(literals) => { + if partial { + return Resolution::Pending; + } + match literals.iter().find(|literal| text.starts_with(literal.as_str())) { + Some(literal) => Resolution::Matched { + len: literal.len(), + captures: Captures::new(), + }, + None => Resolution::NoMatch, + } + } + Self::Pattern(pattern) => { + if markers.iter().any(|marker| text.starts_with(marker.as_str())) { + pattern.resolve(text, eof) + } else if partial { + Resolution::Pending + } else { + Resolution::NoMatch + } + } + } + } +} + +impl WatchSet { + /// Collect the watched boundaries and their deduplicated markers. + fn new<'a>(entries: impl IntoIterator) -> Self { + let mut set = Self::default(); + for (watch, boundary) in entries { + set.boundaries.push(watch); + for marker in boundary.markers() { + if !set.markers.contains(marker) { + set.markers.push(marker.clone()); + } + } + } + set + } +} diff --git a/rust/src/parser/src/unified/hf/tests.rs b/rust/src/parser/src/unified/hf/tests.rs new file mode 100644 index 0000000000..7e27a42805 --- /dev/null +++ b/rust/src/parser/src/unified/hf/tests.rs @@ -0,0 +1,933 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Executor tests. +//! +//! Template fixtures and expected values are ported from Transformers +//! `tests/utils/test_chat_parsing.py` at 6d43ab4008, with the expected output +//! dict re-expressed as aggregated unified parser events (`thinking` becomes +//! `reasoning`). + +use std::collections::BTreeMap; +use std::sync::Arc; + +use serde_json::{Value, json}; +use vllm_tokenizer::test_utils::TestTokenizer; +use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution, Tokenizer as _}; + +use super::{HfTemplateError, HfUnifiedParser, ResponseTemplate}; +use crate::tool::Tool; +use crate::tool::test_utils::split_by_chars; +use crate::unified::{UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput}; + +fn cohere_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "fields": { + "content": {"open": "<|START_RESPONSE|>", "close": "<|END_RESPONSE|>", "content": "text"}, + "thinking": {"open": "<|START_THINKING|>", "close": "<|END_THINKING|>", "content": "text"}, + "tool_calls": { + "open": "<|START_ACTION|>", + "close": "<|END_ACTION|>", + "content": "json", + "transform_each": true, + "transform": {"type": "function", "function": {"name": "{tool_name}", "arguments": "{parameters}"}}, + }, + }, + }) +} + +fn gpt_oss_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|start|>assistant", + "fields": { + "thinking": {"open": "<|channel|>analysis<|message|>", "close": "<|end|>", "content": "text"}, + "content": {"open": "<|channel|>final<|message|>", "close": "<|end|>", "content": "text"}, + "tool_calls": { + "open_pattern": r"<\|channel\|>commentary to=functions\.(?P\w+).*?<\|message\|>", + "close": "<|call|>", + "repeats": true, + "content": "json", + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, + }) +} + +fn smollm_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "tool_calls": { + "open": "", + "close": "", + "repeats": true, + "content": "json", + "transform": {"type": "function", "function": "{content}"}, + }, + "content": {"close": "<|im_end|>", "content": "text"}, + }, + }) +} + +fn qwen3_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "thinking": {"open": "", "close": "", "content": "text"}, + "tool_calls": { + "open_pattern": r"\s*\w+)>", + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "value_parser": {"name": "json", "args": {"allow_non_json": true}}, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, + }) +} + +/// Identical to the template shipped in the Gemma 4 IT checkpoints. +fn gemma4_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": ["<|turn>model\n", ""], + "fields": { + "thinking": {"open": "<|channel>thought\n", "close": "", "content": "text"}, + "tool_calls": { + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "close": "", + "repeats": true, + "content": "json", + "content_args": {"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + "content": {"close": ["", "<|tool_response>", ""], "content": "text"}, + }, + }) +} + +/// A routed-message protocol with XML-style invocations: no implicit field +/// (text outside regions is discarded), Unicode word boundaries and lazy +/// quantifiers in the opener, `xml-inline` arguments, and joined content. +fn invoke_template() -> Value { + json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|begin|>assistant", + "fields": { + "reasoning_content": {"open_pattern": r"to=self<\|msg\|>", "close": "<|pause|>", "content": "text"}, + "tool_calls": { + "open_pattern": r#"]*?\bname="(?P[^"]+)">"#, + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": r#"]*?\bname="(?P[^"]+)"[^>]*?>(?P.*?)"#, + "value_parser": {"name": "json", "args": {"allow_non_json": true}}, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + "content": { + "open_pattern": r"to=(?:user|note)<\|msg\|>", + "close": ["<|end|>", "<|pause|>"], + "repeats": true, + "join": "", + "content": "text", + }, + }, + }) +} + +fn compile(template: Value) -> Arc { + Arc::new(ResponseTemplate::from_json(&template).unwrap()) +} + +fn tokenizer() -> Arc { + Arc::new(TestTokenizer::new()) +} + +/// Parse `chunks` after initializing from `prompt`, returning all events. +fn parse_events( + template: &Arc, + tools: &[Tool], + prompt: &str, + chunks: &[&str], +) -> Result, UnifiedParserError> { + let tokenizer = tokenizer(); + let prompt_ids = tokenizer.encode(prompt, false).unwrap(); + let mut parser = HfUnifiedParser::new(template.clone(), tools, tokenizer); + parser.initialize(&prompt_ids)?; + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + parser.parse_into(DecodedText::unattributed(*chunk), &mut output)?; + } + output.append(parser.finish()?); + Ok(output.events) +} + +/// Aggregate events into the shape of the Transformers output dict. +fn message(events: &[UnifiedParserEvent]) -> Value { + let mut reasoning = String::new(); + let mut content = String::new(); + let mut calls: BTreeMap = BTreeMap::new(); + for event in events { + match event { + UnifiedParserEvent::Text(text) => content.push_str(text), + UnifiedParserEvent::Reasoning(piece) => reasoning.push_str(&piece.text), + UnifiedParserEvent::ToolCall(delta) => { + let call = calls.entry(delta.tool_index).or_default(); + if let Some(name) = &delta.name { + call.0.clone_from(name); + } + call.1.push_str(&delta.arguments); + } + } + } + let mut out = serde_json::Map::new(); + if !reasoning.is_empty() { + out.insert("reasoning".to_string(), reasoning.into()); + } + if !content.is_empty() { + out.insert("content".to_string(), content.into()); + } + if !calls.is_empty() { + let calls = calls + .into_values() + .map(|(name, arguments)| { + let arguments = + serde_json::from_str(&arguments).unwrap_or(Value::String(arguments)); + json!({"name": name, "arguments": arguments}) + }) + .collect(); + out.insert("tool_calls".to_string(), Value::Array(calls)); + } + Value::Object(out) +} + +fn parse(template: Value, text: &str) -> Value { + message(&parse_events(&compile(template), &[], "", &[text]).unwrap()) +} + +/// Split `text` at pseudo-random char boundaries (xorshift, seeded). +fn random_chunks(text: &str, seed: u64) -> Vec<&str> { + let mut state = seed | 1; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let mut chunks = Vec::new(); + let mut start = 0; + for (index, _) in text.char_indices().skip(1) { + if next() % 3 == 0 { + chunks.push(&text[start..index]); + start = index; + } + } + chunks.push(&text[start..]); + chunks +} + +#[test] +fn upstream_fixtures_match_transformers_values() { + let cases = [ + ( + cohere_template(), + "<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[\n {\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"temperature_format\": \"Celsius\"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>", + json!({ + "reasoning": "I should call a tool.", + "tool_calls": [{"name": "simple_tool", "arguments": {"temperature_format": "Celsius"}}], + }), + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>We will call function get_current_weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\n \"location\": \"San Francisco, CA\"\n}", + json!({ + "reasoning": "We will call function get_current_weather.", + "tool_calls": [{"name": "get_current_weather", "arguments": {"location": "San Francisco, CA"}}], + }), + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>User asks a simple math question: 2+2 = 4. Provide answer.<|end|><|start|>assistant<|channel|>final<|message|>2", + json!({ + "reasoning": "User asks a simple math question: 2+2 = 4. Provide answer.", + "content": "2", + }), + ), + ( + smollm_template(), + "\nOkay, let me greet them.\n\n\n{\"name\": \"greet_user\", \"arguments\": {\"greeting\": \"Hello!\"}}", + json!({ + "reasoning": "Okay, let me greet them.", + "tool_calls": [{"name": "greet_user", "arguments": {"greeting": "Hello!"}}], + }), + ), + ( + smollm_template(), + "{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}", + json!({"tool_calls": [{"name": "get_weather", "arguments": {"city": "Paris"}}]}), + ), + ( + smollm_template(), + "\nOkay, gravity.\nSome content about gravity goes here!", + json!({"reasoning": "Okay, gravity.", "content": "Some content about gravity goes here!"}), + ), + ( + qwen3_template(), + "\n\n\n[{\"country\": \"France\", \"city\": \"Paris\"}]\n\n\ncelsius\n\n\n", + json!({ + "tool_calls": [{ + "name": "get_weather", + "arguments": {"locations": [{"country": "France", "city": "Paris"}], "temp_units": "celsius"}, + }], + }), + ), + ( + gemma4_template(), + "<|channel>thought\nThe user is asking for the temperature.<|tool_call>call:get_current_temperature{detail_level:0,location:<|\"|>Paris, France<|\"|>,unit:<|\"|>celsius<|\"|>}<|tool_response>", + json!({ + "reasoning": "The user is asking for the temperature.", + "tool_calls": [{ + "name": "get_current_temperature", + "arguments": {"detail_level": 0, "location": "Paris, France", "unit": "celsius"}, + }], + }), + ), + ( + gemma4_template(), + "<|channel>thought\nLet me call the tool.<|tool_call>call:foo{bool_value:true,list_value:[<|\"|>foo<|\"|>,<|\"|>bar<|\"|>],null_value:null,number_value:1,string_value:<|\"|>foo<|\"|>,struct_value:{foo:<|\"|>bar<|\"|>}}", + json!({ + "reasoning": "Let me call the tool.", + "tool_calls": [{ + "name": "foo", + "arguments": { + "bool_value": true, + "list_value": ["foo", "bar"], + "null_value": null, + "number_value": 1, + "string_value": "foo", + "struct_value": {"foo": "bar"}, + }, + }], + }), + ), + ]; + for (template, text, expected) in cases { + assert_eq!(parse(template, text), expected, "{text}"); + } +} + +/// Upstream fixtures whose patterns have no literal prefix are rejected at load. +#[test] +fn patterns_without_literal_prefix_are_unsupported() { + let ernie = json!({ + "start_anchor": "Assistant:", + "fields": {"thinking": {"open_pattern": r"(?:^|\s*)", "close": ""}}, + }); + let inkling = json!({ + "start_anchor": "<|message_model|>", + "fields": {"content": {"open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_text\|>", "close": "<|end_message|>"}}, + }); + for template in [ernie, inkling] { + assert!(matches!( + ResponseTemplate::from_json(&template), + Err(HfTemplateError::Unsupported { .. }) + )); + } +} + +#[test] +fn streaming_matches_whole_string_for_every_chunking() { + let fixtures = [ + ( + cohere_template(), + "<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[{\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"a\": 1}}]<|END_ACTION|>", + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>thinking chunk<|end|><|channel|>final<|message|>done text", + ), + ( + gpt_oss_template(), + "<|channel|>analysis<|message|>Let me check.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\"location\": \"San Francisco, CA\"}<|call|>", + ), + ( + smollm_template(), + "thinking\n{\"name\": \"fn\", \"arguments\": {\"x\": 1}}", + ), + ( + qwen3_template(), + "short thought\n\n\n\nParis\n\n\n", + ), + ( + gemma4_template(), + "<|channel>thought\nhi 天气前言 <|tool_call>call:foo{a:1,b:<|\"|>bar<|\"|>} after", + ), + ( + invoke_template(), + "header to=self<|msg|>plan it<|pause|>to=user<|msg|>Hello <|pause|>\"Paris\"3to=note<|msg|>world<|end|>", + ), + ]; + for (template, text) in fixtures { + let template = compile(template); + let expected = message(&parse_events(&template, &[], "", &[text]).unwrap()); + let mut chunkings: Vec> = + [1, 2, 3, 5, 7, 13, 31].map(|step| split_by_chars(text, step)).into(); + chunkings.extend((0..30).map(|seed| random_chunks(text, 0xC0DE_5EED + seed))); + for chunks in chunkings { + assert_eq!(chunks.concat(), text); + let actual = message(&parse_events(&template, &[], "", &chunks).unwrap()); + assert_eq!(actual, expected, "{chunks:?}"); + } + } +} + +#[test] +fn gemma4_streams_text_and_starts_calls_at_the_opener() { + let template = compile(gemma4_template()); + let chunks = [ + "<|channel>thought\n", + "Check ", + "the weather. ", + "Sure", + ". <|tool_call>call:get_weather", + "{location:<|\"|>Paris<|\"|>}", + "", + ]; + let tokenizer = tokenizer(); + let mut parser = HfUnifiedParser::new(template, &[], tokenizer); + parser.initialize(&[]).unwrap(); + let steps: Vec<_> = chunks + .iter() + .map(|chunk| { + let mut output = UnifiedParserOutput::default(); + parser.parse_into(DecodedText::unattributed(*chunk), &mut output).unwrap(); + (*chunk, output.events) + }) + .collect(); + expect_test::expect![[r#" + [ + ( + "<|channel>thought\n", + [], + ), + ( + "Check ", + [ + Reasoning( + DecodedText { + text: "Check", + attributions: [], + }, + ), + ], + ), + ( + "the weather. ", + [ + Reasoning( + DecodedText { + text: " the weather.", + attributions: [], + }, + ), + ], + ), + ( + "Sure", + [ + Text( + "Sure", + ), + ], + ), + ( + ". <|tool_call>call:get_weather", + [ + Text( + ".", + ), + ], + ), + ( + "{location:<|\"|>Paris<|\"|>}", + [ + ToolCall( + ToolCallDelta { + tool_index: 0, + name: Some( + "get_weather", + ), + arguments: "", + }, + ), + ], + ), + ( + "", + [ + ToolCall( + ToolCallDelta { + tool_index: 0, + name: None, + arguments: "{\"location\":\"Paris\"}", + }, + ), + ], + ), + ] + "#]] + .assert_debug_eq(&steps); +} + +#[test] +fn invoke_protocol_discards_unrouted_text_and_joins_content() { + let text = "stray to=self<|msg|> plan <|pause|>to=user<|msg|>Hello, <|pause|>betweenxto=note<|msg|>world<|end|>tail"; + expect_test::expect![[r#" + Object { + "reasoning": String("plan"), + "content": String("Hello,world"), + "tool_calls": Array [ + Object { + "name": String("get"), + "arguments": Object { + "n": String("x"), + }, + }, + ], + } + "#]] + .assert_debug_eq(&parse(invoke_template(), text)); + + // Unicode word boundaries: `é` is a word character, so `éname=` is not the `name` attribute. + let text = "北京"; + assert_eq!( + parse(invoke_template(), text), + json!({"tool_calls": [{"name": "获取", "arguments": {"城市": "北京"}}]}) + ); +} + +#[test] +fn join_concatenates_repeated_matches() { + let template = json!({ + "defaults": {"role": "assistant"}, + "start_anchor": "<|assistant|>", + "fields": { + "thinking": {"open": "", "close": "", "repeats": true, "join": " "}, + "content": {"repeats": true, "join": " "}, + }, + }); + assert_eq!( + parse( + template.clone(), + "firstmiddleseconddone" + ), + json!({"reasoning": "first second", "content": "middle done"}) + ); + assert_eq!( + parse(template, "only"), + json!({"reasoning": "only"}) + ); +} + +#[test] +fn transforms_with_dotted_paths() { + let template = json!({ + "start_anchor": "<|assistant|>", + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform_each": true, + "transform": {"type": "function", "function": {"name": "{fn.name}", "arguments": "{fn.args}"}}, + }, + }, + }); + assert_eq!( + parse( + template, + r#"[{"fn": {"name": "a", "args": {"x": 1}}}, {"fn": {"name": "b", "args": {}}}]"# + ), + json!({"tool_calls": [{"name": "a", "arguments": {"x": 1}}, {"name": "b", "arguments": {}}]}) + ); +} + +#[test] +fn literal_boundaries() { + let template = json!({ + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": ["", ""], "close": ["", ""]}}, + }); + for (opener, closer) in [("", ""), ("", ""), ("", "")] { + assert_eq!( + parse(template.clone(), &format!("{opener}hi{closer}")), + json!({"content": "hi"}) + ); + } + + // A literal that is a strict prefix of another in the same list defers at the buffer edge. + let template = compile(json!({ + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": "", "close": ["END", "ENDX"]}}, + })); + let events = parse_events(&template, &[], "", &["hiEND", " more"]).unwrap(); + assert_eq!(message(&events), json!({"content": "hi"})); + + // A field without a close runs to the end of the stream. + let template = + json!({"start_anchor": "<|assistant|>", "fields": {"content": {"open": ""}}}); + assert_eq!( + parse(template, "hello world"), + json!({"content": "hello world"}) + ); +} + +#[test] +fn required_field_missing_fails_at_finish() { + let template = compile(json!({ + "start_anchor": "<|assistant|>", + "fields": {"content": {"open": "", "close": "", "optional": false}}, + })); + let error = parse_events(&template, &[], "", &["no response here"]).unwrap_err(); + expect_test::expect![[r#" + ParsingFailed { + message: "Required response_template fields missing from parsed output: [\"content\"]", + } + "#]] + .assert_debug_eq(&error); +} + +#[test] +fn prefix_sets_the_initial_state() { + let template = compile(qwen3_template()); + + // The prompt ends inside the thinking block; the prefilled newline is stripped. + let prompt = "<|im_start|>system\nYou are helpful<|im_end|>\n<|im_start|>user\nHi<|im_end|>\n<|im_start|>assistant\n\n"; + let events = parse_events(&template, &[], prompt, &["Let me think..."]).unwrap(); + assert_eq!(message(&events), json!({"reasoning": "Let me think..."})); + + // Only the slice after the last anchor matters. + let prompt = "<|im_start|>assistant\nEarlier reply<|im_end|>\n<|im_start|>user\nFollowup<|im_end|>\n<|im_start|>assistant\n\n"; + let events = parse_events(&template, &[], prompt, &["done"]).unwrap(); + assert_eq!(message(&events), json!({"reasoning": "done"})); + + // A prefix ending mid-delimiter is completed by the generated text. + let events = parse_events( + &template, + &[], + "<|im_start|>assistant\nreal body"], + ) + .unwrap(); + assert_eq!(message(&events), json!({"reasoning": "real body"})); + + // Gemma 4 continues inside the same turn after a tool response. + let template = compile(gemma4_template()); + let prompt = "<|turn>model\n<|tool_call>call:f{}<|tool_response>response:f{}<|channel>thought\n"; + let events = parse_events(&template, &[], prompt, &["Done.It is sunny."]).unwrap(); + assert_eq!( + message(&events), + json!({"reasoning": "Done.", "content": "It is sunny."}) + ); + + // Prompt text is never emitted, but it counts for leading-whitespace stripping. + let events = parse_events(&template, &[], "<|turn>model\nSure,", &[" here", " it is"]).unwrap(); + assert_eq!(message(&events), json!({"content": " here it is"})); +} + +/// Diverges from Transformers, which runs the whole prompt through the parser +/// when the start anchor is absent. +#[test] +fn missing_start_anchor_starts_from_the_initial_state() { + let template = compile(smollm_template()); + let events = parse_events(&template, &[], "\n", &["hi"]).unwrap(); + assert_eq!(message(&events), json!({"content": "hi"})); +} + +#[test] +fn tools_convert_string_arguments() { + let template = compile(json!({ + "start_anchor": "<|im_start|>assistant\n", + "fields": { + "tool_calls": { + "open_pattern": r"\s*\w+)>", + "close": "", + "repeats": true, + "content": "xml-inline", + "content_args": { + "tag_pattern": r"\w+)>\s*(?P.*?)\s*", + "merge_duplicates": true, + }, + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}}, + }, + }, + })); + let tools = [Tool { + name: "set_alarm".to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "hour": {"type": "integer"}, + "enabled": {"type": "boolean"}, + "label": {"type": "string"}, + "weekday": {"type": "integer"}, + }, + }), + strict: None, + }]; + let text = "\n\n\n7\n\n\ntrue\n\n\nwake up\n\n12\n\n"; + assert_eq!( + message(&parse_events(&template, &tools, "", &[text]).unwrap()), + json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": 7, "enabled": true, "label": "wake up", "weekday": [1, 2]}}]}) + ); + // Without tools, values stay strings. + assert_eq!( + message(&parse_events(&template, &[], "", &[text]).unwrap()), + json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "7", "enabled": "true", "label": "wake up", "weekday": ["1", "2"]}}]}) + ); +} + +#[test] +fn stripped_reasoning_whitespace_keeps_its_tokens() { + let template = compile(gemma4_template()); + let mut parser = HfUnifiedParser::new(template, &[], tokenizer()); + parser.initialize(&[]).unwrap(); + let text = "<|channel>thought\n \n plan \n"; + let mut output = UnifiedParserOutput::default(); + for (index, c) in text.chars().enumerate() { + let piece = DecodedText { + text: c.to_string(), + attributions: [TokenAttribution { + token_id: u32::try_from(index).unwrap(), + anchor: TokenAnchor::Visible { byte_offset: 0 }, + }] + .into_iter() + .collect(), + }; + parser.parse_into(piece, &mut output).unwrap(); + } + output.append(parser.finish().unwrap()); + let (reasoning, tokens) = + output + .events + .iter() + .fold((String::new(), 0), |(text, tokens), event| match event { + UnifiedParserEvent::Reasoning(piece) => { + (text + &piece.text, tokens + piece.attributions.len()) + } + _ => (text, tokens), + }); + assert_eq!(reasoning, "plan"); + // Every character between the markers is a reasoning token, stripped or not. + assert_eq!(tokens, " \n plan \n".chars().count()); +} + +#[test] +fn reset_returns_uncommitted_text() { + let template = compile(gemma4_template()); + let mut parser = HfUnifiedParser::new(template, &[], tokenizer()); + parser.initialize(&[]).unwrap(); + let mut output = UnifiedParserOutput::default(); + parser + .parse_into( + DecodedText::unattributed("Hi <|tool_call>call:f{a:<|\"|>x"), + &mut output, + ) + .unwrap(); + assert_eq!(parser.reset(), "<|tool_call>call:f{a:<|\"|>x"); +} + +#[test] +fn malformed_tool_arguments_fail_the_parser() { + let template = compile(gemma4_template()); + let error = + parse_events(&template, &[], "", &["<|tool_call>call:f{a:}"]).unwrap_err(); + assert!( + matches!(error, UnifiedParserError::ParsingFailed { .. }), + "{error:?}" + ); +} + +#[test] +fn template_validation_errors() { + let error = |template: Value| ResponseTemplate::from_json(&template).unwrap_err(); + expect_test::expect![[r#" + [ + Invalid { + message: "unsupported response_template version: 2", + }, + Invalid { + message: "At most one field may omit 'open'/'open_pattern' (that field becomes the implicit-open / leftover sink). Found: content, thinking", + }, + Invalid { + message: "response_template must define 'start_anchor' or 'start_anchor_pattern'.", + }, + Invalid { + message: "Field 'content': 'join' requires 'repeats': true", + }, + Invalid { + message: "Field 'tool_calls': open_pattern/close_pattern declares named group(s) [\"name\"], but the field has no 'transform'. Named captures are only surfaced through a 'transform' template (where they appear alongside 'content'). Either add a 'transform' that uses the captures, or remove the named groups from the pattern.", + }, + Invalid { + message: "Field 'tool_calls': transform string \"name: {content}\" mixes a {placeholder} with literal text. Use either a whole-string placeholder (e.g. \"{content}\") or a plain literal; string interpolation is not supported.", + }, + Invalid { + message: "Field 'content': 'open' literals cannot be empty strings", + }, + Unsupported { + message: "Field 'count': only 'content', 'reasoning_content'/'thinking', and 'tool_calls' fields can be reported", + }, + Unsupported { + message: "Field 'thinking': text and reasoning fields must use the 'text' content parser without a transform", + }, + Invalid { + message: "response_template.fields.content.content: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`", + }, + Invalid { + message: "response_template.fields.content.repeats: invalid type: string \"yes\", expected a boolean", + }, + Invalid { + message: "Field 'content': cannot specify both 'open' and 'open_pattern'", + }, + Invalid { + message: "Field 'tool_calls': transform_each is set but no transform was provided", + }, + Unsupported { + message: "Field 'tool_calls': 'join' requires each match to parse to a string", + }, + Invalid { + message: "Field 'tool_calls': transform placeholder '{id}' is neither 'content' nor a named group of open_pattern", + }, + ] + "#]] + .assert_debug_eq(&[ + error(json!({"version": 2, "start_anchor": "a", "fields": {"content": {}}})), + error(json!({"start_anchor": "a", "fields": {"content": {}, "thinking": {}}})), + error(json!({"fields": {"content": {}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"open": "", "join": ""}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open_pattern": r"\w+)>", "close": ""}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "", "transform": {"label": "name: {content}"}}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"open": [""]}}})), + error(json!({"start_anchor": "a", "fields": {"count": {"open": "", "content": "int"}}})), + error(json!({"start_anchor": "a", "fields": {"thinking": {"open": "", "content": "json"}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"content": "yaml"}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"repeats": "yes"}}})), + error(json!({"start_anchor": "a", "fields": {"content": {"open": "", "open_pattern": ""}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "", "transform_each": true}}})), + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "", "repeats": true, "join": ""}}})), + error(json!({ + "start_anchor": "a", + "fields": {"tool_calls": { + "open_pattern": r"\w+)>", + "close_pattern": r"\d+)>", + "content": "json", + "transform": {"function": {"name": "{name}", "arguments": "{content}", "id": "{id}"}}, + }}, + })), + ]); +} + +/// Cases where the Rust parser intentionally differs from Transformers, keyed by +/// generated text. +const DIVERGENCES: &[(&str, &str)] = &[ + ( + // A field without `repeats` keeps only its last occurrence in Transformers; + // streamed text cannot be retracted, so every occurrence is reported. + r#"Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}} after"#, + r#"{"content": "Before after", "tool_calls": [{"name": "f", "arguments": {"x": "a, b", "y": [1, 2.5, -3], "z": {"w": true}}}]}"#, + ), + ( + // Arguments use the tool parsers' shared conversion, which keeps a JSON + // number's spelling; Transformers turns `1e3` into the integer 1000. + "\n\nhour: seven\nratio: 1e3\nenabled: nope\n", + r#"{"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "seven", "ratio": 1000.0, "enabled": "nope"}}]}"#, + ), +]; + +/// Compare against `fixtures/differential.json`, generated from Transformers +/// `parse_response` by `fixtures/generate_differential.py`. +#[test] +fn differential_against_transformers() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("src/unified/hf/fixtures/differential.json"); + let corpus: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + let templates: BTreeMap<&str, Arc> = corpus["templates"] + .as_object() + .unwrap() + .iter() + .map(|(name, template)| { + ( + name.as_str(), + Arc::new(ResponseTemplate::from_json(template).unwrap()), + ) + }) + .collect(); + + for case in corpus["cases"].as_array().unwrap() { + let template = &templates[case["template"].as_str().unwrap()]; + let prefix = case["prefix"].as_str().unwrap(); + let text = case["text"].as_str().unwrap(); + let tools: Vec = case["tools"] + .as_array() + .into_iter() + .flatten() + .map(|tool| Tool { + name: tool["function"]["name"].as_str().unwrap().to_string(), + description: None, + parameters: tool["function"]["parameters"].clone(), + strict: None, + }) + .collect(); + let expected = match DIVERGENCES.iter().find(|(divergent, _)| *divergent == text) { + Some((_, expected)) => serde_json::from_str(expected).unwrap(), + None => case["expected"].clone(), + }; + + for chunks in [vec![text], split_by_chars(text, 1)] { + let actual = parse_events(template, &tools, prefix, &chunks); + match (&actual, expected.get("error")) { + (Err(_), Some(_)) => {} + (Ok(events), None) => assert_eq!(message(events), expected, "{text:?}"), + _ => panic!("{text:?}: expected {expected}, got {actual:?}"), + } + } + } +} + +#[test] +fn tool_call_value_shapes() { + // No transform: the parsed JSON is the tool call itself. + let template = compile(json!({ + "start_anchor": "<|assistant|>", + "fields": {"tool_calls": {"open": "", "close": "", "content": "json"}}, + })); + let parse = + |text: &str| parse_events(&template, &[], "", &[text]).map(|events| message(&events)); + + assert_eq!( + parse(r#"{"type": "function", "function": {"name": "f", "arguments": {"a": 1}}}"#).unwrap(), + json!({"tool_calls": [{"name": "f", "arguments": {"a": 1}}]}) + ); + assert_eq!( + parse(r#"[{"name": "f", "arguments": null}, {"name": "g", "arguments": "raw"}]"#).unwrap(), + json!({"tool_calls": [{"name": "f", "arguments": {}}, {"name": "g", "arguments": "raw"}]}) + ); + expect_test::expect![[r#" + ParsingFailed { + message: "tool call must be {\"function\": {\"name\": ..., \"arguments\": ...}} or {\"name\": ..., \"arguments\": ...} with a string name, got {\"name\":1}", + } + "#]] + .assert_debug_eq(&parse(r#"{"name": 1}"#).unwrap_err()); +} diff --git a/rust/src/parser/src/unified/hf/transform.rs b/rust/src/parser/src/unified/hf/transform.rs new file mode 100644 index 0000000000..9d3ab6913f --- /dev/null +++ b/rust/src/parser/src/unified/hf/transform.rs @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Transform templates: restructure a parsed region value into the output shape. +//! +//! Port of `_apply_transform`, `validate_transform_strings`, and `process_field` +//! from `content_parsers.py`. + +use std::sync::LazyLock; + +use regex_automata::meta::Regex; +use serde_json::{Map, Value}; + +use super::pattern::Captures; +use super::{Result, invalid}; +use crate::unified::{self, parsing_failed}; + +/// `\{(\w+(?:\.\w+)*)\}`: a whole-string placeholder such as `{content}` or +/// `{content.args}`. +static PLACEHOLDER: LazyLock = + LazyLock::new(|| Regex::new(r"\{\w+(?:\.\w+)*\}").expect("valid regex")); + +/// A compiled transform template. +#[derive(Debug, Clone, PartialEq)] +pub(super) enum Transform { + Object(Vec<(String, Transform)>), + Array(Vec), + /// A dotted placeholder path, e.g. `["content", "args"]`. + Placeholder(Vec), + /// Any other JSON value, copied verbatim. + Literal(Value), +} + +impl Transform { + /// Compile a transform template, rejecting any string that mixes a + /// `{name}` placeholder with literal text. + pub fn compile(scope: &str, template: &Value) -> Result { + Ok(match template { + Value::Object(object) => Self::Object( + object + .iter() + .map(|(key, value)| Ok((key.clone(), Self::compile(scope, value)?))) + .collect::>()?, + ), + Value::Array(values) => Self::Array( + values.iter().map(|value| Self::compile(scope, value)).collect::>()?, + ), + Value::String(string) => match PLACEHOLDER.find(string) { + Some(found) if found.range() == (0..string.len()) => Self::Placeholder( + string[1..string.len() - 1].split('.').map(str::to_string).collect(), + ), + Some(_) => { + return Err(invalid!( + "{scope}: transform string {string:?} mixes a {{placeholder}} with literal text. \ + Use either a whole-string placeholder (e.g. \"{{content}}\") or a plain literal; \ + string interpolation is not supported." + )); + } + None => Self::Literal(template.clone()), + }, + other => Self::Literal(other.clone()), + }) + } + + /// Return the placeholder path found at `key_path` inside this template, if any. + pub fn placeholder_at(&self, key_path: &[&str]) -> Option<&[String]> { + match (self, key_path) { + (Self::Placeholder(path), []) => Some(path), + (Self::Object(entries), [key, rest @ ..]) => entries + .iter() + .find(|(entry, _)| entry == key) + .and_then(|(_, transform)| transform.placeholder_at(rest)), + _ => None, + } + } + + /// Root names of every placeholder in this template. + pub fn placeholder_roots(&self) -> Box + '_> { + match self { + Self::Object(entries) => { + Box::new(entries.iter().flat_map(|(_, transform)| transform.placeholder_roots())) + } + Self::Array(transforms) => { + Box::new(transforms.iter().flat_map(Self::placeholder_roots)) + } + Self::Placeholder(path) => Box::new(path.first().map(String::as_str).into_iter()), + Self::Literal(_) => Box::new(std::iter::empty()), + } + } + + /// Recursively instantiate this template against `scope`. + pub fn apply(&self, scope: &Map) -> unified::Result { + Ok(match self { + Self::Object(entries) => Value::Object( + entries + .iter() + .map(|(key, transform)| Ok((key.clone(), transform.apply(scope)?))) + .collect::>()?, + ), + Self::Array(transforms) => Value::Array( + transforms + .iter() + .map(|transform| transform.apply(scope)) + .collect::>()?, + ), + Self::Literal(value) => value.clone(), + Self::Placeholder(path) => { + let dotted = path.join("."); + let (root, keys) = path.split_first().expect("placeholder path is non-empty"); + let Some(mut value) = scope.get(root) else { + let available: Vec<_> = scope.keys().collect(); + return Err(parsing_failed!( + "transform placeholder '{{{dotted}}}' is not defined. Available: {available:?}" + )); + }; + for key in keys { + let Some(object) = value.as_object() else { + return Err(parsing_failed!( + "transform placeholder '{{{dotted}}}' cannot index into {} at '{key}'", + json_type_name(value) + )); + }; + let Some(next) = object.get(key) else { + let available: Vec<_> = object.keys().collect(); + return Err(parsing_failed!( + "transform placeholder '{{{dotted}}}' is missing key '{key}'. Available: {available:?}" + )); + }; + value = next; + } + value.clone() + } + }) + } +} + +/// A field's transform: applied to the parsed value, or to each of its elements. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct FieldTransform { + pub template: Transform, + /// `transform_each`: the parsed content must be a list of dicts, and the + /// template is applied to each element (with the element's keys unpacked into + /// the template scope, alongside any regex captures). + pub each: bool, +} + +impl FieldTransform { + /// Apply the transform to a parsed region value. + pub fn apply( + &self, + field_name: &str, + parsed: Value, + captures: &Captures, + ) -> unified::Result { + if !self.each { + let mut scope = captures.clone(); + scope.insert("content".to_string(), parsed); + return self.template.apply(&scope); + } + + let Value::Array(items) = parsed else { + return Err(parsing_failed!( + "Field '{field_name}': transform_each requires the parsed content to be an array, got {}.", + json_type_name(&parsed) + )); + }; + items + .into_iter() + .map(|item| { + let Value::Object(item) = item else { + return Err(parsing_failed!( + "Field '{field_name}': transform_each requires each array element to be an object, got {}.", + json_type_name(&item) + )); + }; + let mut scope = captures.clone(); + scope.extend(item); + self.template.apply(&scope) + }) + .collect::>() + .map(Value::Array) + } +} + +/// JSON type name of a value, for error messages. +fn json_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn scope(value: Value) -> Map { + value.as_object().unwrap().clone() + } + + #[test] + fn dotted_placeholders_descend_into_objects() { + let transform = Transform::compile( + "test", + &json!({"type": "function", "function": {"name": "{content.name}", "arguments": "{content.args}"}}), + ) + .unwrap(); + let value = transform + .apply(&scope(json!({"content": {"name": "f", "args": {"a": 1}}}))) + .unwrap(); + assert_eq!( + value, + json!({"type": "function", "function": {"name": "f", "arguments": {"a": 1}}}) + ); + assert_eq!( + transform.placeholder_at(&["function", "name"]), + Some(["content".to_string(), "name".to_string()].as_slice()) + ); + } + + #[test] + fn placeholder_errors_match_transformers() { + let apply = |template: Value, scope_value: Value| { + Transform::compile("test", &template) + .unwrap() + .apply(&scope(scope_value)) + .unwrap_err() + }; + expect_test::expect![[r#" + ParsingFailed { + message: "transform placeholder '{name}' is not defined. Available: [\"content\"]", + } + "#]] + .assert_debug_eq(&apply(json!("{name}"), json!({"content": 1}))); + expect_test::expect![[r#" + ParsingFailed { + message: "transform placeholder '{content.a}' cannot index into number at 'a'", + } + "#]] + .assert_debug_eq(&apply(json!("{content.a}"), json!({"content": 1}))); + expect_test::expect![[r#" + ParsingFailed { + message: "transform placeholder '{content.b}' is missing key 'b'. Available: [\"a\"]", + } + "#]] + .assert_debug_eq(&apply(json!("{content.b}"), json!({"content": {"a": 1}}))); + } + + #[test] + fn interpolation_is_rejected_at_load() { + assert!(Transform::compile("test", &json!({"name": "fn_{name}"})).is_err()); + assert_eq!( + Transform::compile("test", &json!("no placeholder")).unwrap(), + Transform::Literal(json!("no placeholder")) + ); + } +} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs index c621075a02..aa0e92e6e1 100644 --- a/rust/src/parser/src/unified/mod.rs +++ b/rust/src/parser/src/unified/mod.rs @@ -5,12 +5,14 @@ mod combined; mod gemma4; +mod hf; mod hy; mod inkling; mod kimi_k3; pub use combined::CombinedParser; pub use gemma4::Gemma4UnifiedParser; +pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate}; pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser}; pub use inkling::InklingUnifiedParser; pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser}; @@ -271,8 +273,13 @@ pub trait UnifiedParser: Send { #[derive(Debug, Error, Macro)] #[thiserror_ext(macro(path = "crate::unified", mangle))] pub enum UnifiedParserError { - #[error("combined parser is constructed from split parser instances")] - CombinedParserConstructor, + /// The parser is built from inputs other than tools and a tokenizer, so + /// [`UnifiedParser::create`] cannot construct it. + #[error("the `{parser}` unified parser is built from {built_from}, not by name")] + NoNamedConstructor { + parser: &'static str, + built_from: &'static str, + }, #[error("tokenizer is missing unified parser token `{token}`")] MissingToken { token: String }, #[error("unified parser parsing failed: {message}")] diff --git a/rust/src/text/src/backend/hf/config.rs b/rust/src/text/src/backend/hf/config.rs index 0768adf4ae..0864df641b 100644 --- a/rust/src/text/src/backend/hf/config.rs +++ b/rust/src/text/src/backend/hf/config.rs @@ -23,6 +23,9 @@ pub struct HfTokenizerConfig { /// containing "Tiktoken" which can be used as a hint for backend /// selection. pub tokenizer_class: Option, + /// The Hugging Face `response_template`: a declarative description of the + /// model's output protocol, executed by the `hf` unified parser. + pub response_template: Option, } /// Hugging Face named special tokens may be serialized as a string or an From bcee730b1a9d25f0fd283a0ef6c19133ebeebf4f Mon Sep 17 00:00:00 2001 From: Sundri Lai Date: Thu, 1 Oct 2026 01:26:14 -0700 Subject: [PATCH 194/248] [Docs] Remove references to removed env vars (#59530) Signed-off-by: Sundri Lai --- docs/features/engram.md | 9 +++------ docs/serving/online_serving/renderer.md | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/features/engram.md b/docs/features/engram.md index 73dc05e9f1..023477b345 100644 --- a/docs/features/engram.md +++ b/docs/features/engram.md @@ -83,10 +83,7 @@ directly from the Triton kernel through unified virtual addressing (UVA), prefetching on a side CUDA stream so transfers overlap with the compute of preceding layers. The freed GPU memory goes to the KV cache. -`cpu_offload` defaults to `true` (on): it follows the -`VLLM_PLE_CPU_OFFLOAD` environment variable, which also defaults to `true`. -An explicit `--engram-config` value takes precedence over the environment -variable. +`cpu_offload` defaults to `true` (on) To keep the tables resident on GPU instead (e.g. when host memory is scarce): @@ -96,8 +93,8 @@ vllm serve deepseek-ai/DeepSeek-V4.1-Flash \ --engram-config.cpu_offload false ``` -or equivalently `VLLM_PLE_CPU_OFFLOAD=0`. CPU offload requires a GPU with UVA -support; vLLM fails fast when it is unavailable. +CPU offload requires a GPU with UVA support; +vLLM fails fast when it is unavailable. ## Data-parallel topologies (DeepSeek V4.1) diff --git a/docs/serving/online_serving/renderer.md b/docs/serving/online_serving/renderer.md index 9d494df22e..fc6579c258 100644 --- a/docs/serving/online_serving/renderer.md +++ b/docs/serving/online_serving/renderer.md @@ -51,8 +51,7 @@ and provision transport limits and memory accordingly. For example, start a standard inference server with scale-out endpoints enabled: ```bash -VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \ - vllm serve meta-llama/Llama-3.1-8B-Instruct +vllm serve meta-llama/Llama-3.1-8B-Instruct --enable-scale-out ``` Send a Responses request and use `jq` to extract the `token_ids` field: From 08d77cad7be4c2ba1f72ebbd5d58571e7e91b2de Mon Sep 17 00:00:00 2001 From: YueshenZ <844936239@qq.com> Date: Thu, 1 Oct 2026 17:25:45 +0800 Subject: [PATCH 195/248] [Bugfix] Gate Kimi-K3 KDA warmup on sys.modules to skip Kimi import for non-Kimi models (#59257) Signed-off-by: YueshenZ Signed-off-by: zjy0516 Co-authored-by: YueshenZ Co-authored-by: Claude Code Co-authored-by: Codex Co-authored-by: zjy0516 --- vllm/model_executor/warmup/kimi_k3_triton_warmup.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/warmup/kimi_k3_triton_warmup.py b/vllm/model_executor/warmup/kimi_k3_triton_warmup.py index 4c4a0327c7..e7883fff44 100644 --- a/vllm/model_executor/warmup/kimi_k3_triton_warmup.py +++ b/vllm/model_executor/warmup/kimi_k3_triton_warmup.py @@ -4,6 +4,7 @@ from __future__ import annotations +import sys from typing import TYPE_CHECKING import torch @@ -19,7 +20,10 @@ logger = init_logger(__name__) def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None: - from vllm.models.kimi_k3.nvidia.kda import KimiK3DeltaAttention + # Kimi model construction already imports kda. Avoid importing it here. + kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda") + if kda is None: + return None compilation_config = getattr( worker.model_runner, @@ -33,7 +37,7 @@ def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None: ( layer for layer in static_context.values() - if isinstance(layer, KimiK3DeltaAttention) + if isinstance(layer, kda.KimiK3DeltaAttention) ), None, ) From cb6458c5ce61632524c0447d49790725c1e1d9a2 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Thu, 1 Oct 2026 17:59:37 +0800 Subject: [PATCH 196/248] [Core] Remove deprecated mamba_cache_mode "all" (#58997) Signed-off-by: zjy0516 Co-authored-by: Claude Opus 5.5 Co-authored-by: Thomas Parnell --- tests/kernels/mamba/test_cpu_short_conv.py | 10 - tests/kernels/mamba/test_ssu_dispatch.py | 5 - tests/test_config.py | 9 +- .../attention/test_mamba_metadata_builder.py | 58 +++ .../test_mamba_update_block_table.py | 466 ------------------ tests/v1/core/test_scheduler.py | 76 +-- .../run_mamba_prefix_cache_test.sh | 4 +- .../unit/test_moriio_hma_scheduler.py | 61 +-- .../unit/test_nixl_connector_hma.py | 42 +- .../unit/test_offloading_connector.py | 7 +- tests/v1/kv_connector/unit/utils.py | 2 +- tests/v1/simple_kv_offload/test_scheduler.py | 51 +- tests/v1/worker/test_gpu_warmup_blocks.py | 12 +- tests/v1/worker/test_mamba_utils.py | 2 +- vllm/config/cache.py | 19 +- vllm/config/vllm.py | 12 - .../kv_transfer/kv_connector/utils.py | 8 +- .../v1/moriio/moriio_connector.py | 17 +- .../kv_connector/v1/nixl/base_scheduler.py | 17 +- .../kv_connector/v1/nixl/base_worker.py | 13 +- .../kv_connector/v1/offloading/scheduler.py | 7 +- .../layers/mamba/mamba_mixer.py | 57 +-- .../layers/mamba/mamba_mixer2.py | 199 +------- vllm/model_executor/models/config.py | 17 +- vllm/model_executor/models/interns2_mobius.py | 6 - vllm/model_executor/models/lfm2.py | 7 - vllm/model_executor/models/lfm2_moe.py | 7 - vllm/model_executor/models/qwen3_5.py | 6 - vllm/model_executor/models/qwen3_5_mtp.py | 7 - vllm/model_executor/models/qwen3_next.py | 6 - vllm/model_executor/models/qwen3_next_mtp.py | 7 - vllm/models/kimi_k3/nvidia/kda_metadata.py | 2 +- vllm/models/qwen4_exp/amd/model.py | 5 - vllm/models/qwen4_exp/amd/mtp.py | 7 - vllm/models/qwen4_exp/nvidia/model.py | 5 - vllm/models/qwen4_exp/nvidia/mtp.py | 7 - vllm/platforms/interface.py | 37 +- vllm/v1/attention/backends/mamba1_attn.py | 23 +- vllm/v1/attention/backends/mamba2_attn.py | 1 - vllm/v1/attention/backends/mamba_attn.py | 267 +--------- vllm/v1/attention/backends/short_conv_attn.py | 13 +- vllm/v1/attention/backends/utils.py | 7 +- vllm/v1/kv_cache_interface.py | 7 +- vllm/v1/worker/gpu_model_runner.py | 32 -- vllm/v1/worker/mamba_utils.py | 47 -- 45 files changed, 201 insertions(+), 1476 deletions(-) create mode 100644 tests/v1/attention/test_mamba_metadata_builder.py delete mode 100644 tests/v1/attention/test_mamba_update_block_table.py diff --git a/tests/kernels/mamba/test_cpu_short_conv.py b/tests/kernels/mamba/test_cpu_short_conv.py index c8e85a4551..85665c205b 100644 --- a/tests/kernels/mamba/test_cpu_short_conv.py +++ b/tests/kernels/mamba/test_cpu_short_conv.py @@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config): state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32), num_accepted_tokens=None, query_start_loc_d=None, - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, - num_computed_tokens_p=None, seq_lens=torch.tensor([5]), ) @@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config): state_indices_tensor_d=state_indices_tensor_d, num_accepted_tokens=None, query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32), - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, - num_computed_tokens_p=None, seq_lens=torch.tensor([1, 1]), ) diff --git a/tests/kernels/mamba/test_ssu_dispatch.py b/tests/kernels/mamba/test_ssu_dispatch.py index 082b61c43d..3b3dc2dc6b 100644 --- a/tests/kernels/mamba/test_ssu_dispatch.py +++ b/tests/kernels/mamba/test_ssu_dispatch.py @@ -568,15 +568,10 @@ def test_replayssm_mixer_selects_mtp_layout( num_reqs=2, has_initial_states_p=None, query_start_loc_p=None, - num_computed_tokens_p=None, state_indices_tensor_p=None, state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32), query_start_loc_d=query_start_loc_d, num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32), - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, seq_lens=torch.tensor([104, 102], dtype=torch.int32), replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)), replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32), diff --git a/tests/test_config.py b/tests/test_config.py index e7389d18b4..48d286b521 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -270,9 +270,6 @@ def test_kda_recoverssm_derivation_is_revalidated(): with pytest.raises(ValueError, match="VLLM_USE_V2_MODEL_RUNNER=1"): VllmConfig.validate_mamba_cached_kernel(config) config.use_v2_model_runner = True - config.cache_config.mamba_cache_mode = "all" - with pytest.raises(ValueError, match="only none and align"): - VllmConfig.validate_mamba_cached_kernel(config) config.cache_config.mamba_cache_mode = "none" config.model_config.architecture = "NemotronHForCausalLM" @@ -286,6 +283,12 @@ def test_kda_recoverssm_derivation_is_revalidated(): VllmConfig.validate_mamba_cached_kernel(config) +def test_mamba_cache_mode_all_is_rejected(): + """The removed 'all' mode must fail validation instead of being ignored.""" + with pytest.raises(ValidationError, match="mamba_cache_mode"): + CacheConfig(mamba_cache_mode="all") + + def test_per_request_spec_decode_metrics_requires_spec_decode(): # The flag only makes sense with speculative decoding configured; enabling # it without --speculative-config should fail fast rather than silently diff --git a/tests/v1/attention/test_mamba_metadata_builder.py b/tests/v1/attention/test_mamba_metadata_builder.py new file mode 100644 index 0000000000..85d9e0b764 --- /dev/null +++ b/tests/v1/attention/test_mamba_metadata_builder.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for BaseMambaAttentionMetadataBuilder batch classification.""" + +from types import SimpleNamespace + +from tests.v1.attention.utils import MockMambaBuilder +from vllm.config.compilation import CUDAGraphMode + + +def _make_vllm_config( + max_model_len: int, + max_num_seqs: int, + num_speculative_tokens: int = 0, + block_size: int | None = None, +): + """Create a minimal mock VllmConfig with only the fields the builder + accesses, avoiding any model download / HF config inspection.""" + speculative_config = ( + SimpleNamespace( + num_speculative_tokens=num_speculative_tokens, + parallel_drafting=False, + ) + if num_speculative_tokens > 0 + else None + ) + return SimpleNamespace( + cache_config=SimpleNamespace( + block_size=block_size, + mamba_cache_mode="none", + use_replayssm=False, + replayssm_buffer_len=16, + ), + compilation_config=SimpleNamespace( + cudagraph_mode=CUDAGraphMode.FULL, + max_cudagraph_capture_size=None, + ), + speculative_config=speculative_config, + num_speculative_tokens=num_speculative_tokens, + parallel_config=SimpleNamespace(decode_context_parallel_size=1), + scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs), + model_config=SimpleNamespace(max_model_len=max_model_len), + ) + + +def test_mamba_single_token_prompt_runs_as_prefill(): + seq_lens = [8, 9, 1] + config = _make_vllm_config(256, len(seq_lens), block_size=16) + metadata = MockMambaBuilder.build_mamba_metadata( + config, + seq_lens=seq_lens, + query_lens=[1] * len(seq_lens), + is_prefilling=[False, False, True], + ) + + assert metadata.num_decodes == 2 + assert metadata.num_prefills == 1 + assert metadata.has_initial_states_p.tolist() == [False] diff --git a/tests/v1/attention/test_mamba_update_block_table.py b/tests/v1/attention/test_mamba_update_block_table.py deleted file mode 100644 index 49a298fe19..0000000000 --- a/tests/v1/attention/test_mamba_update_block_table.py +++ /dev/null @@ -1,466 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Regression test for https://github.com/vllm-project/vllm/issues/34865. - -When multiple KV cache groups share the same MambaSpec (as in Nemotron -hybrid models), the metadata caching optimization reuses metadata from -an earlier group via update_block_table(). In 'all' mode with CUDA graphs, -update_block_table() must copy block_idx_last_scheduled_token and -block_idx_last_computed_token to the *current* builder's persistent -buffers, otherwise CUDA graph replay reads stale values from uninitialized -buffers. -""" - -from types import SimpleNamespace - -import torch - -from tests.v1.attention.utils import MockMambaBuilder -from vllm.config.compilation import CUDAGraphMode -from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadata -from vllm.v1.kv_cache_interface import MambaSpec - - -def _make_vllm_config( - max_model_len: int, - max_num_seqs: int, - num_speculative_tokens: int = 0, - block_size: int | None = None, -): - """Create a minimal mock VllmConfig with only the fields the builder - accesses, avoiding any model download / HF config inspection.""" - speculative_config = ( - SimpleNamespace( - num_speculative_tokens=num_speculative_tokens, - parallel_drafting=False, - ) - if num_speculative_tokens > 0 - else None - ) - return SimpleNamespace( - cache_config=SimpleNamespace( - block_size=block_size, - mamba_cache_mode="all", - use_replayssm=False, - replayssm_buffer_len=16, - ), - compilation_config=SimpleNamespace( - cudagraph_mode=CUDAGraphMode.FULL, - max_cudagraph_capture_size=None, - ), - speculative_config=speculative_config, - num_speculative_tokens=num_speculative_tokens, - parallel_config=SimpleNamespace(decode_context_parallel_size=1), - scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs), - model_config=SimpleNamespace(max_model_len=max_model_len), - ) - - -def test_mamba_single_token_prompt_runs_as_prefill(): - seq_lens = [8, 9, 1] - config = _make_vllm_config(256, len(seq_lens), block_size=16) - metadata = MockMambaBuilder.build_mamba_metadata( - config, - seq_lens=seq_lens, - query_lens=[1] * len(seq_lens), - is_prefilling=[False, False, True], - ) - - assert metadata.num_decodes == 2 - assert metadata.num_prefills == 1 - assert metadata.has_initial_states_p.tolist() == [False] - - -def test_update_block_table_copies_block_idx_to_persistent_buffers(): - """update_block_table() must write block_idx tensors to the current - builder's persistent buffers, not leave them pointing to a different - builder's buffers.""" - block_size = 16 - max_model_len = 256 - num_reqs = 4 - device = torch.device("cpu") - - vllm_config = _make_vllm_config(max_model_len, num_reqs) - - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - ) - - # Two builders simulating two KV cache groups with the same MambaSpec. - builder_a = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - builder_b = MockMambaBuilder(spec, ["layer1"], vllm_config, device) - - # Sanity: each builder has its own persistent buffer. - assert ( - builder_a.block_idx_last_scheduled_token.data_ptr() - != builder_b.block_idx_last_scheduled_token.data_ptr() - ) - - # Construct decode-only metadata as if builder_a.build() produced it. - max_blocks = max_model_len // block_size - seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device) - block_idx_vals = (seq_lens - 1) // block_size # [3, 3, 3, 3] - - builder_a.block_idx_last_scheduled_token[:num_reqs].copy_(block_idx_vals) - builder_a.block_idx_last_computed_token[:num_reqs].copy_(block_idx_vals) - - metadata_a = BaseMambaAttentionMetadata( - num_prefills=0, - num_prefill_tokens=0, - num_decodes=num_reqs, - num_decode_tokens=num_reqs, - num_reqs=num_reqs, - has_initial_states_p=None, - query_start_loc_p=None, - num_computed_tokens_p=None, - state_indices_tensor_p=None, - query_start_loc_d=None, - num_accepted_tokens=None, - state_indices_tensor_d=builder_a.state_indices_tensor_d[:num_reqs], - block_idx_last_scheduled_token=( - builder_a.block_idx_last_scheduled_token[:num_reqs] - ), - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=( - builder_a.block_idx_last_computed_token[:num_reqs] - ), - block_idx_last_scheduled_token_prev_step=None, - seq_lens=seq_lens, - ) - - # Call update_block_table on builder_b (simulates the metadata caching - # optimization reusing metadata from builder_a's group). - blk_table = torch.randint( - 0, 100, (num_reqs, max_blocks), dtype=torch.int32, device=device - ) - slot_mapping = torch.zeros(num_reqs, dtype=torch.int64, device=device) - - metadata_b = builder_b.update_block_table(metadata_a, blk_table, slot_mapping) - - # block_idx tensors must live in builder_b's persistent buffers. - def shares_storage(tensor, buffer): - return ( - tensor.untyped_storage().data_ptr() == buffer.untyped_storage().data_ptr() - ) - - assert shares_storage( - metadata_b.block_idx_last_scheduled_token, - builder_b.block_idx_last_scheduled_token, - ), "block_idx_last_scheduled_token not in builder_b's persistent buffer" - - assert shares_storage( - metadata_b.block_idx_last_computed_token, - builder_b.block_idx_last_computed_token, - ), "block_idx_last_computed_token not in builder_b's persistent buffer" - - # Must NOT point to builder_a's buffers. - assert not shares_storage( - metadata_b.block_idx_last_scheduled_token, - builder_a.block_idx_last_scheduled_token, - ), "block_idx_last_scheduled_token still points to builder_a's buffer" - - # Values must be correct (copied from metadata_a). - torch.testing.assert_close( - metadata_b.block_idx_last_scheduled_token, - block_idx_vals, - ) - torch.testing.assert_close( - metadata_b.block_idx_last_computed_token, - block_idx_vals, - ) - - -def test_state_indices_tensor_d_includes_num_speculative_blocks(): - """Regression test for https://github.com/vllm-project/vllm/issues/39809 - bug 1: with mamba_cache_mode='all' and speculative decoding enabled, - the cudagraph buffer for state_indices_tensor_d must allocate the same - per-request column count as the runtime block table, which includes - num_speculative_blocks trailing scratch columns.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 4 - num_speculative_tokens = 1 - num_speculative_blocks = 2 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=num_speculative_blocks, - ) - - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - expected_cols = (max_model_len // block_size) + num_speculative_blocks - assert builder.state_indices_tensor_d.shape == (max_num_seqs, expected_cols) - - -def test_block_idx_cudagraph_capture_padded_by_num_reqs(): - """Regression test for https://github.com/vllm-project/vllm/issues/39809 - bug 2: with mamba_cache_mode='all' and spec decode, _update_metadata_for - _cudagraph_capture must slice block_idx_last_{scheduled,computed}_token - by the request count (padded_bs == num_reqs), not by num_decode_tokens. - Past num_decodes, the slice must be zero-filled.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - num_speculative_tokens = 1 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=2, - ) - - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - builder.block_idx_last_scheduled_token.fill_(-1) - builder.block_idx_last_computed_token.fill_(-1) - - num_decodes = 2 - num_reqs = 3 - num_decode_tokens = num_decodes * (1 + num_speculative_tokens) - seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device) - block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device) - state_indices_d = torch.zeros( - (num_decodes, builder.state_indices_tensor_d.shape[1]), - dtype=torch.int32, - device=device, - ) - query_start_loc_d = torch.arange( - num_decodes + 1, dtype=torch.int32, device=device - ) * (1 + num_speculative_tokens) - num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device) - - metadata = BaseMambaAttentionMetadata( - num_prefills=0, - num_prefill_tokens=0, - num_decodes=num_decodes, - num_decode_tokens=num_decode_tokens, - num_reqs=num_reqs, - has_initial_states_p=None, - query_start_loc_p=None, - num_computed_tokens_p=None, - state_indices_tensor_p=None, - state_indices_tensor_d=state_indices_d, - query_start_loc_d=query_start_loc_d, - num_accepted_tokens=num_accepted_tokens, - block_idx_last_scheduled_token=block_idx_vals, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=block_idx_vals, - block_idx_last_scheduled_token_prev_step=None, - seq_lens=seq_lens, - ) - - out = builder._update_metadata_for_cudagraph_capture(metadata) - - assert out.block_idx_last_scheduled_token.shape == (num_reqs,) - assert out.block_idx_last_computed_token.shape == (num_reqs,) - torch.testing.assert_close( - out.block_idx_last_scheduled_token[:num_decodes], block_idx_vals - ) - torch.testing.assert_close( - out.block_idx_last_computed_token[:num_decodes], block_idx_vals - ) - assert torch.all(out.block_idx_last_scheduled_token[num_decodes:] == 0) - assert torch.all(out.block_idx_last_computed_token[num_decodes:] == 0) - - -def test_block_idx_prev_step_persistent_buffer_allocated(): - """With mamba_cache_mode='all' + spec decode, the builder must allocate - block_idx_last_scheduled_token_prev_step as a persistent buffer with the - same shape as the existing block_idx_last_{scheduled,computed}_token - buffers, so cudagraph capture records a stable pointer for the prev-step - input anchor consumed by mamba_mixer2's input gather.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - num_speculative_tokens = 1 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=2, - ) - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - assert hasattr(builder, "block_idx_last_scheduled_token_prev_step") - assert builder.block_idx_last_scheduled_token_prev_step.shape == (max_num_seqs,) - assert builder.block_idx_last_scheduled_token_prev_step.dtype == torch.int32 - - -def test_block_idx_prev_step_persistent_buffer_skipped_without_spec_decode(): - """Without spec decode, the prev-step buffer is unused and must not be - allocated — the input anchor reduces to last_computed_token.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, max_num_seqs, num_speculative_tokens=0 - ) - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - ) - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - - assert not hasattr(builder, "block_idx_last_scheduled_token_prev_step") - - -def test_block_idx_prev_step_cudagraph_capture_uses_persistent_buffer(): - """_update_metadata_for_cudagraph_capture must copy the prev-step anchor - into the builder's persistent buffer (so cudagraph replay reads from the - same underlying memory), pad past num_decodes with zero, and return a - slice of the persistent buffer in the metadata.""" - block_size = 16 - max_model_len = 256 - max_num_seqs = 8 - num_speculative_tokens = 1 - device = torch.device("cpu") - - vllm_config = _make_vllm_config( - max_model_len, - max_num_seqs, - num_speculative_tokens=num_speculative_tokens, - ) - spec = MambaSpec( - block_size=block_size, - shapes=((1,), (1,)), - dtypes=(torch.float32,), - mamba_cache_mode="all", - num_speculative_blocks=2, - ) - builder = MockMambaBuilder(spec, ["layer0"], vllm_config, device) - builder.block_idx_last_scheduled_token.fill_(-1) - builder.block_idx_last_computed_token.fill_(-1) - builder.block_idx_last_scheduled_token_prev_step.fill_(-1) - - num_decodes = 2 - num_reqs = 3 - num_decode_tokens = num_decodes * (1 + num_speculative_tokens) - seq_lens = torch.full((num_reqs,), 64, dtype=torch.int32, device=device) - block_idx_vals = torch.tensor([3, 5], dtype=torch.int32, device=device) - prev_step_vals = torch.tensor([2, 4], dtype=torch.int32, device=device) - state_indices_d = torch.zeros( - (num_decodes, builder.state_indices_tensor_d.shape[1]), - dtype=torch.int32, - device=device, - ) - query_start_loc_d = torch.arange( - num_decodes + 1, dtype=torch.int32, device=device - ) * (1 + num_speculative_tokens) - num_accepted_tokens = torch.ones(num_decodes, dtype=torch.int32, device=device) - - metadata = BaseMambaAttentionMetadata( - num_prefills=0, - num_prefill_tokens=0, - num_decodes=num_decodes, - num_decode_tokens=num_decode_tokens, - num_reqs=num_reqs, - has_initial_states_p=None, - query_start_loc_p=None, - num_computed_tokens_p=None, - state_indices_tensor_p=None, - state_indices_tensor_d=state_indices_d, - query_start_loc_d=query_start_loc_d, - num_accepted_tokens=num_accepted_tokens, - block_idx_last_scheduled_token=block_idx_vals, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=block_idx_vals, - block_idx_last_scheduled_token_prev_step=prev_step_vals, - seq_lens=seq_lens, - ) - - out = builder._update_metadata_for_cudagraph_capture(metadata) - - # Output field exists and is identity-shared with the persistent buffer. - assert out.block_idx_last_scheduled_token_prev_step is not None - assert ( - out.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr() - == builder.block_idx_last_scheduled_token_prev_step.untyped_storage().data_ptr() - ), ( - "prev-step buffer must live in the builder's persistent buffer, not " - "in the caller-provided tensor" - ) - - # Padded by num_reqs (not num_decode_tokens) — same fix as bug 2 for the - # other block_idx_* fields. - assert out.block_idx_last_scheduled_token_prev_step.shape == (num_reqs,) - - # First num_decodes values: input values copied through. - torch.testing.assert_close( - out.block_idx_last_scheduled_token_prev_step[:num_decodes], - prev_step_vals, - ) - - # Tail values past num_decodes: zero-filled padding for cudagraph capture. - assert torch.all(out.block_idx_last_scheduled_token_prev_step[num_decodes:] == 0) - - -def test_num_state_writes_p_matches_block_idx_tensors(): - """The prefill state save sizes its batched copy from - num_state_writes_p instead of reading the block index tensors back from - the GPU, so the host count must equal what those tensors say.""" - # (block_size, seq_lens, query_lens, is_prefilling) - cases = [ - # block-aligned and unaligned computed tokens, and a chunked prefill - (16, [64, 33, 17], [48, 17, 1], [True, True, True]), - (32, [200, 100, 64], [200, 36, 64], [True, True, True]), - # decode rows ahead of the prefills - (16, [40, 41, 100, 200], [1, 1, 60, 200], [False, False, True, True]), - # prefills that stay inside their first block: nothing to write - (16, [65, 66, 67], [1, 2, 3], [True, True, True]), - ] - - for block_size, seq_lens, query_lens, is_prefilling in cases: - metadata = MockMambaBuilder.build_mamba_metadata( - _make_vllm_config(4096, len(seq_lens), block_size=block_size), - seq_lens=seq_lens, - query_lens=query_lens, - is_prefilling=is_prefilling, - ) - num_reqs, num_prefills = metadata.num_reqs, metadata.num_prefills - block_idx_last_p = metadata.block_idx_last_scheduled_token[ - num_reqs - num_prefills : num_reqs - ] - expected = int( - (block_idx_last_p - metadata.block_idx_first_scheduled_token_p) - .clamp(min=0) - .sum() - ) - - assert metadata.num_state_writes_p == expected, ( - f"seq_lens={seq_lens} query_lens={query_lens} block_size={block_size}" - ) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 722cbfb79e..2451ea5ec0 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -39,6 +39,7 @@ from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.utils.hashing import sha256 from vllm.v1.core.encoder_cache_manager import EncoderCacheManager from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator +from vllm.v1.core.kv_cache_manager import KVCacheManager from vllm.v1.core.kv_cache_utils import get_request_block_hasher, init_none_hash from vllm.v1.core.sched.diffusion_scheduler import ( DiffusionAsyncScheduler, @@ -6751,7 +6752,7 @@ def _create_hybrid_mamba_connector_scheduler( num_blocks: int = 100, supports_divergent_hits: bool = True, ) -> Scheduler: - """FA + Mamba ("all" cache mode) scheduler with a MockKVConnector.""" + """FA + Mamba ("align" cache mode) scheduler with a MockKVConnector.""" model_config = ModelConfig( model="facebook/opt-125m", trust_remote_code=True, @@ -6772,7 +6773,7 @@ def _create_hybrid_mamba_connector_scheduler( cache_config=CacheConfig( block_size=block_size, enable_prefix_caching=True, - mamba_cache_mode="all", + mamba_cache_mode="align", ), kv_transfer_config=KVTransferConfig( kv_connector="MockKVConnector", @@ -6804,7 +6805,7 @@ def _create_hybrid_mamba_connector_scheduler( block_size=block_size, shapes=((1, 1),), dtypes=(torch.float32,), - mamba_cache_mode="all", + mamba_cache_mode="align", ), ), ], @@ -6820,6 +6821,33 @@ def _create_hybrid_mamba_connector_scheduler( ) +def _seed_hybrid_prefix( + manager: KVCacheManager, num_blocks: int, block_size: int +) -> tuple[list[int], list[int]]: + """Prefill a prefix one block per step so that align-mode Mamba caches + the state at every block boundary. Returns the FA and Mamba block ids.""" + [fill] = create_requests( + num_requests=1, + num_tokens=num_blocks * block_size, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["fill"], + ) + computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) + assert num_computed == 0 + mamba_ids = [] + for i in range(num_blocks): + manager.allocate_slots( + fill, block_size, new_computed_blocks=computed_blocks if i == 0 else None + ) + fill.num_computed_tokens += block_size + fa_ids, group_mamba_ids = manager.get_block_ids(fill.request_id) + mamba_ids.append(group_mamba_ids[i]) + manager.free(fill) + return fa_ids, mamba_ids + + @pytest.mark.parametrize( "matched_tokens,expected_num_computed", [ @@ -6845,23 +6873,8 @@ def test_hybrid_per_group_hit_divergence_with_connector( manager = scheduler.kv_cache_manager assert isinstance(manager.coordinator, HybridKVCacheCoordinator) - # Seed a 4-block prefix so both groups cache all four boundaries - # (mamba cache mode "all" caches every block's state densely). - [fill] = create_requests( - num_requests=1, - num_tokens=4 * block_size, - max_tokens=1, - same_prompt=True, - block_size=block_size, - req_ids=["fill"], - ) - computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) - blocks = manager.allocate_slots( - fill, fill.num_tokens, num_computed, computed_blocks - ) - fa_ids = [b.block_id for b in blocks.blocks[0]] - mamba_ids = [b.block_id for b in blocks.blocks[1]] - manager.free(fill) + # Seed a 4-block prefix so both groups cache all four boundaries. + fa_ids, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size) # Evict the FA tail and the middle mamba states; block 0 (both groups) # and the deep mamba state at block 3 survive. @@ -6885,8 +6898,8 @@ def test_hybrid_per_group_hit_divergence_with_connector( scheduler.add_request(replay) output = scheduler.schedule() - num_scheduled = output.num_scheduled_tokens[replay.request_id] - assert replay.num_tokens - num_scheduled == expected_num_computed + [new_req] = output.scheduled_new_reqs + assert new_req.num_computed_tokens == expected_num_computed @pytest.mark.parametrize( @@ -6921,20 +6934,7 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy( assert isinstance(manager.coordinator, HybridKVCacheCoordinator) # Seed a 4-block prefix in both groups. - [fill] = create_requests( - num_requests=1, - num_tokens=4 * block_size, - max_tokens=1, - same_prompt=True, - block_size=block_size, - req_ids=["fill"], - ) - computed_blocks, num_computed, _ = manager.get_computed_blocks(fill) - blocks = manager.allocate_slots( - fill, fill.num_tokens, num_computed, computed_blocks - ) - mamba_ids = [b.block_id for b in blocks.blocks[1]] - manager.free(fill) + _, mamba_ids = _seed_hybrid_prefix(manager, 4, block_size) # Keep all FA blocks; evict every mamba state but block 0. FA reaches 4 # blocks, the mamba hit only reaches 1 -> diverged (FA > Mamba). @@ -6955,8 +6955,8 @@ def test_hybrid_fa_deeper_hit_respects_connector_lookup_policy( scheduler.add_request(replay) output = scheduler.schedule() - num_scheduled = output.num_scheduled_tokens[replay.request_id] - assert replay.num_tokens - num_scheduled == expected_num_computed + [new_req] = output.scheduled_new_reqs + assert new_req.num_computed_tokens == expected_num_computed def _make_encoder_instance_request(scheduler, text_prefix=8, image_tokens=16): diff --git a/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh b/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh index ddd8d06b01..c6f79f3721 100755 --- a/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh +++ b/tests/v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh @@ -60,7 +60,7 @@ vllm serve "$MODEL" \ --block-size 128 \ --trust-remote-code \ --enable-prefix-caching \ - --mamba-cache-mode all \ + --mamba-cache-mode align \ --kv-transfer-config "$KV_CONFIG" \ "${EXTRA_ARGS[@]}" & @@ -78,7 +78,7 @@ vllm serve "$MODEL" \ --block-size 128 \ --trust-remote-code \ --enable-prefix-caching \ - --mamba-cache-mode all \ + --mamba-cache-mode align \ --kv-transfer-config "$KV_CONFIG" \ "${EXTRA_ARGS[@]}" & diff --git a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py index 87ea53225a..28bc8bebe2 100644 --- a/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py +++ b/tests/v1/kv_connector/unit/test_moriio_hma_scheduler.py @@ -67,7 +67,6 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore self._mamba_group_ids: list[int] = [] self._attn_group_ids: list[int] = [0] self._num_ssm_scratch_blocks = 0 - self._ssm_state_slots_are_positional = False self._is_hma_required = False self.kv_cache_config = SimpleNamespace( kv_cache_groups=[None, None], @@ -130,7 +129,7 @@ def _mamba_spec(num_states: int = 2, *, num_speculative_blocks: int = 0): block_size=16, shapes=shapes, dtypes=(torch.float32,) * num_states, - mamba_cache_mode="all", + mamba_cache_mode="align", num_speculative_blocks=num_speculative_blocks, ) @@ -146,7 +145,7 @@ def _gate_vllm_config( block_size: int = 16, num_lookahead_tokens: int = 0, num_speculative_tokens: int = 0, - mamba_cache_mode: str = "all", + mamba_cache_mode: str = "align", dcp_size: int = 1, ): return SimpleNamespace( @@ -193,7 +192,7 @@ def test_split_block_groups_ignores_transfer_disabled_group(): block_size=16, shapes=((1, 1),), dtypes=(torch.float32,), - mamba_cache_mode="all", + mamba_cache_mode="align", ) config = SimpleNamespace( kv_cache_groups=[ @@ -361,7 +360,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention( block_size=4, num_lookahead_tokens=7, num_speculative_tokens=7, - mamba_cache_mode="align", dcp_size=dcp_size, ) scheduler = moriio_connector.MoRIIOConnectorScheduler(vllm_config, "engine", config) @@ -398,21 +396,6 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention( ) -def test_scheduler_rejects_dspark_with_positional_mamba_cache(): - """Positional mode counts lookahead in the Mamba allocation, so the - one-block recurrent-state tail bound no longer holds.""" - with pytest.raises(moriio_common.MoRIIOError, match="mamba_cache_mode"): - moriio_connector.MoRIIOConnectorScheduler( - _gate_vllm_config( - speculative_config=_spec_config("dspark"), - block_size=4, - mamba_cache_mode="all", - ), - "engine", - _dspark_wrapped_attention_config(), - ) - - def test_scheduler_rejects_hybrid_write(): config = SimpleNamespace( kv_cache_groups=[ @@ -441,20 +424,7 @@ def test_split_block_groups_accepts_empty_abort_payload(): assert sched.split_block_groups(()) == ([], []) -def test_split_block_groups_keeps_positional_slots_in_all_mode(): - # mamba_cache_mode="all" keeps a state per block position. - sched = _FakeScheduler( - _has_mamba=True, - _attn_group_ids=[0], - _mamba_group_ids=[1], - _ssm_state_slots_are_positional=True, - ) - attn, mamba = sched.split_block_groups(([1], [40, 41, 42])) - assert attn == [1] - assert mamba == [[40, 41, 42]] - - -def test_split_block_groups_keeps_only_running_state_outside_all_mode(): +def test_split_block_groups_keeps_only_running_state(): sched = _FakeScheduler( _has_mamba=True, _attn_group_ids=[0], @@ -464,24 +434,21 @@ def test_split_block_groups_keeps_only_running_state_outside_all_mode(): @pytest.mark.parametrize( - "positional,blocks,expected", + "blocks,expected", [ - # Trailing scratch slots go in either cache mode. - (True, [40, 41, 42], [40]), - # Outside "all" the running state is picked only after they are gone. - (False, [40, 41, 42, 43], [41]), + # The running state is picked only after the scratch slots are gone. + ([40, 41, 42, 43], [41]), # Never empty: a list shorter than the scratch count keeps one state. - (True, [40], [40]), - (False, [], []), + ([40], [40]), + ([], []), ], ) -def test_split_block_groups_strips_dspark_scratch_slots(positional, blocks, expected): +def test_split_block_groups_strips_dspark_scratch_slots(blocks, expected): sched = _FakeScheduler( _has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1], _num_ssm_scratch_blocks=2, - _ssm_state_slots_are_positional=positional, ) assert sched.split_block_groups(([1], blocks)) == ([1], [expected]) @@ -555,7 +522,6 @@ def _make_read_scheduler(): _has_mamba=True, _attn_group_ids=[0], _mamba_group_ids=[1], - _ssm_state_slots_are_positional=True, request_id_to_transfer_id={}, transfer_id_to_request_id={}, _reqs_need_recv={}, @@ -578,17 +544,17 @@ def _make_read_request(remote_block_ids): def test_update_state_drops_decode_recompute_tail_block(): sched = _make_read_scheduler() - request = _make_read_request([[10, 11], [90, 91]]) + request = _make_read_request([[10, 11], [91]]) blocks = _FakeBlocks( all_groups=([100, 101, 102], [200, 201, 202]), ) sched.update_state_after_alloc(request, blocks, num_external_tokens=256) - assert sched._reqs_need_recv["req"][1] == [[100, 101], [200, 201]] + assert sched._reqs_need_recv["req"][1] == [[100, 101], [202]] assert sched._req_kv_params["req"]["remote_block_ids"] == [ [10, 11], - [90, 91], + [91], ] @@ -609,7 +575,6 @@ def test_update_state_pairs_trimmed_mamba_state_with_remote_state(): """The DSpark shape end to end: scratch slots are stripped, the running state is then selected, and the attention list loses its lookahead tail.""" sched = _make_read_scheduler() - sched._ssm_state_slots_are_positional = False sched._num_ssm_scratch_blocks = 2 sched._max_decode_tail_blocks = 2 request = _make_read_request([[10, 11], [90]]) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 629e8c7ecc..199ac2fe8c 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -856,28 +856,6 @@ def test_apply_prefix_caching_mamba_hybrid( [[6, 7, 8, 9], [99]], id="fa_prefix_hit_and_ssm_trim", ), - # Multi-slot SSM ("all" mode): a local prefix hit leaves fewer local - # slots; the earlier remote slots are covered locally → remote tail. - pytest.param( - 10, - 10, - [list(range(10)), [5, 6]], - [list(range(10)), [1, 2, 3]], - [list(range(10)), [5, 6]], - [list(range(10)), [2, 3]], - id="ssm_multi_block_local_hit_tail", - ), - # Multi-slot SSM ("all" mode): the one trailing local position holds - # the token D recomputes itself → local head-clip. - pytest.param( - 10, - 10, - [list(range(10)), [4, 5, 6]], - [list(range(10)), [8, 9]], - [list(range(10)), [4, 5]], - [list(range(10)), [8, 9]], - id="ssm_multi_block_local_extra_head_clip", - ), ], ) def test_apply_prefix_caching_ssm_prefix_cache_hit( @@ -2034,13 +2012,12 @@ def test_logical_to_kernel_block_ids_with_remote_ratio( @pytest.mark.cpu_test def test_exchange_clipped_blocks_ssm_single_state(): - """In single-state cache modes, SSM lists are reduced to the running - state slot: speculative scratch slots, null placeholders and the previous - step's state carry nothing. Attention groups pass through untouched.""" + """SSM lists are reduced to the running state slot: speculative scratch + slots, null placeholders and the previous step's state carry nothing. + Attention groups pass through untouched.""" sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True) sched.blocks_per_sw = [0, 0] sched._ssm_spec_blocks = [None, 2] - sched._ssm_state_slots_are_positional = False # Align-mode list: null placeholders, state block, 2 speculative slots. clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 0, 7, 8, 9])) @@ -2069,19 +2046,6 @@ def test_exchange_clipped_blocks_ssm_single_state(): assert fa_sched.get_exchange_clipped_blocks(([1, 2],)) == ([1, 2],) -@pytest.mark.cpu_test -def test_exchange_clipped_blocks_ssm_positional_states(): - """In "all" mode every position holds a state, so only the speculative - slots go; placeholders stay to keep the list position-indexed.""" - sched = make_nixl_scheduler(has_mamba=True, is_hma_required=True) - sched.blocks_per_sw = [0, 0] - sched._ssm_spec_blocks = [None, 2] - sched._ssm_state_slots_are_positional = True - - clipped = sched.get_exchange_clipped_blocks(([1, 2, 3], [0, 5, 6, 7, 8, 9])) - assert clipped == ([1, 2, 3], [0, 5, 6, 7]) - - # ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ───────────────────── diff --git a/tests/v1/kv_connector/unit/test_offloading_connector.py b/tests/v1/kv_connector/unit/test_offloading_connector.py index db7ecca621..76d2197469 100644 --- a/tests/v1/kv_connector/unit/test_offloading_connector.py +++ b/tests/v1/kv_connector/unit/test_offloading_connector.py @@ -566,10 +566,7 @@ def test_fs_tiering_offloading(tmp_path) -> None: ("state-spaces/mamba-1.4b-hf", 16, 1), ], ) -@pytest.mark.parametrize("mamba_cache_mode", ["align", "all"]) -def test_mamba_cpu_offload_boundary( - model: str, block_size: int, tp_size: int, mamba_cache_mode: str -): +def test_mamba_cpu_offload_boundary(model: str, block_size: int, tp_size: int): kv_transfer_config = KVTransferConfig( kv_connector="OffloadingConnector", kv_role="kv_both", @@ -586,7 +583,7 @@ def test_mamba_cpu_offload_boundary( kv_transfer_config=kv_transfer_config, language_model_only=True, enable_prefix_caching=True, - mamba_cache_mode=mamba_cache_mode, + mamba_cache_mode="align", # Use lossless state storage so this exact-equality regression isolates # offload boundary selection from low-precision Mamba checkpointing. mamba_cache_dtype="float32", diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index f154816369..2febe3109f 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -474,7 +474,7 @@ def make_kv_cache_config( mamba_enabled: bool = False, sw_size: int = 128, num_blocks: int = 100, - mamba_cache_mode: Literal["all", "align", "none"] = "none", + mamba_cache_mode: Literal["align", "none"] = "none", ) -> KVCacheConfig: kv_cache_groups = [ KVCacheGroupSpec( diff --git a/tests/v1/simple_kv_offload/test_scheduler.py b/tests/v1/simple_kv_offload/test_scheduler.py index 8a85c88d70..1c0cd74565 100644 --- a/tests/v1/simple_kv_offload/test_scheduler.py +++ b/tests/v1/simple_kv_offload/test_scheduler.py @@ -2335,7 +2335,7 @@ def test_hybrid_store_uses_resolved_group_block_sizes() -> None: # of the hash block, so it cannot exceed the smallest of them. hash_block_size=BLOCK_SIZE, dcp_world_size=dcp_world_size, - mamba_cache_mode="all", + mamba_cache_mode="none", ) sched = fix.scheduler gpu_pool = fix.gpu_block_pool @@ -2747,55 +2747,6 @@ def test_eager_store_does_not_scan_mamba_blocks_positionally() -> None: assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 0] -def test_eager_store_scans_mamba_all_blocks_positionally() -> None: - """Mamba all-mode retains positional identity and needs no handoff.""" - block_size = 4 * BLOCK_SIZE - fix = _make_hybrid_attention_mamba_scheduler( - num_cpu_blocks=16, - num_gpu_blocks=24, - block_size=block_size, - mamba_cache_mode="all", - ) - sched = fix.scheduler - gpu_pool = fix.gpu_block_pool - attention_manager, mamba_manager = sched.cpu_coordinator.single_type_managers - assert attention_manager.has_positionally_stable_blocks - assert mamba_manager.has_positionally_stable_blocks - - req = _make_cp_request(num_blocks=2, virtual_block_size=block_size) - attention_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=0 - ) - mamba_blocks = _allocate_cp_gpu_blocks( - gpu_pool, req, 2, virtual_block_size=block_size, group_id=1 - ) - kv_blocks = KVCacheBlocks(blocks=(attention_blocks, mamba_blocks)) - sched.update_state_after_alloc(req, kv_blocks, num_external_tokens=0) - - # build_connector_meta runs before the scheduler commits this step's - # tokens, so the positional scan picks the blocks up on the next step. - first = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 2 * block_size}, - new_reqs={req.request_id: kv_blocks.get_block_ids()}, - ) - ) - assert first.store_gpu_blocks == [] - req.num_computed_tokens = 2 * block_size - meta = sched.build_connector_meta( - make_scheduler_output( - {req.request_id: 1}, - cached_req_new_blocks={req.request_id: None}, - ) - ) - - assert set(meta.store_gpu_blocks) == { - *(block.block_id for block in attention_blocks), - *(block.block_id for block in mamba_blocks), - } - assert sched._reqs_to_store[req.request_id].num_stored_blocks == [2, 2] - - @pytest.mark.parametrize("gone_via", ["preempted", "finished", "unregistered"]) def test_boundary_handoff_dropped_for_departing_request(gone_via: str) -> None: """Handoffs for a request leaving this step must not be read. diff --git a/tests/v1/worker/test_gpu_warmup_blocks.py b/tests/v1/worker/test_gpu_warmup_blocks.py index 067f31e935..04179ae760 100644 --- a/tests/v1/worker/test_gpu_warmup_blocks.py +++ b/tests/v1/worker/test_gpu_warmup_blocks.py @@ -189,7 +189,7 @@ def test_mixed_warmup_reserves_lookahead_blocks(): _assert_covers_lookahead(recorder.steps, num_lookahead_tokens) -@pytest.mark.parametrize("mamba_cache_mode", ["none", "all", "align"]) +@pytest.mark.parametrize("mamba_cache_mode", ["none", "align"]) def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode): """Mamba groups hold the running-state block plus the speculative tail. @@ -227,14 +227,7 @@ def test_warmup_reserves_mamba_speculative_blocks(mamba_cache_mode): def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: - """Attention, circular, and Mamba groups exercise every reservation branch. - - "none" and "all" reach the same branch of both `_reserved_block_count` and - `MambaManager`, which tests only for "align". They are still both listed: - the mode is a spec-level input, and having the real manager confirm the - prediction for each is what keeps a future divergence between them from - landing unnoticed. - """ + """Attention, circular, and Mamba groups exercise every reservation branch.""" return KVCacheConfig( num_blocks=num_blocks, kv_cache_tensors=[], @@ -242,7 +235,6 @@ def _hybrid_kv_cache_config(num_blocks: int) -> KVCacheConfig: _attention_group(), _circular_group(), _mamba_group("none"), - _mamba_group("all"), _mamba_group("align"), ], ) diff --git a/tests/v1/worker/test_mamba_utils.py b/tests/v1/worker/test_mamba_utils.py index 47a27795ca..50faf0fb8b 100644 --- a/tests/v1/worker/test_mamba_utils.py +++ b/tests/v1/worker/test_mamba_utils.py @@ -398,7 +398,7 @@ def _make_kv_cache_config(cfg: _TestConfig, layer_names: list[str]) -> KVCacheCo (cfg.temporal_state_dim,), ), dtypes=(cfg.dtype, cfg.dtype), - mamba_cache_mode="all", + mamba_cache_mode="align", ) group = KVCacheGroupSpec( layer_names=layer_names, diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 13665f3cff..92b984c0fa 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -59,7 +59,7 @@ CacheDType = Literal[ MambaDType = Literal["auto", "float32", "float16", "bfloat16"] -MambaCacheMode = Literal["all", "align", "none"] +MambaCacheMode = Literal["align", "none"] PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] KVOffloadingBackend = Literal["native", "lmcache"] @@ -75,8 +75,6 @@ class CacheConfig: Accepts None (meaning "use default"). After construction, always int.""" user_specified_block_size: bool = field(default=False, init=False) """Whether block_size was explicitly provided. Derived automatically.""" - user_specified_mamba_block_size: bool = field(default=False, init=False) - """Whether mamba_block_size was explicitly provided. Derived automatically.""" kv_cache_layout: str | None = field(default=None, init=False) """Resolved physical KV cache layout name (a ``KVCacheLayout`` member). @@ -191,7 +189,6 @@ class CacheConfig: """The cache strategy for Mamba layers: - "none": set when prefix caching is disabled. - - "all": cache the mamba state of all tokens at position i * block_size. - "align": only cache the mamba state of the last token of each scheduler step and when the token is at position i * block_size. This is the default when prefix caching is enabled. @@ -293,7 +290,6 @@ class CacheConfig: "mamba_page_size_padded", "skip_page_size_padded", "user_specified_block_size", - "user_specified_mamba_block_size", "_block_size_resolved", # Post-init/derived counters "num_gpu_blocks", @@ -337,21 +333,8 @@ class CacheConfig: self.block_size = self.DEFAULT_BLOCK_SIZE else: self.user_specified_block_size = True - if self.mamba_block_size is not None: - self.user_specified_mamba_block_size = True return self - @field_validator("mamba_cache_mode", mode="after") - @classmethod - def _validate_mamba_cache_mode(cls, mode: MambaCacheMode) -> MambaCacheMode: - if mode == "all": - logger.warning_once( - "Mamba cache mode 'all' is deprecated and will be removed in an " - "upcoming release. If this is a problem, please open an issue " - "at https://github.com/vllm-project/vllm/issues." - ) - return mode - @field_validator("cache_dtype", mode="after") @classmethod def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType: diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index c25ed27a28..281666a633 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -3091,9 +3091,6 @@ class VllmConfig: if self.parallel_config.use_ubatching: unsupported.extend(self._get_dbo_unsupported_features()) - if self.cache_config.mamba_cache_mode == "all": - unsupported.append("mamba cache mode 'all'") - return unsupported def _get_v1_model_runner_unsupported_features(self) -> list[str]: @@ -3499,10 +3496,6 @@ class VllmConfig: "SSM state caches, not --enable-mamba-cache-stochastic-" "rounding, which requires an explicit float16 cache" ) - if self.cache_config.mamba_cache_mode not in ("none", "align"): - raise ValueError( - "RecoverSSM supports only none and align Mamba cache modes" - ) if ( self.cache_config.mamba_cache_mode == "align" and not self.use_v2_model_runner @@ -3535,11 +3528,6 @@ class VllmConfig: "Mamba2 ReplaySSM speculative decoding requires " "--mamba-backend flashinfer" ) - elif self.cache_config.mamba_cache_mode == "all": - raise ValueError( - "--use-replayssm supports prefix caching only in align mode; " - "pass --mamba-cache-mode align" - ) elif self.mamba_config.backend == MambaBackendEnum.FLASHINFER: if self.cache_config.mamba_cache_mode == "align": raise ValueError( diff --git a/vllm/distributed/kv_transfer/kv_connector/utils.py b/vllm/distributed/kv_transfer/kv_connector/utils.py index 30021cde50..a0ead27c2f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/utils.py @@ -33,16 +33,14 @@ EngineId = str BlockIds = tuple[list[int], ...] | list[list[int]] -def clip_ssm_state_blocks( - blocks: list[int], num_spec_blocks: int, positional: bool -) -> list[int]: - """Drop speculative scratch slots, retaining position-indexed or running state.""" +def clip_ssm_state_blocks(blocks: list[int], num_spec_blocks: int) -> list[int]: + """Drop speculative scratch slots and keep only the running-state slot.""" if not blocks: return blocks # Keep at least the running-state slot, even before scratch allocation. if num_scratch := min(num_spec_blocks, len(blocks) - 1): blocks = blocks[:-num_scratch] - return blocks if positional else blocks[-1:] + return blocks[-1:] def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index b482e6a8c4..ec24f0d199 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -450,13 +450,6 @@ def _validate_hybrid_speculation(vllm_config: VllmConfig) -> None: "MoRIIO hybrid READ supports DSpark speculative decoding only, got " f"method={speculative_config.method!r}" ) - if vllm_config.cache_config.mamba_cache_mode == "all": - # Positional allocation counts lookahead, so its local tail can exceed - # the single running-state block supported by hybrid READ. - raise MoRIIOError( - "MoRIIO hybrid READ does not support DSpark with " - "mamba_cache_mode='all'; use 'align'" - ) def _validate_mamba_specs(specs: Collection[MambaSpec]) -> MambaSpec | None: @@ -521,11 +514,6 @@ class MoRIIOConnectorScheduler: "MoRIIO hybrid (mamba/KDA) transfer is implemented for READ " "mode only; set kv_connector_extra_config.read_mode=true" ) - # Only "all" mode keeps a state per block position; the other modes keep - # a single running state in the last slot. - self._ssm_state_slots_are_positional = ( - vllm_config.cache_config.mamba_cache_mode == "all" - ) self._num_ssm_scratch_blocks = ( mamba_spec.num_speculative_blocks if mamba_spec is not None else 0 ) @@ -1288,8 +1276,8 @@ class MoRIIOConnectorScheduler: """Select transferable attention and Mamba block groups. The wire payload stores the attention group first, followed by every - Mamba group in transfer-group order. Outside ``mamba_cache_mode="all"``, - only each group's running-state slot is transferred. + Mamba group in transfer-group order. Only each group's running-state + slot is transferred. """ if not block_ids: return [], [] @@ -1301,7 +1289,6 @@ class MoRIIOConnectorScheduler: clip_ssm_state_blocks( list(transfer_block_ids[group_id]), self._num_ssm_scratch_blocks, - self._ssm_state_slots_are_positional, ) for group_id in self._mamba_group_ids ] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py index 0705f944f6..92f0cbc58a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -154,11 +154,6 @@ class NixlBaseConnectorScheduler: else None for g in kv_cache_config.transfer_groups ] - # Only "all" mode keeps a state per block position; the other modes - # keep a single running state in the last non-speculative slot. - self._ssm_state_slots_are_positional = ( - vllm_config.cache_config.mamba_cache_mode == "all" - ) # Threshold to decide whether to compute kv cache locally # or pull from a remote node: minimum number of remote @@ -258,11 +253,9 @@ class NixlBaseConnectorScheduler: out-of-window blocks only prior to the `request_finished_all_groups` hook. - SSM groups keep only their state-bearing slots: the trailing - speculative scratch slots always go, and in single-state cache modes - so does everything before the running state (null placeholders and - the previous step's superseded state). "all" mode keeps its remaining - slots, which the worker pairs position-wise. + SSM groups keep only their state-bearing slot: the trailing + speculative scratch slots and everything before the running state + (null placeholders and the previous step's superseded state) go. Use this at every block-id exchange point. Pass ``clip_ssm=False`` for per-step partial lists (host-buffer save), where the SSM strip @@ -289,9 +282,7 @@ class NixlBaseConnectorScheduler: and blocks and (n_spec_blocks := self._ssm_spec_blocks[i]) is not None ): - blocks = clip_ssm_state_blocks( - blocks, n_spec_blocks, self._ssm_state_slots_are_positional - ) + blocks = clip_ssm_state_blocks(blocks, n_spec_blocks) clipped.append(blocks) return tuple(clipped) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 0cbdd786d8..1e22180b3d 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -3510,13 +3510,12 @@ class NixlBaseConnectorWorker: if _is_ssm_spec(self._group_spec_types[i]): if num_decode_blocks == num_prefill_blocks: continue - # Only state-bearing slots reach here, single-state modes - # just one (see get_exchange_clipped_blocks), so differing - # counts mean position-indexed "all"-mode lists. A longer - # prefill list carries earlier positions the decode side - # already has (prefix hit) -> take its tail; a longer decode - # list holds the position D recomputes itself, which gets - # no prefill state. + # Only state-bearing slots reach here, normally just one + # (see get_exchange_clipped_blocks). A longer prefill list + # carries earlier positions the decode side already has + # (prefix hit) -> take its tail; a longer decode list holds + # the position D recomputes itself, which gets no prefill + # state. assert num_decode_blocks - num_prefill_blocks <= 1, ( f"Group {i}: unpairable SSM state slots, " f"decode={num_decode_blocks} prefill={num_prefill_blocks}" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 47aa8654fe..64b3ad5bb2 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -156,7 +156,7 @@ def resolve_mamba_align_size( """Scan all KV cache groups in *spec* and return the single mamba alignment size, or None if no group requires mamba alignment. - For MambaSpec groups in "align" or "all" cache mode the hit window must be + For MambaSpec groups in "align" cache mode the hit window must be rounded down to a multiple of the offloaded chunk size. Asserts that all such groups agree on the same value. """ @@ -164,10 +164,7 @@ def resolve_mamba_align_size( for group in spec.config.groups: tokens_per_block = group.tokens_per_block kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec - if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in ( - "align", - "all", - ): + if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align": tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk assert mamba_align_size is None or mamba_align_size == tokens_per_chunk mamba_align_size = tokens_per_chunk diff --git a/vllm/model_executor/layers/mamba/mamba_mixer.py b/vllm/model_executor/layers/mamba/mamba_mixer.py index 2b1b1f9c9d..8f3b8a2638 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer.py @@ -260,8 +260,6 @@ class MambaMixer(MambaBase, PluggableLayer): attn_metadata_raw = forward_context.attn_metadata assert self.cache_config is not None - mamba_block_size = self.cache_config.mamba_block_size - is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" attn_metadata: AttentionMetadata | None = None if attn_metadata_raw is not None: @@ -297,8 +295,6 @@ class MambaMixer(MambaBase, PluggableLayer): num_prefill_tokens = attn_metadata.num_prefill_tokens # token count num_decode_tokens = attn_metadata.num_decode_tokens - num_prefills = attn_metadata.num_prefills # request count - num_decodes = attn_metadata.num_decode_tokens # token count (=request) has_prefill = num_prefill_tokens > 0 has_decode = num_decode_tokens > 0 num_actual_tokens = num_prefill_tokens + num_decode_tokens @@ -314,34 +310,6 @@ class MambaMixer(MambaBase, PluggableLayer): gate_p = prefill_decode_split.gate_p gate_d = prefill_decode_split.gate_d - if is_mamba_cache_all: - block_idx_last_computed_token_d, block_idx_last_computed_token_p = ( - torch.split( - attn_metadata.block_idx_last_computed_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = ( - torch.split( - attn_metadata.block_idx_last_scheduled_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - - block_idx_first_scheduled_token_p = ( - attn_metadata.block_idx_first_scheduled_token_p - ) - num_computed_tokens_p = attn_metadata.num_computed_tokens_p - else: - block_idx_last_computed_token_d = None - block_idx_last_computed_token_p = None - block_idx_last_scheduled_token_d = None - block_idx_last_scheduled_token_p = None - block_idx_first_scheduled_token_p = None - num_computed_tokens_p = None - ssm_outputs = [] if has_prefill: @@ -355,11 +323,6 @@ class MambaMixer(MambaBase, PluggableLayer): has_initial_state=has_initial_states_p, cache_indices=state_indices_tensor_p, query_start_loc=query_start_loc_p, - block_idx_first_scheduled_token=block_idx_first_scheduled_token_p, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_p, - initial_state_idx=block_idx_last_computed_token_p, - num_computed_tokens=num_computed_tokens_p, - block_size_to_align=mamba_block_size, metadata=attn_metadata, ) # 3. State Space Model sequence transformations. @@ -383,10 +346,6 @@ class MambaMixer(MambaBase, PluggableLayer): cache_indices=state_indices_tensor_p, has_initial_state=has_initial_states_p, query_start_loc=query_start_loc_p, - block_size=mamba_block_size, - block_idx_first_scheduled_token=block_idx_first_scheduled_token_p, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_p, - initial_state_idx=block_idx_last_computed_token_p, cu_chunk_seqlen=cu_chunk_seqlen_p, last_chunk_indices=last_chunk_indices_p, ) @@ -396,16 +355,6 @@ class MambaMixer(MambaBase, PluggableLayer): # state_indices_tensor_d is assigned when attn_metadata is not None, # and has_decode is only True when attn_metadata is not None assert state_indices_tensor_d is not None - if is_mamba_cache_all: - state_indices_tensor_d_input = state_indices_tensor_d.gather( - 1, block_idx_last_computed_token_d.unsqueeze(1) - ).squeeze(1) - state_indices_tensor_d_output = state_indices_tensor_d.gather( - 1, block_idx_last_scheduled_token_d.unsqueeze(1) - ).squeeze(1) - else: - state_indices_tensor_d_input = state_indices_tensor_d - state_indices_tensor_d_output = state_indices_tensor_d # 2. Convolution sequence transformation conv_out_d = causal_conv1d_update( hidden_states_BC_d.transpose(0, 1), @@ -414,8 +363,6 @@ class MambaMixer(MambaBase, PluggableLayer): self.conv1d.bias, self.activation, conv_state_indices=state_indices_tensor_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_d, - initial_state_idx=block_idx_last_computed_token_d, ).transpose(0, 1) # 3. State Space Model sequence transformation. @@ -437,8 +384,8 @@ class MambaMixer(MambaBase, PluggableLayer): time_proj_bias, z=gate_d.transpose(0, 1), dt_softplus=True, - state_batch_indices=state_indices_tensor_d_input, - dst_state_batch_indices=state_indices_tensor_d_output, + state_batch_indices=state_indices_tensor_d, + dst_state_batch_indices=state_indices_tensor_d, out=scan_outputs_d, ) scan_outputs_d = scan_outputs_d.transpose(0, 1) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index fc58aed91c..a67480ebc2 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -549,12 +549,6 @@ class MambaMixer2(MambaBase, PluggableLayer): self._updates_replayssm_trackers = True self.num_spec = vllm_config.num_speculative_tokens - if self.num_spec > 0: - self.register_buffer( - "_decode_state_offsets", - torch.arange(1 + self.num_spec, dtype=torch.int32).unsqueeze(0), - persistent=False, - ) # Pre-compute sizes for forward pass self.tped_intermediate_size = self.intermediate_size // self.tp_size @@ -732,8 +726,6 @@ class MambaMixer2(MambaBase, PluggableLayer): attn_metadata_raw = forward_context.attn_metadata assert self.cache_config is not None - mamba_block_size = self.cache_config.mamba_block_size - is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" use_spec_decode = self.num_spec > 0 ring_start = prev_num_accepted = prev_query_len = None @@ -803,45 +795,6 @@ class MambaMixer2(MambaBase, PluggableLayer): dim=0, ) - if is_mamba_cache_all: - # If prefix caching is enabled, retrieve the relevant variables - # for prefill and decode - block_idx_last_computed_token_d, block_idx_last_computed_token_p = ( - torch.split( - attn_metadata.block_idx_last_computed_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - block_idx_last_scheduled_token_d, block_idx_last_scheduled_token_p = ( - torch.split( - attn_metadata.block_idx_last_scheduled_token, - [num_decodes, num_prefills], - dim=0, - ) - ) - if attn_metadata.block_idx_last_scheduled_token_prev_step is not None: - block_idx_last_scheduled_token_prev_step_d, _ = torch.split( - attn_metadata.block_idx_last_scheduled_token_prev_step, - [num_decodes, num_prefills], - dim=0, - ) - else: - block_idx_last_scheduled_token_prev_step_d = None - # Prefill-only variables: - block_idx_first_scheduled_token_p = ( - attn_metadata.block_idx_first_scheduled_token_p - ) - num_computed_tokens_p = attn_metadata.num_computed_tokens_p - else: - block_idx_last_computed_token_p = None - block_idx_last_scheduled_token_p = None - block_idx_first_scheduled_token_p = None - block_idx_last_scheduled_token_d = None - block_idx_last_computed_token_d = None - block_idx_last_scheduled_token_prev_step_d = None - num_computed_tokens_p = None - preallocated_ssm_out_d, preallocated_ssm_out_p = torch.split( output[:num_actual_tokens], [num_decode_tokens, num_prefill_tokens], @@ -858,11 +811,6 @@ class MambaMixer2(MambaBase, PluggableLayer): # to by "state_indices_tensor_p". # In particular, it will always write the state at the # sequence end. - # In addition, "block_idx_first_scheduled_token_p" and - # "block_idx_last_scheduled_token_p" - # are provided (which are pointers into - # "state_indices_tensor_p"), it will write additional cache - # states aligned at "block_size_to_align". x = hidden_states_B_C_p.transpose( 0, 1 ) # this is the form that causal-conv see @@ -874,11 +822,6 @@ class MambaMixer2(MambaBase, PluggableLayer): conv_states=conv_state, has_initial_state=has_initial_states_p, cache_indices=state_indices_tensor_p, - block_idx_first_scheduled_token=block_idx_first_scheduled_token_p, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_p, - initial_state_idx=block_idx_last_computed_token_p, - num_computed_tokens=num_computed_tokens_p, - block_size_to_align=mamba_block_size, metadata=attn_metadata, query_start_loc=query_start_loc_p, ).transpose(0, 1)[:num_prefill_tokens] @@ -891,14 +834,9 @@ class MambaMixer2(MambaBase, PluggableLayer): initial_states = None if has_initial_states_p is not None and prep_initial_states: assert state_indices_tensor_p is not None - kernel_ssm_indices = state_indices_tensor_p - if is_mamba_cache_all: - kernel_ssm_indices = state_indices_tensor_p.gather( - 1, block_idx_last_computed_token_p.unsqueeze(1) - ).squeeze(1) initial_states = torch.where( has_initial_states_p[:, None, None, None], - ssm_state[kernel_ssm_indices], + ssm_state[state_indices_tensor_p], 0, ) @@ -921,130 +859,29 @@ class MambaMixer2(MambaBase, PluggableLayer): cu_chunk_seqlens=cu_chunk_seqlen_p, last_chunk_indices=last_chunk_indices_p, initial_states=initial_states, - return_intermediate_states=is_mamba_cache_all, dt_softplus=True, dt_limit=(0.0, float("inf")), out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim), state_dtype=ssm_state.dtype, ) - if is_mamba_cache_all: - assert mamba_block_size is not None - assert state_indices_tensor_p is not None - assert block_idx_first_scheduled_token_p is not None - assert block_idx_last_scheduled_token_p is not None - assert last_chunk_indices_p is not None - assert num_computed_tokens_p is not None - - # The chunk_stride is the number of chunks per mamba block - # e.g., if mamba_block_size = 512 and chunk_size = 256, - # then chunk_stride = 2 - chunk_stride = mamba_block_size // chunk_size - - # Save state for sequences with more than just final state, - # batched over sequences so that the number of kernel launches - # does not scale with the number of prefills. - n_blocks_to_fill = ( - block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p - ).clamp(min=0) - total_fill = attn_metadata.num_state_writes_p - if total_fill > 0: - first_chunk = torch.zeros_like(last_chunk_indices_p) - first_chunk[1:] = last_chunk_indices_p[:-1] + 1 - - # The chunk that completes this sequence's first mamba block. - # A block spans chunk_stride chunks. When the sequence resumed - # mid-block, the chunks before this step are not in varlen_states, - # so subtract them: the boundary sits earlier by their count. - num_unaligned_computed_tokens = ( - num_computed_tokens_p % mamba_block_size - ) - unaligned_chunk_shift = num_unaligned_computed_tokens // chunk_size - first_aligned_chunk = ( - first_chunk + (chunk_stride - 1) - unaligned_chunk_shift - ) - - # Flatten all writes into one list: fill_counts=[2,1,2] - # -> seq_ids=[0,0,1,2,2], write_offsets=[0,1,0,0,1] - fill_counts = n_blocks_to_fill.long() - # output_size, or repeat_interleave reads the size off the device - seq_ids = torch.repeat_interleave( - torch.arange(num_prefills, device=ssm_state.device), - fill_counts, - output_size=total_fill, - ) - write_starts = torch.cumsum(fill_counts, dim=0) - fill_counts - write_offsets = torch.arange( - total_fill, device=ssm_state.device - ) - torch.repeat_interleave( - write_starts, fill_counts, output_size=total_fill - ) - - cache_blocks_to_fill = state_indices_tensor_p[ - seq_ids, - block_idx_first_scheduled_token_p[seq_ids] + write_offsets, - ] - # Write the states. Each row goes to its own cache block, - # so the indexed copy has no duplicate destinations. - ssm_state[cache_blocks_to_fill] = varlen_states[ - first_aligned_chunk[seq_ids] + write_offsets * chunk_stride - ] - - # For all seqs, store the last state (note: might be partial): - assert state_indices_tensor_p is not None - ssm_state[ - state_indices_tensor_p.gather( - 1, block_idx_last_scheduled_token_p.unsqueeze(1) - ).squeeze(1) - ] = varlen_states[last_chunk_indices_p] - - else: - # update ssm states - # - varlen state is a (num_prefills, nheads, headdim, dstate) - # tensor - assert state_indices_tensor_p is not None - ssm_state[state_indices_tensor_p] = varlen_states - if ring_start is not None and self._updates_replayssm_trackers: - assert prev_num_accepted is not None - reset_replayssm_ring_trackers( - ring_start, - prev_num_accepted, - prev_query_len, - state_indices_tensor_p, - ) + # update ssm states + # - varlen state is a (num_prefills, nheads, headdim, dstate) + # tensor + assert state_indices_tensor_p is not None + ssm_state[state_indices_tensor_p] = varlen_states + if ring_start is not None and self._updates_replayssm_trackers: + assert prev_num_accepted is not None + reset_replayssm_ring_trackers( + ring_start, + prev_num_accepted, + prev_query_len, + state_indices_tensor_p, + ) # Process decode requests if has_decode: assert state_indices_tensor_d is not None - if is_mamba_cache_all: - if use_spec_decode: - assert block_idx_last_scheduled_token_prev_step_d is not None - input_indices = ( - block_idx_last_scheduled_token_prev_step_d.unsqueeze(1) - + self._decode_state_offsets - ) - output_indices = ( - block_idx_last_scheduled_token_d.unsqueeze(1) - + self._decode_state_offsets - ) - state_indices_tensor_d_input = state_indices_tensor_d.gather( - 1, input_indices - ) - state_indices_tensor_d_output = state_indices_tensor_d.gather( - 1, output_indices - ) - else: - state_indices_tensor_d_input = state_indices_tensor_d.gather( - 1, block_idx_last_computed_token_d.unsqueeze(1) - ).squeeze(1) - state_indices_tensor_d_output = state_indices_tensor_d.gather( - 1, block_idx_last_scheduled_token_d.unsqueeze(1) - ).squeeze(1) - else: - # Without caching, read and write in-place to the same blocks: - state_indices_tensor_d_input = state_indices_tensor_d - state_indices_tensor_d_output = state_indices_tensor_d - if ( self.use_replayssm and self.mamba_config.backend == MambaBackendEnum.FLASHINFER @@ -1081,8 +918,6 @@ class MambaMixer2(MambaBase, PluggableLayer): self.conv1d.bias, self.activation, conv_state_indices=state_indices_tensor_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token_d, - initial_state_idx=block_idx_last_computed_token_d, num_accepted_tokens=num_accepted_tokens, query_start_loc=query_start_loc_d, # ReplaySSM keeps one physical state block while a speculative @@ -1207,7 +1042,7 @@ class MambaMixer2(MambaBase, PluggableLayer): write_pos=attn_metadata.write_pos_d, is_flush=attn_metadata.is_flush_d, max_cache_len=self.replayssm_buffer_len, - state_batch_indices=state_indices_tensor_d_input, + state_batch_indices=state_indices_tensor_d, out=preallocated_ssm_out_d, enable_stochastic_rounding=( self.mamba_config.enable_stochastic_rounding @@ -1227,8 +1062,8 @@ class MambaMixer2(MambaBase, PluggableLayer): D_d, dt_bias, dt_softplus=True, - state_batch_indices=state_indices_tensor_d_input, - dst_state_batch_indices=state_indices_tensor_d_output, + state_batch_indices=state_indices_tensor_d, + dst_state_batch_indices=state_indices_tensor_d, out=preallocated_ssm_out_d, num_accepted_tokens=num_accepted_tokens, cu_seqlens=query_start_loc_d, diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 3c87b06167..8831c5f6c4 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -646,20 +646,9 @@ class MambaModelConfig(VerifyAndUpdateConfig): cache_config.mamba_cache_mode, model_config.architecture, ) - if ( - cache_config.mamba_cache_mode == "all" - and not model_config.supports_mamba_prefix_caching - ): - cache_config.mamba_cache_mode = "align" - logger.warning( - "Hybrid or mamba-based model detected without support " - "for prefix caching with Mamba cache 'all' mode: " - "falling back to 'align' mode." - ) - if cache_config.mamba_cache_mode == "align": - assert vllm_config.scheduler_config.enable_chunked_prefill, ( - "Chunked prefill is required for mamba cache mode 'align'." - ) + assert vllm_config.scheduler_config.enable_chunked_prefill, ( + "Chunked prefill is required for mamba cache mode 'align'." + ) # By default, mamba block size will be set to max_model_len (see # below). When enabling prefix caching, we align mamba block size # to the block size as the basic granularity for prefix caching. diff --git a/vllm/model_executor/models/interns2_mobius.py b/vllm/model_executor/models/interns2_mobius.py index e186dc575b..4ff312515b 100644 --- a/vllm/model_executor/models/interns2_mobius.py +++ b/vllm/model_executor/models/interns2_mobius.py @@ -418,12 +418,6 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLMBase): self.quant_config = vllm_config.quant_config self.scheduler_config = vllm_config.scheduler_config - if vllm_config.cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Intern-S2-Mobius does not support 'all' prefix caching; " - "use '--mamba-cache-mode=align' instead." - ) - nn.Module.__init__(self) self.config = config self.model = InternS2MobiusModel( diff --git a/vllm/model_executor/models/lfm2.py b/vllm/model_executor/models/lfm2.py index c43a9dcbca..be6dfbd3e4 100644 --- a/vllm/model_executor/models/lfm2.py +++ b/vllm/model_executor/models/lfm2.py @@ -458,13 +458,6 @@ class Lfm2ForCausalLM( def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Lfm2 currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - super().__init__() self.config = config self.model = Lfm2Model( diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 29bc56a696..f6e819c263 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -573,13 +573,6 @@ class Lfm2MoeForCausalLM( def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - cache_config = vllm_config.cache_config - - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Lfm2Moe currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) super().__init__() self.config = config diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 2f5e01f7ae..32660be515 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -329,14 +329,8 @@ class Qwen3_5ForCausalLMBase( config = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config self.model_config = vllm_config.model_config - cache_config = vllm_config.cache_config scheduler_config = vllm_config.scheduler_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3.5 currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 421292e880..c17d17014f 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -227,13 +227,6 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): config = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3_5MTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index e431c36f30..19fc8c1562 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -828,14 +828,8 @@ class Qwen3NextForCausalLM( config = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config self.model_config = vllm_config.model_config - cache_config = vllm_config.cache_config scheduler_config = vllm_config.scheduler_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3Next currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 6b527acd62..e5e306bb86 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -187,13 +187,6 @@ class Qwen3NextMTP(nn.Module, QwenNextMixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): config = vllm_config.model_config.hf_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen3NextMTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index fe50bd4f53..e5a63f13a3 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -104,7 +104,7 @@ def _mamba_get_block_table_tensor( kv_cache_spec: MambaSpec, mamba_cache_mode: str, ) -> torch.Tensor: - if mamba_cache_mode in ("all", "none"): + if mamba_cache_mode == "none": return block_table assert block_table.is_cuda and seq_lens.is_cuda diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index db94a2a7d6..2cdc3c26fa 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -638,11 +638,6 @@ class Qwen4ExpForCausalLM( self.quant_config = vllm_config.quant_config self.config = config self.scheduler_config = vllm_config.scheduler_config - if vllm_config.cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4Exp currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.model = Qwen4ExpModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py index a2b420603f..f4302aa757 100644 --- a/vllm/models/qwen4_exp/amd/mtp.py +++ b/vllm/models/qwen4_exp/amd/mtp.py @@ -395,13 +395,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4ExpMTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 6f35bea130..20521f1800 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -669,11 +669,6 @@ class Qwen4ExpForCausalLM( self.quant_config = vllm_config.quant_config self.config = config self.scheduler_config = vllm_config.scheduler_config - if vllm_config.cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4Exp currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) self.model = Qwen4ExpModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index af3bcd4659..d9f26fc022 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -375,13 +375,6 @@ class Qwen4ExpMTP(nn.Module, SupportsPP, Qwen4ExpMixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config self.vllm_config = vllm_config - cache_config = vllm_config.cache_config - if cache_config.mamba_cache_mode == "all": - raise NotImplementedError( - "Qwen4ExpMTP currently does not support 'all' prefix caching, " - "please use '--mamba-cache-mode=align' instead" - ) - self.quant_config = vllm_config.quant_config super().__init__() diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 5b053c4c78..769ffe5e8e 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -939,13 +939,6 @@ class Platform: if mamba_page_size == 0: return - # mamba_block_size here should either be user specified value or None - mamba_block_size = ( - cache_config.mamba_block_size - if cache_config.user_specified_mamba_block_size - else None - ) - # Get kernel block alignment from the backend's supported sizes with set_current_vllm_config(vllm_config): kernel_block_alignment_size = max( @@ -962,27 +955,15 @@ class Platform: # multiple of 128 so split kernel blocks keep that invariant. kernel_block_alignment_size = max(kernel_block_alignment_size, 128) - if cache_config.mamba_cache_mode == "all": - # With prefix caching, align to mamba chunk size for kernel perf - # TODO(tdoublep): this constraint can be relaxed fairly - # easily by changing the way we layout chunks in the - # mamba2 kernels. - base_chunk_size = mamba_block_size or model_config.get_mamba_chunk_size() - assert base_chunk_size is not None - attn_tokens_per_mamba_state = cdiv(mamba_page_size, attn_page_size_1_token) - chunk_size = lcm(base_chunk_size, kernel_block_alignment_size) - attn_block_size = chunk_size * cdiv(attn_tokens_per_mamba_state, chunk_size) - cache_config.mamba_block_size = attn_block_size - else: - # Without prefix caching, use minimum block size that satisfies - # both backend alignment and mamba page size compatibility - attn_block_size = kernel_block_alignment_size * cdiv( - mamba_page_size, - kernel_block_alignment_size * attn_page_size_1_token, - ) - indexer_align = cls._get_indexer_block_alignment(vllm_config) - if indexer_align: - attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align) + # Use minimum block size that satisfies both backend alignment and + # mamba page size compatibility + attn_block_size = kernel_block_alignment_size * cdiv( + mamba_page_size, + kernel_block_alignment_size * attn_page_size_1_token, + ) + indexer_align = cls._get_indexer_block_alignment(vllm_config) + if indexer_align: + attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align) if cache_config.block_size < attn_block_size: cache_config.block_size = attn_block_size diff --git a/vllm/v1/attention/backends/mamba1_attn.py b/vllm/v1/attention/backends/mamba1_attn.py index 925fceb024..53dc5973fb 100644 --- a/vllm/v1/attention/backends/mamba1_attn.py +++ b/vllm/v1/attention/backends/mamba1_attn.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass, replace +from dataclasses import dataclass from typing import Any from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata @@ -42,23 +42,4 @@ class Mamba1AttentionMetadataBuilder( fast_build: bool = False, **kwargs: Any, ) -> Mamba1AttentionMetadata: - common = self._compute_common_metadata(common_attn_metadata) - - if ( - common.num_prefills > 0 - and self.vllm_config.cache_config.mamba_cache_mode == "all" - ): - cu_chunk_seqlen_p, _, last_chunk_indices_p = ( - self._build_chunk_metadata_tensors( - self.kv_cache_spec.block_size, - common, - common_attn_metadata, - ) - ) - return replace( - common, - cu_chunk_seqlen_p=cu_chunk_seqlen_p, - last_chunk_indices_p=last_chunk_indices_p, - ) - - return common + return self._compute_common_metadata(common_attn_metadata) diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index fc71e3485f..e5e3827bcd 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -139,7 +139,6 @@ class Mamba2AttentionMetadataBuilder( common = self._compute_common_metadata( common_attn_metadata, num_accepted_tokens=kwargs.get("num_accepted_tokens"), - prev_last_scheduled_idx=kwargs.get("prev_last_scheduled_idx"), num_decode_draft_tokens_cpu=kwargs.get("num_decode_draft_tokens_cpu"), ) diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 7c543570aa..9953461500 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -39,7 +39,6 @@ class BaseMambaAttentionMetadata: # the batch has no prefill requests. has_initial_states_p: torch.Tensor | None query_start_loc_p: torch.Tensor | None - num_computed_tokens_p: torch.Tensor | None state_indices_tensor_p: torch.Tensor | None # The following tensors are used for decode requests and @@ -52,13 +51,6 @@ class BaseMambaAttentionMetadata: # Includes the bonus token (so minimum is 1) num_accepted_tokens: torch.Tensor | None # shape: [batch,] - # The following tensors are only used for prefix caching in all mode and - # are None if disabled - block_idx_last_scheduled_token: torch.Tensor | None - block_idx_first_scheduled_token_p: torch.Tensor | None - block_idx_last_computed_token: torch.Tensor | None - block_idx_last_scheduled_token_prev_step: torch.Tensor | None - # The following tensor is only used for prefix caching in align mode seq_lens: torch.Tensor @@ -70,9 +62,6 @@ class BaseMambaAttentionMetadata: # last_chunk_indices_p is a tensor of shape (batch,) that contains the # index of the last chunk for every sequence in the (prefill) batch. last_chunk_indices_p: torch.Tensor | None = None - # Number of block-aligned SSM states the prefill path writes, i.e. - # sum(block_idx_last_scheduled_token_p - block_idx_first_scheduled_token_p). - num_state_writes_p: int = 0 # The following attributes are for triton implementation of causal_conv1d nums_dict: dict | None = None @@ -130,47 +119,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): self.compilation_config.max_cudagraph_capture_size, ) - if self.vllm_config.cache_config.mamba_cache_mode == "all": - max_num_blocks = ( - cdiv( - self.vllm_config.model_config.max_model_len, - kv_cache_spec.block_size, - ) - + kv_cache_spec.num_speculative_blocks - ) - # TODO: reduce this size as needed for decode-only cudagraph capture - self.state_indices_tensor_d: torch.Tensor = torch.empty( - ( - self.decode_cudagraph_max_bs, - max_num_blocks, - ), - dtype=torch.int32, - device=device, - ) - self.block_idx_last_scheduled_token: torch.Tensor = torch.empty( - (self.decode_cudagraph_max_bs,), - dtype=torch.int32, - device=device, - ) - self.block_idx_last_computed_token: torch.Tensor = torch.empty( - (self.decode_cudagraph_max_bs,), - dtype=torch.int32, - device=device, - ) - if self.use_spec_decode: - self.block_idx_last_scheduled_token_prev_step: torch.Tensor = ( - torch.empty( - (self.decode_cudagraph_max_bs,), - dtype=torch.int32, - device=device, - ) - ) - else: - self.state_indices_tensor_d = torch.empty( - (self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens), - dtype=torch.int32, - device=device, - ) + self.state_indices_tensor_d = torch.empty( + (self.decode_cudagraph_max_bs, 1 + self.num_spec_tokens), + dtype=torch.int32, + device=device, + ) # For speculative decoding, we need to store the following buffers # for CUDA graph capture during decode @@ -259,23 +212,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): if self.num_spec_tokens > 0: num_accepted_tokens = torch.diff(m.query_start_loc) - prev_last_scheduled_idx = None - if ( - self.use_spec_decode - and self.vllm_config.cache_config.mamba_cache_mode == "all" - ): - prev_last_scheduled_idx = torch.zeros( - (m.num_reqs,), - dtype=torch.int32, - device=m.query_start_loc.device, - ) - - return self.build( - 0, - m, - num_accepted_tokens=num_accepted_tokens, - prev_last_scheduled_idx=prev_last_scheduled_idx, - ) + return self.build(0, m, num_accepted_tokens=num_accepted_tokens) def build( self, @@ -284,7 +221,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): fast_build: bool = False, *, num_accepted_tokens: torch.Tensor | None = None, - prev_last_scheduled_idx: torch.Tensor | None = None, num_decode_draft_tokens_cpu: torch.Tensor | None = None, **kwargs: Any, ) -> M: @@ -294,7 +230,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): return self._compute_common_metadata( common_attn_metadata, num_accepted_tokens=num_accepted_tokens, - prev_last_scheduled_idx=prev_last_scheduled_idx, num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, ) @@ -423,56 +358,11 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): ) return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p - def _compute_prefix_caching_block_indices( - self, - common_attn_metadata: CommonAttentionMetadata, - mamba_block_size: int, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() - # Block index of the last computed token - block_idx_last_computed_token = ( - torch.div( - num_computed_tokens + mamba_block_size - 1, - mamba_block_size, - rounding_mode="floor", - ) - - 1 - ) - # which is <= block index for the first scheduled token - block_idx_first_scheduled_token = ( - torch.div( - num_computed_tokens + mamba_block_size, - mamba_block_size, - rounding_mode="floor", - ) - - 1 - ) - # which is <= block index of the last scheduled token - block_idx_last_scheduled_token = ( - torch.div( - common_attn_metadata.seq_lens + mamba_block_size - 1, - mamba_block_size, - rounding_mode="floor", - ) - - 1 - ) - # -1 in case it's non-computed and causes later issues with indexing - block_idx_last_computed_token.clamp_(min=0) - # -1 in the case we have a padded request (0 seq-len) - block_idx_last_scheduled_token.clamp_(min=0) - - return ( - block_idx_last_computed_token, - block_idx_first_scheduled_token, - block_idx_last_scheduled_token, - ) - def _compute_common_metadata( self, common_attn_metadata: CommonAttentionMetadata, *, num_accepted_tokens: torch.Tensor | None = None, - prev_last_scheduled_idx: torch.Tensor | None = None, num_decode_draft_tokens_cpu: torch.Tensor | None = None, ) -> M: """Compute metadata common to both Mamba1 and Mamba2.""" @@ -540,16 +430,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): has_initial_states_p = None query_start_loc_p = None query_start_loc_d = None - num_computed_tokens = None - num_computed_tokens_p = None - - # for prefix caching - block_idx_first_scheduled_token = None - block_idx_first_scheduled_token_p = None - block_idx_last_computed_token = None - block_idx_last_scheduled_token = None - block_idx_last_scheduled_token_prev_step = None - num_state_writes_p = 0 # for causal_conv1d nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None @@ -557,35 +437,12 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): is_flush_d = None replayssm_scratch = None - if self.vllm_config.cache_config.mamba_cache_mode == "all": - num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() - - # Return a tensor of shape (#requests, #max blocks) - state_indices_tensor = common_attn_metadata.block_table_tensor - # Additional cache-related variables: - mamba_block_size = self.kv_cache_spec.block_size - ( - block_idx_last_computed_token, - block_idx_first_scheduled_token, - block_idx_last_scheduled_token, - ) = self._compute_prefix_caching_block_indices( - common_attn_metadata, mamba_block_size - ) - if self.use_spec_decode and prev_last_scheduled_idx is not None: - fallback = (num_computed_tokens - 1) // mamba_block_size - fallback.clamp_(min=0) - block_idx_last_scheduled_token_prev_step = torch.where( - prev_last_scheduled_idx >= 0, - prev_last_scheduled_idx, - fallback, - ) - else: - state_indices_tensor = mamba_get_block_table_tensor( - common_attn_metadata.block_table_tensor, - common_attn_metadata.seq_lens, - self.kv_cache_spec, - self.vllm_config.cache_config.mamba_cache_mode, - ) + state_indices_tensor = mamba_get_block_table_tensor( + common_attn_metadata.block_table_tensor, + common_attn_metadata.seq_lens, + self.kv_cache_spec, + self.vllm_config.cache_config.mamba_cache_mode, + ) if state_indices_tensor.dim() == 1: state_indices_tensor = state_indices_tensor.unsqueeze(-1) @@ -595,11 +452,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): [num_decodes, num_prefills], dim=0, ) - if self.vllm_config.cache_config.mamba_cache_mode != "all": - state_indices_tensor_d = state_indices_tensor_d[ - :, : 1 + self.num_spec_tokens - ] - state_indices_tensor_p = state_indices_tensor_p[:, 0] + state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens] + state_indices_tensor_p = state_indices_tensor_p[:, 0] if num_decodes > 0 and self.use_spec_decode: query_start_loc_d = common_attn_metadata.query_start_loc[: num_decodes + 1] @@ -613,8 +467,7 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_accepted_tokens = num_accepted_tokens[:num_decodes] if num_prefills > 0: - if num_computed_tokens is None: - num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() + num_computed_tokens = common_attn_metadata.compute_num_computed_tokens() query_start_loc_p = ( common_attn_metadata.query_start_loc[-num_prefills - 1 :] @@ -636,33 +489,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): ) ) - if self.vllm_config.cache_config.mamba_cache_mode == "all": - assert num_computed_tokens is not None - num_computed_tokens_p = num_computed_tokens[ - num_reqs - num_prefills : num_reqs - ] - assert block_idx_first_scheduled_token is not None - block_idx_first_scheduled_token_p = block_idx_first_scheduled_token[ - num_reqs - num_prefills : num_reqs - ] - - # How many block-aligned states prefill will write: the same - # (block_idx_last - block_idx_first) as above, from CPU data so - # the mixer needs no D2H read. The -1 in both indices cancels. - block_size = self.kv_cache_spec.block_size - seq_lens_p_cpu = seq_lens_cpu[num_reqs - num_prefills : num_reqs] - num_computed_tokens_p_cpu, _ = self._prefill_cpu_metadata( - common_attn_metadata, num_reqs, num_prefills, num_decode_tokens - ) - num_state_writes_p = int( - ( - (seq_lens_p_cpu + block_size - 1) // block_size - - (num_computed_tokens_p_cpu + block_size) // block_size - ) - .clamp(min=0) - .sum() - ) - if self.use_replayssm and not self.use_flashinfer_replayssm and num_decodes > 0: decode_base_cpu = common_attn_metadata.replayssm_decode_base_cpu seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound @@ -764,14 +590,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): replayssm_scratch=replayssm_scratch, num_accepted_tokens=num_accepted_tokens, query_start_loc_d=query_start_loc_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token, - block_idx_first_scheduled_token_p=block_idx_first_scheduled_token_p, - num_state_writes_p=num_state_writes_p, - block_idx_last_computed_token=block_idx_last_computed_token, - block_idx_last_scheduled_token_prev_step=( - block_idx_last_scheduled_token_prev_step - ), - num_computed_tokens_p=num_computed_tokens_p, num_reqs=num_reqs, seq_lens=common_attn_metadata.seq_lens, nums_dict=nums_dict, @@ -791,11 +609,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): state_indices_tensor_d = metadata.state_indices_tensor_d query_start_loc_d = metadata.query_start_loc_d num_accepted_tokens = metadata.num_accepted_tokens - block_idx_last_scheduled_token = metadata.block_idx_last_scheduled_token - block_idx_last_computed_token = metadata.block_idx_last_computed_token - block_idx_last_scheduled_token_prev_step = ( - metadata.block_idx_last_scheduled_token_prev_step - ) write_pos_d = metadata.write_pos_d is_flush_d = metadata.is_flush_d bc_pre_scratch = metadata.bc_pre_scratch @@ -824,44 +637,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): 1 # pad with 1st slot index ) - if self.vllm_config.cache_config.mamba_cache_mode == "all": - assert block_idx_last_scheduled_token is not None - assert block_idx_last_computed_token is not None - self.block_idx_last_scheduled_token[: metadata.num_decodes].copy_( - block_idx_last_scheduled_token[: metadata.num_decodes], - non_blocking=True, - ) - block_idx_last_scheduled_token = self.block_idx_last_scheduled_token[ - :padded_bs - ] - block_idx_last_scheduled_token[metadata.num_decodes :] = 0 - - self.block_idx_last_computed_token[: metadata.num_decodes].copy_( - block_idx_last_computed_token[: metadata.num_decodes], - non_blocking=True, - ) - block_idx_last_computed_token = self.block_idx_last_computed_token[ - :padded_bs - ] - block_idx_last_computed_token[metadata.num_decodes :] = 0 - - if ( - self.use_spec_decode - and block_idx_last_scheduled_token_prev_step is not None - ): - self.block_idx_last_scheduled_token_prev_step[ - : metadata.num_decodes - ].copy_( - block_idx_last_scheduled_token_prev_step[ - : metadata.num_decodes - ], - non_blocking=True, - ) - block_idx_last_scheduled_token_prev_step = ( - self.block_idx_last_scheduled_token_prev_step[:padded_bs] - ) - block_idx_last_scheduled_token_prev_step[metadata.num_decodes :] = 0 - if self.use_replayssm and not self.use_flashinfer_replayssm: assert write_pos_d is not None assert is_flush_d is not None @@ -914,11 +689,6 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): bc_pre_scratch=bc_pre_scratch, replayssm_scratch=replayssm_scratch, replayssm_state_indices_d=replayssm_state_indices_d, - block_idx_last_scheduled_token=block_idx_last_scheduled_token, - block_idx_last_computed_token=block_idx_last_computed_token, - block_idx_last_scheduled_token_prev_step=( - block_idx_last_scheduled_token_prev_step - ), ) def update_block_table( @@ -950,11 +720,8 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): [metadata.num_decodes, metadata.num_prefills], dim=0, ) - if self.vllm_config.cache_config.mamba_cache_mode != "all": - state_indices_tensor_d = state_indices_tensor_d[ - :, : 1 + self.num_spec_tokens - ] - state_indices_tensor_p = state_indices_tensor_p[:, 0] + state_indices_tensor_d = state_indices_tensor_d[:, : 1 + self.num_spec_tokens] + state_indices_tensor_p = state_indices_tensor_p[:, 0] new_metadata = replace( metadata, diff --git a/vllm/v1/attention/backends/short_conv_attn.py b/vllm/v1/attention/backends/short_conv_attn.py index f2cc3dae99..24bbee228d 100644 --- a/vllm/v1/attention/backends/short_conv_attn.py +++ b/vllm/v1/attention/backends/short_conv_attn.py @@ -283,7 +283,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): has_initial_states_p = None has_initial_states_d = None - num_computed_tokens_p = None # Per-request classification by mask, NOT by position. With # spec-decode, the front decode group can contain both spec-decode @@ -402,10 +401,9 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): assert non_spec_query_start_loc is not None query_start_loc_d = non_spec_query_start_loc[: num_decodes + 1] if num_prefills > 0: - num_computed_tokens_p = num_computed_tokens[ - num_decodes : num_decodes + num_prefills - ] - has_initial_states_p = num_computed_tokens_p > 0 + has_initial_states_p = ( + num_computed_tokens[num_decodes : num_decodes + num_prefills] > 0 + ) assert non_spec_query_start_loc is not None query_start_loc_p = ( non_spec_query_start_loc[num_decodes:] - num_decode_tokens @@ -478,11 +476,6 @@ class PleShortConvAttentionMetadataBuilder(ShortConvAttentionMetadataBuilder): query_start_loc_d=query_start_loc_d, state_indices_tensor_p=state_indices_tensor_p, state_indices_tensor_d=state_indices_tensor_d, - num_computed_tokens_p=num_computed_tokens_p, - block_idx_last_scheduled_token=None, - block_idx_first_scheduled_token_p=None, - block_idx_last_computed_token=None, - block_idx_last_scheduled_token_prev_step=None, seq_lens=m.seq_lens, ) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index c97483912d..eb28fd1e78 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -1162,11 +1162,6 @@ def mamba_get_block_table_tensor( """Get the block table tensor for mamba kernels from the input common_attn_metadata.block_table_tensor given different mamba cache modes. - - "all": input (#requests, cdiv(max_model_len, block_size) - + num_speculative_blocks); - output (#requests, cdiv(max_model_len, block_size) - + num_speculative_blocks). - - "none": input (#requests, 1 + num_speculative_blocks); output (#requests, 1 + num_speculative_blocks). @@ -1174,7 +1169,7 @@ def mamba_get_block_table_tensor( output (#requests, 1 + num_speculative_blocks), which are the last 1 + num_speculative_blocks of each request. """ - if mamba_cache_mode in ("all", "none"): + if mamba_cache_mode == "none": return block_table else: assert isinstance(kv_cache_spec, MambaSpec) diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index cabbbfadc3..248f03dd02 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -1079,12 +1079,7 @@ class MambaSpec(KVCacheSpec): return None def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - if vllm_config.cache_config.mamba_cache_mode == "all": - max_model_len = vllm_config.model_config.max_model_len - return ( - cdiv(max_model_len, self.block_size) + self.num_speculative_blocks - ) * self.page_size_bytes - elif vllm_config.cache_config.mamba_cache_mode == "align": + if vllm_config.cache_config.mamba_cache_mode == "align": return self.page_size_bytes * ( 2 + self.num_speculative_blocks + self.num_prefill_checkpoint_blocks ) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 4ccdec2f21..9a0a4eeda9 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -967,11 +967,6 @@ class GPUModelRunner( self.mamba_state_idx: dict[str, int] = {} self._mamba_bufs: mamba_utils.MambaBuffers | None = None self._mamba_state_copy_funcs: MambaStateCopyFuncsByType | None = None - self.mamba_prev_last_scheduled_idx: CpuGpuBuffer | None = None - if self.cache_config.mamba_cache_mode == "all" and self.num_spec_tokens > 0: - self.mamba_prev_last_scheduled_idx = self._make_buffer( - self.max_num_reqs, dtype=torch.int32 - ) self.layerwise_nvtx_hooks_registered = False def update_max_model_len(self, max_model_len: int) -> None: @@ -1608,17 +1603,6 @@ class GPUModelRunner( assert self.num_accepted_tokens_event is not None self.num_accepted_tokens_event.record() - if self.cache_config.mamba_cache_mode == "all": - mamba_utils.postprocess_mamba_all( - scheduler_output, - self.kv_cache_config, - self.input_batch, - self.requests, - self.mamba_state_idx, - self.num_spec_tokens, - num_reqs, - ) - def _update_streaming_request( self, req_id: str, new_req_data: NewRequestData ) -> CachedRequestState: @@ -2127,15 +2111,6 @@ class GPUModelRunner( self.num_accepted_tokens.np.fill(1) self.num_accepted_tokens.gpu.fill_(1) - if self.mamba_prev_last_scheduled_idx is not None: - mamba_utils.preprocess_mamba_all_specdec( - scheduler_output, - self.input_batch, - self.mamba_state_idx, - num_reqs, - self.mamba_prev_last_scheduled_idx, - ) - # Update num_computed_tokens on GPU. In async spec decode, # CPU values are optimistic (all drafts accepted). The kernel # corrects on GPU using the previous step's @@ -2511,13 +2486,6 @@ class GPUModelRunner( :num_reqs_padded ], ) - if ( - isinstance(builder, Mamba2AttentionMetadataBuilder) - and self.mamba_prev_last_scheduled_idx is not None - ): - extra_attn_metadata_args["prev_last_scheduled_idx"] = ( - self.mamba_prev_last_scheduled_idx.gpu[:num_reqs_padded] - ) if for_cudagraph_capture: attn_metadata_i = builder.build_for_cudagraph_capture( diff --git a/vllm/v1/worker/mamba_utils.py b/vllm/v1/worker/mamba_utils.py index c2e0120e46..33a2b8b480 100644 --- a/vllm/v1/worker/mamba_utils.py +++ b/vllm/v1/worker/mamba_utils.py @@ -1535,53 +1535,6 @@ def preprocess_mamba( do_mamba_copy_block(copy_bufs) -def postprocess_mamba_all( - scheduler_output: SchedulerOutput, - kv_cache_config: KVCacheConfig, - input_batch: InputBatch, - requests: dict[str, CachedRequestState], - mamba_state_idx: dict[str, int], - num_spec_tokens: int, - num_reqs: int, -): - """All-mode postprocess (only meaningful with num_spec_tokens > 0): - record per-request the block index of the last token scheduled this - step, so the next step can anchor its in-place writes when accepted - drafts leave the sequence at a non-block-aligned position. - """ - if num_spec_tokens <= 0: - return - mamba_groups = get_mamba_groups(kv_cache_config) - block_sizes = {mamba_spec.block_size for mamba_spec in mamba_groups} - assert len(block_sizes) == 1, "all mamba groups must share block_size" - block_size = next(iter(block_sizes)) - full_decode_len = 1 + num_spec_tokens - scheduled = scheduler_output.num_scheduled_tokens - for req_id in input_batch.req_ids[:num_reqs]: - num_query = scheduled.get(req_id, 0) - if num_query == full_decode_len: - req = requests[req_id] - seq_len = req.num_computed_tokens + num_query - mamba_state_idx[req_id] = max(0, (seq_len - 1) // block_size) - else: - mamba_state_idx.pop(req_id, None) - - -def preprocess_mamba_all_specdec( - scheduler_output: SchedulerOutput, - input_batch: InputBatch, - mamba_state_idx: dict[str, int], - num_reqs: int, - prev_last_scheduled_idx_buf: CpuGpuBuffer, -) -> None: - cleanup_mamba_state_idx(scheduler_output, mamba_state_idx) - np_view = prev_last_scheduled_idx_buf.np - for i, req_id in enumerate(input_batch.req_ids[:num_reqs]): - np_view[i] = mamba_state_idx.get(req_id, -1) - np_view[num_reqs:].fill(-1) - prev_last_scheduled_idx_buf.copy_to_gpu() - - def postprocess_mamba_align_gpu( *, bufs: "MambaBuffers", From ab5266769e702434a0968d47319d0731d8ccac35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 1 Oct 2026 12:19:02 +0200 Subject: [PATCH 197/248] [CI/Build] Add mooncake auto-label rule and assign topic owners (#59192) Signed-off-by: NickLucche --- .github/mergify.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/mergify.yml b/.github/mergify.yml index 707936b93c..5006fd798b 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -660,6 +660,30 @@ pull_request_rules: add: - kv-connector +- name: label-mooncake + description: Automatically apply mooncake label + conditions: + - label != stale + - or: + - files~=^vllm/distributed/kv_transfer/kv_connector/v1/mooncake/ + # Covers the mooncake/ connector dir, mooncake_store_embedding/ and + # mooncake_ec_connector.py. + - files~=^vllm/distributed/ec_transfer/ec_connector/mooncake + - files=vllm/distributed/mooncake_store.py + - files~=^tests/v1/kv_connector/.*mooncake + - files~=^tests/v1/ec_connector/.*mooncake + - files~=^docs/features/mooncake.*\.md + - files=.buildkite/test_areas/disaggregated_mooncake.yaml + - title~=(?i)mooncake + actions: + label: + add: + - mooncake + assign: + users: + - "stmatengss" + - "LCAIZJ" + - name: label-scheduler description: Automatically apply scheduler label conditions: From f3b77eff640b5ed5cc9221a4f5f402ab65adc021 Mon Sep 17 00:00:00 2001 From: Idder Ghanbaja Date: Thu, 1 Oct 2026 15:03:03 +0200 Subject: [PATCH 198/248] [Rust][Benchmark] Warn when temperature is left to the server default (#59247) --- rust/src/bench/src/config.rs | 59 ++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/rust/src/bench/src/config.rs b/rust/src/bench/src/config.rs index b5a24d942f..84f55ad427 100644 --- a/rust/src/bench/src/config.rs +++ b/rust/src/bench/src/config.rs @@ -309,6 +309,14 @@ impl BenchConfig { } } + if uses_server_default_temperature(args.backend, extra_body.as_ref()) { + tracing::warn!( + "vllm-bench does not set temperature==0 (greedy) in requests by default. \ + The default will be determined on the server side and can be \ + model/API specific. For greedy decoding, include --temperature=0." + ); + } + // Parse metadata let metadata = match &args.metadata { None => None, @@ -862,6 +870,21 @@ fn parse_ramp_up(args: &BenchServeArgs) -> Result> { })) } +/// Whether generation requests leave `temperature` to the server-side default. +/// +/// Python `vllm bench serve` defaulted to greedy decoding before v0.15, so +/// results are only comparable with older runs when temperature is set explicitly. +/// An explicit `"temperature": null` also falls back to the server default. +fn uses_server_default_temperature( + backend: BackendKind, + extra_body: Option<&serde_json::Value>, +) -> bool { + !backend.is_pooling() + && extra_body + .and_then(|b| b.get("temperature")) + .is_none_or(serde_json::Value::is_null) +} + #[cfg(test)] mod tests { use clap::Parser; @@ -910,6 +933,42 @@ mod tests { assert_eq!(args.speed_bench_output_len, 4096); } + /// Temperature left unset is flagged, since older Python versions defaulted to greedy. + #[test] + fn test_uses_server_default_temperature() { + let check = |backend, extra: &[&str]| { + let mut argv = vec!["vllm-bench", "--model", "test-model", "--backend", backend]; + argv.extend_from_slice(extra); + let config = BenchConfig::from_args(&parse_args(argv)).unwrap(); + uses_server_default_temperature(config.backend, config.extra_body.as_ref()) + }; + + assert!(check("openai-chat", &[])); + assert!(check("vllm", &["--extra-body", r#"{"top_p": 0.9}"#])); + assert!(!check("openai-chat", &["--temperature", "0"])); + assert!(!check( + "openai", + &["--extra-body", r#"{"temperature": 0.6}"#] + )); + assert!(!check("openai-embeddings", &[])); + + // `"temperature": null` leaves it to the server, and `--extra-body` + // overrides `--temperature`, so both cases must still warn. + assert!(check( + "openai-chat", + &["--extra-body", r#"{"temperature": null}"#] + )); + assert!(check( + "openai-chat", + &[ + "--temperature", + "0", + "--extra-body", + r#"{"temperature": null}"# + ] + )); + } + #[test] fn test_prefix_sharing_defaults_to_zero() { let args = base_multi_turn_args(); From 5d9214d6b2ebecb05a9b4cafc2763404acb79289 Mon Sep 17 00:00:00 2001 From: Maroon Ayoub Date: Thu, 1 Oct 2026 16:44:40 +0300 Subject: [PATCH 199/248] [Bugfix][Frontend] Strip `x-anthropic-billing-header` billing header from `/v1/chatcompletions` (#59419) Signed-off-by: Maroon Ayoub Signed-off-by: Robert Shaw Co-authored-by: Claude Co-authored-by: Robert Shaw Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- .../unit_tests/test_chat_billing_header.py | 77 +++++++++++++++++++ vllm/entrypoints/chat_utils.py | 37 +++++++++ 2 files changed, 114 insertions(+) create mode 100644 tests/entrypoints/unit_tests/test_chat_billing_header.py diff --git a/tests/entrypoints/unit_tests/test_chat_billing_header.py b/tests/entrypoints/unit_tests/test_chat_billing_header.py new file mode 100644 index 0000000000..31fa97e939 --- /dev/null +++ b/tests/entrypoints/unit_tests/test_chat_billing_header.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Claude Code's attribution header in chat completions system messages.""" + +from unittest.mock import MagicMock + +import pytest + +from vllm.entrypoints.chat_utils import _parse_chat_message_content + +HEADER = ( + "x-anthropic-billing-header: cc_version=2.1.220.300; " + "cc_entrypoint=claude-desktop; cch=7adab;" +) + + +def _parse(message, content_format="openai"): + tracker = MagicMock() + tracker.create_parser.return_value.mm_placeholder_storage.return_value = {} + return _parse_chat_message_content( + message, tracker, content_format, interleave_strings=False + ) + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize( + "header", + [ + HEADER, + "x-anthropic-billing-header: cc_version=2.1.286.99b; cc_entrypoint=sdk-cli;", + ], +) +def test_system_header_part_is_dropped(header): + (msg,) = _parse( + { + "role": "system", + "content": [ + {"type": "text", "text": header}, + {"type": "text", "text": "You are a Claude agent."}, + ], + } + ) + assert msg["content"] == [{"type": "text", "text": "You are a Claude agent."}] + + +@pytest.mark.skip_global_cleanup +def test_system_header_line_is_stripped_from_string(): + (msg,) = _parse( + {"role": "system", "content": HEADER + "\nYou are a Claude agent."}, + content_format="string", + ) + assert msg["content"] == "You are a Claude agent." + + +@pytest.mark.skip_global_cleanup +def test_requests_differing_only_in_cch_parse_identically(): + def system(cch): + return { + "role": "system", + "content": HEADER.replace("7adab", cch) + "\nYou are a Claude agent.", + } + + assert _parse(system("7adab")) == _parse(system("986dd")) + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize( + "message", + [ + {"role": "system", "content": "You are a Claude agent."}, + {"role": "system", "content": "Quote: " + HEADER}, + {"role": "user", "content": HEADER + "Hello"}, + ], +) +def test_other_content_is_unchanged(message): + (msg,) = _parse(message, content_format="string") + assert msg["content"] == message["content"] diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 10c4866f89..2d875a974b 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -2035,6 +2035,41 @@ _AssistantParser = partial(cast, ChatCompletionAssistantMessageParam) _ToolParser = partial(cast, ChatCompletionToolMessageParam) +_CLAUDE_CODE_BILLING_HEADER = "x-anthropic-billing-header" + + +def _strip_claude_code_billing_header( + parts: Iterable[ChatCompletionContentPartParam], +) -> list[ChatCompletionContentPartParam]: + """Drop Claude Code's attribution header line from system text parts. + + Its ``cch`` value changes on every request, so a prompt that keeps it + misses the prefix cache from the header on. Gateways that translate + Anthropic requests to chat completions keep it; ``/v1/messages`` drops + the same block. + """ + out: list[ChatCompletionContentPartParam] = [] + for part in parts: + if isinstance(part, str): + text: object = part + elif isinstance(part, dict) and part.get("type") == "text": + text = part.get("text") + else: + text = None + if not (isinstance(text, str) and text.startswith(_CLAUDE_CODE_BILLING_HEADER)): + out.append(part) + continue + rest = text.partition("\n")[2] + if not rest: + continue + out.append( + cast(ChatCompletionContentPartParam, rest) + if isinstance(part, str) + else cast(ChatCompletionContentPartParam, {**part, "text": rest}) + ) + return out + + def _parse_chat_message_content( message: ChatCompletionMessageParam, mm_tracker: BaseMultiModalItemTracker, @@ -2050,6 +2085,8 @@ def _parse_chat_message_content( content = [] elif isinstance(content, str): content = [ChatCompletionContentPartTextParam(type="text", text=content)] + if role == "system": + content = _strip_claude_code_billing_header(content) # type: ignore[arg-type] result = _parse_chat_message_content_parts( role, content, # type: ignore From b538d807ffe0976af7e5d107ccfe77e676b83b70 Mon Sep 17 00:00:00 2001 From: ai-jz <156989844+ai-jz@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:59:15 -0700 Subject: [PATCH 200/248] [Bugfix][MRV2] Keep GDN prefill checkpoint metadata local to each cache group (#59536) Signed-off-by: Jingqiao Zhang Signed-off-by: zjy0516 Co-authored-by: Jingqiao Zhang Co-authored-by: zjy0516 --- .../v1/attention/test_gdn_metadata_builder.py | 56 ++++++++++++++++--- .../model_executor/layers/mamba/checkpoint.py | 39 +++++++++---- vllm/v1/attention/backends/gdn_attn.py | 6 ++ 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index 70a77f5929..cd01fba49e 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -17,6 +17,7 @@ from tests.v1.attention.utils import ( ) from vllm.config import SpeculativeConfig from vllm.config.compilation import CUDAGraphMode +from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionMetadata, GDNAttentionMetadataBuilder, @@ -383,14 +384,9 @@ def test_cudagraph_capture_batch_stays_decode_only(): torch.testing.assert_close(staged, common_attn_metadata.block_table_tensor[:, 0]) -@pytest.mark.parametrize("num_spec", [0, 3]) -@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)]) -def test_checkpoint_metadata_preserves_non_spec_order( - num_spec, prefix_match_unit, expected_offset -): - """Only eligible non-spec rows get a checkpoint in their reserved page.""" - from vllm.v1.attention.backends.utils import NULL_BLOCK_ID - +def _create_checkpoint_builder_and_batch( + num_spec: int, prefix_match_unit: int = 16 +) -> tuple[GDNAttentionMetadataBuilder, CommonAttentionMetadata, dict]: vllm_config = create_vllm_config( model_name="Qwen/Qwen3.5-0.8B", block_size=BLOCK_SIZE, @@ -431,7 +427,20 @@ def test_checkpoint_metadata_preserves_non_spec_order( num_decode_draft_tokens_cpu=torch.tensor([2, -1, -1], dtype=torch.int32), num_accepted_tokens=torch.ones(3, dtype=torch.int32), ) + return builder, common, kwargs + +@pytest.mark.parametrize("num_spec", [0, 3]) +@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)]) +def test_checkpoint_metadata_preserves_non_spec_order( + num_spec, prefix_match_unit, expected_offset +): + """Only eligible non-spec rows get a checkpoint in their reserved page.""" + from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + + builder, common, kwargs = _create_checkpoint_builder_and_batch( + num_spec, prefix_match_unit + ) actual = builder.build(0, common, **kwargs) assert actual.checkpoint is not None offsets = [expected_offset, 0] if num_spec else [0, expected_offset, 0] @@ -444,3 +453,34 @@ def test_checkpoint_metadata_preserves_non_spec_order( torch.testing.assert_close( actual.checkpoint.state_indices, torch.tensor(slots, dtype=torch.int32) ) + + +@pytest.mark.parametrize("num_spec", [0, 3]) +def test_update_block_table_regathers_checkpoint(num_spec): + """Checkpoint pages come from the target group's block table, not the + source group's (otherwise the target group's checkpoint page is never + written while the prefix cache treats it as valid).""" + src, common, kwargs = _create_checkpoint_builder_and_batch(num_spec) + dst, _, _ = _create_checkpoint_builder_and_batch(num_spec) + ref, _, _ = _create_checkpoint_builder_and_batch(num_spec) + other_table = common.block_table_tensor + 100 + other = common.replace(block_table_tensor=other_table) + dst.mamba_aligned_state_indices = mamba_get_block_table_tensor( + other_table, other.seq_lens, dst.kv_cache_spec, "align" + ) + + source = src.build(0, common, **kwargs) + expected = ref.build(0, other, **kwargs) + actual = dst.update_block_table(source, other_table, other.slot_mapping) + + assert source.checkpoint is not None and expected.checkpoint is not None + assert actual.checkpoint is not None + torch.testing.assert_close( + actual.checkpoint.state_indices, expected.checkpoint.state_indices + ) + torch.testing.assert_close( + actual.checkpoint.checkpoint_offsets, expected.checkpoint.checkpoint_offsets + ) + assert not torch.equal( + actual.checkpoint.state_indices, source.checkpoint.state_indices + ) diff --git a/vllm/model_executor/layers/mamba/checkpoint.py b/vllm/model_executor/layers/mamba/checkpoint.py index 1f4bf9dc37..feef0afa05 100644 --- a/vllm/model_executor/layers/mamba/checkpoint.py +++ b/vllm/model_executor/layers/mamba/checkpoint.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, replace import torch @@ -58,10 +58,33 @@ def compute_mamba_prefill_checkpoints( return offsets, cols +def _gather_checkpoint_state_indices( + block_table: torch.Tensor, request_rows: torch.Tensor, block_cols: torch.Tensor +) -> torch.Tensor: + return torch.where( + block_cols >= 0, block_table[request_rows, block_cols], NULL_BLOCK_ID + ) + + @dataclass class MambaPrefillCheckpointMetadata: checkpoint_offsets: torch.Tensor state_indices: torch.Tensor + # Gather indices into the block table, kept so that another KV cache group + # with the same spec can re-derive its own ``state_indices``. + request_rows: torch.Tensor | None = None + block_cols: torch.Tensor | None = None + + def regather_state_indices( + self, block_table: torch.Tensor + ) -> "MambaPrefillCheckpointMetadata": + assert self.request_rows is not None and self.block_cols is not None + return replace( + self, + state_indices=_gather_checkpoint_state_indices( + block_table, self.request_rows, self.block_cols + ), + ) class MambaPrefillCheckpointBuilder: @@ -111,17 +134,13 @@ class MambaPrefillCheckpointBuilder: checkpoint_cols_tensor = async_tensor_h2d( checkpoint_cols, dtype=torch.int64, device=m.query_start_loc.device ) - checkpoint_state_indices = m.block_table_tensor[ - request_rows_tensor, checkpoint_cols_tensor - ] - checkpoint_state_indices = torch.where( - checkpoint_cols_tensor >= 0, - checkpoint_state_indices, - NULL_BLOCK_ID, - ) return MambaPrefillCheckpointMetadata( checkpoint_offsets_tensor, - checkpoint_state_indices, + _gather_checkpoint_state_indices( + m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor + ), + request_rows=request_rows_tensor, + block_cols=checkpoint_cols_tensor, ) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index 6b2c6b292b..5fece32a1b 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -621,6 +621,11 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] """Re-gather this group's state indices. The other fields are batch-level and stay shared with ``metadata``.""" m = metadata + checkpoint = ( + m.checkpoint.regather_state_indices(blk_table) + if m.checkpoint is not None + else None + ) if self.vllm_config.cache_config.mamba_cache_mode == "align": assert self.mamba_aligned_state_indices is not None blk_table = self.mamba_aligned_state_indices @@ -662,6 +667,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] spec_state_indices_tensor=spec_indices, non_spec_state_indices_tensor=non_spec_indices, prefill_state_indices=prefill_indices, + checkpoint=checkpoint, ) def build_for_cudagraph_capture( From cfd54ca8819de923dcf555807298eaca34dd342d Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Thu, 1 Oct 2026 17:10:49 +0200 Subject: [PATCH 201/248] [ROCm][CI] Drop four no-GPU CPU groups from the legacy AMD pipeline (#59595) Signed-off-by: Stefan Koncarevic Co-authored-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 96 ---------------------------------------- 1 file changed, 96 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index cc594d192e..acd4d94a53 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -111,102 +111,6 @@ steps: # # ######################################################################################################################################### -- label: ":computer: (CPU) Basic Models Other" # TBD - timeout_in_minutes: 35 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - dind: false - agent_pool: mi250_1 - no_gpu: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/test_utils.py - - tests/models/test_vision.py - - tests/models/test_adapters.py - - tests/models/transformers/fusers/ - - tests/model_executor/layers/test_activation.py - - tests/models/test_qwen3_5_mtp_config.py - commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py - -- label: ":computer: (CPU) Multimodal Processor Shard %N" # TBD - timeout_in_minutes: 90 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - dind: false - agent_pool: mi250_1 - no_gpu: true - parallelism: 6 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - - tests/models/registry.py - commands: - - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - -- label: ":computer: (CPU) V1 Others" # TBD - timeout_in_minutes: 45 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - dind: false - agent_pool: mi250_1 - no_gpu: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1 - commands: - - pytest -v -s -m 'cpu_test' v1/core - - pytest -v -s v1/structured_output - - pytest -v -s v1/test_serial_utils.py - - pytest -v -s v1/test_kv_cache_spec_registry.py - - pytest -v -s v1/cudagraph/test_cudagraph_manager.py - - pytest -v -s -m 'cpu_test' v1/kv_connector/unit - - pytest -v -s -m 'cpu_test' v1/ec_connector/unit - - pytest -v -s -m 'cpu_test' v1/metrics - -- label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config" # TBD - timeout_in_minutes: 80 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - dind: false - agent_pool: mi250_1 - no_gpu: true - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/test_envs.py - - tests/test_outputs.py - - tests/test_pooling_params.py - - tests/test_ray_env.py - - tests/test_sampling_params.py - - tests/multimodal - - tests/renderers - - tests/standalone_tests/lazy_imports.py - - tests/tokenizers_ - - tests/reasoning - - tests/tool_parsers - - tests/parser - - tests/transformers_utils - - tests/config - commands: - - python3 standalone_tests/lazy_imports.py - - pytest -v -s test_envs.py - - pytest -v -s test_outputs.py - - pytest -v -s test_pooling_params.py - - pytest -v -s test_ray_env.py - - pytest -v -s test_sampling_params.py - - pytest -v -s -m 'cpu_test' multimodal - - pytest -v -s renderers - - pytest -v -s reasoning - - pytest -v -s tool_parsers - - pytest -v -s tokenizers_ - - pytest -v -s parser - - pytest -v -s transformers_utils - - pytest -v -s config --ignore=config/test_mp_reducer.py - - label: ":computer: (CPU) Rust Frontend Cargo Style + Clippy" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] From 83cadd65d9ec9e7c2839339a8f70f201246b540b Mon Sep 17 00:00:00 2001 From: aniskumar-nv Date: Thu, 1 Oct 2026 09:26:33 -0700 Subject: [PATCH 202/248] [Bugfix] Tie lm_head.weight for Nemotron Parse when checkpoint omits it (#53020) Signed-off-by: Anish Kumar Co-authored-by: Cyrus Leung --- .../multimodal/generation/test_nemotron_parse.py | 11 ++++++++++- vllm/model_executor/models/nemotron_parse.py | 9 +++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/models/multimodal/generation/test_nemotron_parse.py b/tests/models/multimodal/generation/test_nemotron_parse.py index 69e41b21e3..57e40430c6 100644 --- a/tests/models/multimodal/generation/test_nemotron_parse.py +++ b/tests/models/multimodal/generation/test_nemotron_parse.py @@ -104,7 +104,16 @@ def run_test( ) -@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"]) +@pytest.mark.parametrize( + "model", + [ + "nvidia/NVIDIA-Nemotron-Parse-v1.2", + # v2.0's checkpoint ties lm_head to the decoder's input embeddings + # instead of shipping a separate lm_head.weight tensor; this + # exercises that path (see tie_weights() in nemotron_parse.py). + "nvidia/NVIDIA-Nemotron-Parse-2.0", + ], +) @pytest.mark.parametrize("dtype", ["bfloat16"]) @pytest.mark.parametrize("num_logprobs", [5]) def test_models( diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 9ac38c0021..3ba731d2e0 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -568,6 +568,15 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal): self.lm_head = ParallelLMHead( config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config ) + # Some checkpoints (e.g. compact exports) tie the output head to the + # decoder's input embeddings instead of materializing a separate + # lm_head.weight tensor. + tie_word_embeddings = bool( + getattr(config, "tie_word_embeddings", False) + or getattr(config.decoder, "tie_word_embeddings", False) + ) + if tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens) self.logits_processor = LogitsProcessor( self.vocab_size, config.decoder.vocab_size ) From 9aaef4296f4eb9df7392de6920b6b2ed79f354ba Mon Sep 17 00:00:00 2001 From: Shijie Lyu Date: Thu, 1 Oct 2026 09:26:55 -0700 Subject: [PATCH 203/248] [Bugfix][Frontend] Check reused prompt token ids against the vocab before streaming (#59555) Signed-off-by: Shijie Lyu --- .../chat_completion/test_serving_chat.py | 19 ++++++++-- vllm/renderers/base.py | 37 +++++++++++++++++++ vllm/renderers/online_renderer.py | 11 ++++-- vllm/v1/engine/input_processor.py | 34 ++--------------- 4 files changed, 63 insertions(+), 38 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 241920d1e3..9c80f504f7 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -564,6 +564,9 @@ class MockModelConfig: def get_diff_sampling_param(self): return self.diff_sampling_param or {} + def get_vocab_size(self) -> int: + return 50257 + @dataclass class MockParallelConfig: @@ -2860,7 +2863,16 @@ async def test_chat_kv_transfer_prompt_token_ids_rejects_invalid_ids(ids): @pytest.mark.asyncio -async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector(): +@pytest.mark.parametrize( + ("ids", "stream", "match"), + [ + ([1.5], False, "non-negative integers"), + ([999999], True, r"out of vocabulary.*kv_transfer_params\.prompt_token_ids"), + ], +) +async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector( + ids, stream, match +): """A render-time 400 on a decode request notifies the KV connector.""" mock_engine = _build_mock_engine() mock_engine.notify_kv_transfer_request_rejected = AsyncMock() @@ -2870,9 +2882,10 @@ async def test_chat_kv_transfer_prompt_token_ids_rejection_notifies_kv_connector request = ChatCompletionRequest( model=MODEL_NAME, messages=[{"role": "user", "content": "hi"}], - kv_transfer_params={"prompt_token_ids": [1.5], "do_remote_prefill": True}, + stream=stream, + kv_transfer_params={"prompt_token_ids": ids, "do_remote_prefill": True}, ) - with pytest.raises(VLLMValidationError, match="non-negative integers"): + with pytest.raises(VLLMValidationError, match=match): await serving_chat.create_chat_completion(request) mock_engine.notify_kv_transfer_request_rejected.assert_awaited_once_with( diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 0936dd021e..8c1d91e032 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -417,6 +417,43 @@ class BaseRenderer(ABC, Generic[_T]): return self.tokenizer.eos_token_id + def validate_token_ids( + self, token_ids: Sequence[int], *, parameter: str | None = None + ) -> None: + """Raise VLLMValidationError if a token id is out of vocabulary. + + Skipped when the tokenizer is not initialized. + """ + tokenizer = self.tokenizer + if not token_ids or tokenizer is None: + return + + max_input_id = max(token_ids) + min_input_id = min(token_ids) + + # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while + # self.model_config.get_vocab_size() is the model’s vocab size. + # For Qwen3 models, the language model has extra tokens that do + # not exist in the tokenizer, and vice versa for multimodal + # placeholder tokens in some multimodal models. + # See https://github.com/QwenLM/Qwen3/issues/29#issuecomment-1933720399 # noqa: E501 + # and https://github.com/vllm-project/vllm/pull/22471#discussion_r2312251421 # noqa: E501 + + # Here we take the max of the two to determine if a token id is + # truly out-of-vocabulary. + model_vocab_size = self.model_config.get_vocab_size() + # A negative id is out of vocabulary just like an over-large one, + # but is not caught by the upper-bound check below. Reject it here + # so it is not used as an embedding index downstream. + if min_input_id < 0: + raise VLLMValidationError( + f"Token id {min_input_id} is out of vocabulary", parameter=parameter + ) + if max_input_id > max(tokenizer.max_token_id, model_vocab_size - 1): + raise VLLMValidationError( + f"Token id {max_input_id} is out of vocabulary", parameter=parameter + ) + def get_dec_start_token_id(self) -> int: """Obtain the decoder start token id employed by an encoder/decoder model, raising an error if it is not available. diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index bbe24f7df3..ec8b58f73d 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -103,7 +103,7 @@ def _extract_allowed_tools_from_mcp_requests( def _reused_prompt_token_ids( - request: Any, messages: list[Any] | None = None + request: Any, renderer: BaseRenderer, messages: list[Any] | None = None ) -> list[int] | None: """Pop prompt token ids forwarded for decode-side reuse, if any. @@ -113,7 +113,8 @@ def _reused_prompt_token_ids( Returns None without checking the ids when ``echo`` is set or ``messages`` has non-text content, since both need ``messages`` to be rendered. - Otherwise raises VLLMValidationError if the ids are malformed. + Otherwise raises VLLMValidationError if the ids are malformed or out of + vocabulary. """ kv = getattr(request, "kv_transfer_params", None) if not isinstance(kv, dict): @@ -144,6 +145,8 @@ def _reused_prompt_token_ids( "of non-negative integers.", parameter="kv_transfer_params.prompt_token_ids", ) + # The engine checks this too, but only after a streamed response starts. + renderer.validate_token_ids(ids, parameter="kv_transfer_params.prompt_token_ids") return ids @@ -530,7 +533,7 @@ class OnlineRenderer: should_include_tools: bool = True, ): """Build Harmony (GPT-OSS) messages and engine prompt from a chat request.""" - reuse_ids = _reused_prompt_token_ids(request) + reuse_ids = _reused_prompt_token_ids(request, self.renderer) if reuse_ids: # Decode-side token reuse: feed the forwarded ids straight to the # engine. Harmony has no adjust_request hook to preserve. @@ -781,7 +784,7 @@ class OnlineRenderer: default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), ) - reuse_ids = _reused_prompt_token_ids(request, messages) + reuse_ids = _reused_prompt_token_ids(request, renderer, messages) if reuse_ids: # Decode-side token reuse: feed the forwarded ids straight to the # engine, skipping templating and tokenization. ``messages`` are not diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 3113b90f3e..fa68eb8ec2 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -556,9 +556,6 @@ class InputProcessor: prompt_input: SingletonInput, prompt_type: Literal["encoder", "decoder"], ) -> None: - model_config = self.model_config - tokenizer = self.tokenizer - prompt_ids = ( None if prompt_input["type"] == "embeds" @@ -595,34 +592,9 @@ class InputProcessor: f"by setting --limit-mm-per-prompt at startup." ) - if prompt_ids and tokenizer is not None: - max_input_id = max(prompt_ids, default=0) - min_input_id = min(prompt_ids, default=0) - - # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while - # self.model_config.get_vocab_size() is the model’s vocab size. - # For Qwen3 models, the language model has extra tokens that do - # not exist in the tokenizer, and vice versa for multimodal - # placeholder tokens in some multimodal models. - # See https://github.com/QwenLM/Qwen3/issues/29#issuecomment-1933720399 # noqa: E501 - # and https://github.com/vllm-project/vllm/pull/22471#discussion_r2312251421 # noqa: E501 - - # Here we take the max of the two to determine if a token id is - # truly out-of-vocabulary. - model_vocab_size = model_config.get_vocab_size() - # A negative id is out of vocabulary just like an over-large one, - # but is not caught by the upper-bound check below. Reject it here - # so it is not used as an embedding index downstream. This - # validation path is shared by generate, embedding and pooling - # requests, so the check covers all three. - if min_input_id < 0: - raise VLLMValidationError( - f"Token id {min_input_id} is out of vocabulary" - ) - if max_input_id > max(tokenizer.max_token_id, model_vocab_size - 1): - raise VLLMValidationError( - f"Token id {max_input_id} is out of vocabulary" - ) + # Shared by generate, embedding and pooling requests. + if prompt_ids: + self.renderer.validate_token_ids(prompt_ids) def _validate_model_inputs( self, From 08e03df94028d5e7cff96bf569459d8d2b926d9e Mon Sep 17 00:00:00 2001 From: Mehmet Cagri Date: Thu, 1 Oct 2026 11:32:25 -0500 Subject: [PATCH 204/248] [Bugfix][ROCm] Use a zero default for masked scales in the MXFP8 GEMM (#59454) Signed-off-by: Mehmet Cagri Kaymak --- .../kernels/linear/mxfp8/rocm_block32_gemm.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py b/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py index 875c5314be..3d5e9c8357 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py +++ b/vllm/model_executor/kernels/linear/mxfp8/rocm_block32_gemm.py @@ -145,16 +145,16 @@ def _block32_tiled_kernel( for kk in range(0, K_PER_SPLIT, BLOCK_K): if EVEN_K: x = tl.load(x_ptrs, mask=m_mask[:, None], other=0.0) - xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=127) + xs = tl.load(xs_ptrs, mask=m_mask[:, None], other=0) w = tl.load(w_ptrs, mask=n_mask[:, None], other=0.0) - ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=127) + ws = tl.load(ws_ptrs, mask=n_mask[:, None], other=0) else: k_ok = (offs_k + kk) < K s_ok = (offs_sk + kk // 32) < K // 32 x = tl.load(x_ptrs, mask=m_mask[:, None] & k_ok[None, :], other=0.0) - xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=127) + xs = tl.load(xs_ptrs, mask=m_mask[:, None] & s_ok[None, :], other=0) w = tl.load(w_ptrs, mask=n_mask[:, None] & k_ok[None, :], other=0.0) - ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=127) + ws = tl.load(ws_ptrs, mask=n_mask[:, None] & s_ok[None, :], other=0) acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc) x_ptrs += BLOCK_K w_ptrs += BLOCK_K @@ -240,9 +240,9 @@ def _block32_packed_kernel( ws_mask = ws_mask & (wg < K // 32) x = tl.load(x_ptr + rows[:, None] * K + xk, mask=x_mask, other=0.0) w = tl.load(w_ptr + cols[:, None] * K + wk, mask=w_mask, other=0.0) - xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=127) + xs = tl.load(xs_ptr + rows[:, None] * (K // 32) + xg, mask=xs_mask, other=0) ws = tl.load( - ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=127 + ws_ptr + (cols[:, None] // 32) * (K // 32) + wg, mask=ws_mask, other=0 ) acc = tl.dot_scaled(x, xs, "e4m3", w.T, ws, "e4m3", acc=acc) # Keep only products of matching K panels: the block diagonal. From 362cfb72de36efca1a16c1c92c64e82729536e3d Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Thu, 1 Oct 2026 12:54:54 -0400 Subject: [PATCH 205/248] [Agents] Expose PR checklist skill to Claude (#59638) Signed-off-by: mgoin Co-authored-by: Claude --- .claude/skills/pr-checklist | 1 + 1 file changed, 1 insertion(+) create mode 120000 .claude/skills/pr-checklist diff --git a/.claude/skills/pr-checklist b/.claude/skills/pr-checklist new file mode 120000 index 0000000000..a908a31603 --- /dev/null +++ b/.claude/skills/pr-checklist @@ -0,0 +1 @@ +../../.agents/skills/pr-checklist \ No newline at end of file From 26bfdfb6bc5ef85e2128b9ddcf27c369356edd3b Mon Sep 17 00:00:00 2001 From: fululi12 Date: Thu, 1 Oct 2026 10:00:15 -0700 Subject: [PATCH 206/248] [ROCm][Perf] Parallelise AITER MLA page-index expansion over token chunks (#57978) Signed-off-by: lifulu Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../v1/attention/test_expand_page_indices.py | 159 ++++++++++++++++++ .../test_rocm_aiter_mla_mtp_split.py | 20 ++- .../attention/backends/mla/rocm_aiter_mla.py | 60 ++++--- 3 files changed, 217 insertions(+), 22 deletions(-) create mode 100644 tests/v1/attention/test_expand_page_indices.py diff --git a/tests/v1/attention/test_expand_page_indices.py b/tests/v1/attention/test_expand_page_indices.py new file mode 100644 index 0000000000..73163fe13d --- /dev/null +++ b/tests/v1/attention/test_expand_page_indices.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Equivalence tests for the AITER MLA page-index expansion kernel. + +`_expand_page_indices_kernel` turns a block table into the per-token flat page +indices the AITER MLA decode kernel consumes (it always runs with page_size=1 +internally, so `kv_buffer` is flattened and each token needs its own index). + +The kernel was parallelised over token chunks as well as requests; at +concurrency 1 the previous `(num_reqs,)` grid put a single workgroup in a serial +loop over the whole sequence. These tests pin the result to a straightforward +PyTorch expansion so the parallelisation cannot change what is produced. + +Both block-table strides are passed by the caller, so a non-contiguous view is +covered explicitly rather than assumed away. +""" + +import pytest +import torch + +from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( + _expand_page_indices_kernel, +) + + +def _reference(block_table, cu_num_tokens, kernel_block_size, out_len, device): + """Per-token flat index, written the obvious way.""" + out = torch.full((out_len,), -1, dtype=torch.int64, device=device) + num_reqs = cu_num_tokens.numel() - 1 + for r in range(num_reqs): + start = int(cu_num_tokens[r]) + end = int(cu_num_tokens[r + 1]) + n = end - start + if n <= 0: + continue + tok = torch.arange(n, device=device) + blk = tok // kernel_block_size + off = tok % kernel_block_size + out[start:end] = block_table[r, blk].to(torch.int64) * kernel_block_size + off + return out + + +def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device): + out = torch.full((out_len,), -1, dtype=torch.int64, device=device) + num_reqs = cu_num_tokens.numel() - 1 + max_tokens_per_req = block_table.shape[1] * kernel_block_size + num_chunks = max(1, -(-max_tokens_per_req // 1024)) + _expand_page_indices_kernel[(num_reqs, num_chunks)]( + out, + block_table, + block_table.stride(0), + block_table.stride(1), + cu_num_tokens, + KERNEL_BLOCK_SIZE=kernel_block_size, + BLOCK_SIZE=1024, + ) + return out + + +def _make(seq_lens, kernel_block_size, device, noncontig=False): + num_reqs = len(seq_lens) + max_blocks = ( + max(1, max((s + kernel_block_size - 1) // kernel_block_size for s in seq_lens)) + + 2 + ) + width = max_blocks * 2 if noncontig else max_blocks + torch.manual_seed(0) + bt = torch.randint(0, 10_000, (num_reqs, width), dtype=torch.int64, device=device) + if noncontig: + bt = bt[:, ::2] + assert not bt.is_contiguous() + cu = torch.zeros(num_reqs + 1, dtype=torch.int64, device=device) + cu[1:] = torch.tensor(seq_lens, dtype=torch.int64, device=device).cumsum(0) + return bt, cu, int(cu[-1]) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +@pytest.mark.parametrize( + "seq_lens", [[1], [128], [4096], [1, 1], [4096, 7], [37, 4096, 1], [1024] * 8] +) +@pytest.mark.parametrize("kernel_block_size", [1, 16, 64]) +def test_matches_reference(seq_lens, kernel_block_size): + bt, cu, total = _make(seq_lens, kernel_block_size, "cuda") + got = _run(bt, cu, kernel_block_size, total, "cuda") + want = _reference(bt, cu, kernel_block_size, total, "cuda") + torch.testing.assert_close(got, want, atol=0, rtol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +@pytest.mark.parametrize("kernel_block_size", [1, 16]) +def test_non_contiguous_block_table(kernel_block_size): + """Column stride != 1 must be honoured, not assumed away.""" + bt, cu, total = _make([4096, 37], kernel_block_size, "cuda", noncontig=True) + got = _run(bt, cu, kernel_block_size, total, "cuda") + want = _reference(bt, cu, kernel_block_size, total, "cuda") + torch.testing.assert_close(got, want, atol=0, rtol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +def test_ragged_batch_leaves_other_rows_untouched(): + """A short request must not have its tail written by a longer one's chunks.""" + kbs = 16 + bt, cu, total = _make([4096, 1, 512], kbs, "cuda") + out = torch.full((total + 8,), -999, dtype=torch.int64, device="cuda") + num_chunks = max(1, -(-(bt.shape[1] * kbs) // 1024)) + _expand_page_indices_kernel[(3, num_chunks)]( + out, + bt, + bt.stride(0), + bt.stride(1), + cu, + KERNEL_BLOCK_SIZE=kbs, + BLOCK_SIZE=1024, + ) + want = _reference(bt, cu, kbs, total, "cuda") + torch.testing.assert_close(out[:total], want, atol=0, rtol=0) + # The slack past the last request must be untouched. + assert torch.all(out[total:] == -999) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +def test_zero_length_request_writes_nothing(): + """A zero-token request in a padded batch must produce no stores.""" + kbs = 16 + bt, cu, total = _make([256, 0, 128], kbs, "cuda") + got = _run(bt, cu, kbs, total, "cuda") + want = _reference(bt, cu, kbs, total, "cuda") + torch.testing.assert_close(got, want, atol=0, rtol=0) + assert int(cu[2] - cu[1]) == 0 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") +def test_over_provisioned_chunks_do_not_change_result(): + """Extra chunks must be inert. + + Each program covers `chunk_idx * BLOCK_SIZE + [0, BLOCK_SIZE)`, so the grid + has to span the request: 8192 tokens need at least ceil(8192/1024) == 8 + chunks, and fewer would simply leave the tail unwritten. The caller sizes + the grid from `block_table.shape[1] * kernel_block_size`, an upper bound + over the batch, so in a ragged batch most requests get more chunks than + they need. That over-provisioning is the case worth pinning: the surplus + programs must return early and change nothing. + """ + kbs = 1 + bt, cu, total = _make([8192], kbs, "cuda") + ref = _reference(bt, cu, kbs, total, "cuda") + required = -(-total // 1024) + for num_chunks in (required, required * 2, required * 8): + out = torch.full((total,), -1, dtype=torch.int64, device="cuda") + _expand_page_indices_kernel[(1, num_chunks)]( + out, + bt, + bt.stride(0), + bt.stride(1), + cu, + KERNEL_BLOCK_SIZE=kbs, + BLOCK_SIZE=1024, + ) + torch.testing.assert_close(out, ref, atol=0, rtol=0) diff --git a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py index 309bfd609c..da141acf68 100644 --- a/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py +++ b/tests/v1/attention/test_rocm_aiter_mla_mtp_split.py @@ -56,12 +56,21 @@ class _ExpandPageIndicesKernel: self, page_indices, block_table_tensor, - stride, + stride_0, + stride_1, paged_kv_indptr, *, KERNEL_BLOCK_SIZE, BLOCK_SIZE, ): + # Both block-table strides are passed explicitly: the kernel no longer + # assumes the column stride is 1. This stand-in indexes the tensor + # directly so torch applies the strides for it, but the values are + # checked against the tensor actually handed in -- a caller passing the + # wrong stride is exactly the bug the parameter exists to prevent, and + # silently ignoring them here would hide it. + assert stride_0 == block_table_tensor.stride(0) + assert stride_1 == block_table_tensor.stride(1) self.kernel_block_size = KERNEL_BLOCK_SIZE for req_idx in range(self.grid[0]): out_start = int(paged_kv_indptr[req_idx].item()) @@ -1052,7 +1061,14 @@ def test_decode_expands_kernel_block_page_indices(monkeypatch): metadata.paged_kv_indices[: expected_indices.numel()], expected_indices, ) - assert expand_kernel.grid == (seq_lens.numel(),) + # The launch now carries a second grid dimension over token chunks, so + # the expansion is spread across the context instead of one workgroup + # walking it serially. The width comes from the block-table upper bound + # on tokens per request, which is available host-side and so costs no + # device sync; derived here rather than hard-coded so the assertion + # still describes the launch if the chunk size changes. + expected_chunks = max(1, -(-block_table.shape[1] * kernel_block_size // 1024)) + assert expand_kernel.grid == (seq_lens.numel(), expected_chunks) assert expand_kernel.kernel_block_size == kernel_block_size diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py index ab6c11d41a..c833a56a83 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla.py @@ -1331,10 +1331,17 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): # to the original _copy_page_indices_kernel). # When kernel_block_size=K>1, block_table entry b covering K tokens # gets expanded to flat indices b*K, b*K+1, ..., b*K+(K-1). - _expand_page_indices_kernel[(num_reqs,)]( + # Chunk count comes from the block table width, an upper bound on + # tokens per request that is available host-side, so this adds no + # device synchronisation. Programs whose chunk lies past + # num_tokens mask out entirely. + max_tokens_per_req = block_table_tensor.shape[1] * self.kernel_block_size + num_chunks = max(1, cdiv(max_tokens_per_req, 1024)) + _expand_page_indices_kernel[(num_reqs, num_chunks)]( self.paged_kv_indices, block_table_tensor, block_table_tensor.stride(0), + block_table_tensor.stride(1), paged_kv_indptr, KERNEL_BLOCK_SIZE=self.kernel_block_size, BLOCK_SIZE=1024, @@ -1573,7 +1580,8 @@ class AiterMLAMetadataBuilder(MLACommonMetadataBuilder[AiterMLAMetadata]): def _expand_page_indices_kernel( page_indices, block_table, - block_table_stride, + block_table_stride_0, + block_table_stride_1, cu_num_tokens, KERNEL_BLOCK_SIZE: tl.constexpr, BLOCK_SIZE: tl.constexpr, @@ -1591,32 +1599,44 @@ def _expand_page_indices_kernel( When KERNEL_BLOCK_SIZE=K: block table entry b (covering K tokens) is expanded to flat indices b*K, b*K+1, ..., b*K+(K-1). """ + # One program per (request, token-chunk). Parallelising over requests + # alone degenerates at low concurrency: with num_reqs == 1 a single + # workgroup walked the whole sequence in a serial loop. req_idx = tl.program_id(0) - row_ptr = block_table + req_idx * block_table_stride + chunk_idx = tl.program_id(1) + row_ptr = block_table + req_idx * block_table_stride_0 start_idx = tl.load(cu_num_tokens + req_idx) num_tokens = tl.load(cu_num_tokens + req_idx + 1) - start_idx - offset = tl.arange(0, BLOCK_SIZE) - for i in tl.range(0, num_tokens, BLOCK_SIZE): - token_offsets = i + offset - mask = token_offsets < num_tokens + # The grid is sized from the block table width, an upper bound over all + # requests, so a ragged batch launches chunks past a short request's end. + # Returning here keeps those programs from issuing masked-out loads and + # stores at all. + chunk_start = chunk_idx * BLOCK_SIZE + if chunk_start >= num_tokens: + return - # Which block in the block table does this token belong to? - block_idx = token_offsets // KERNEL_BLOCK_SIZE - # Offset within that block - offset_in_block = token_offsets % KERNEL_BLOCK_SIZE + token_offsets = chunk_start + tl.arange(0, BLOCK_SIZE) + mask = token_offsets < num_tokens - # Load the block ID from the block table - block_ids = tl.load(row_ptr + block_idx, mask=mask) + # Which block in the block table does this token belong to? + block_idx = token_offsets // KERNEL_BLOCK_SIZE + # Offset within that block + offset_in_block = token_offsets % KERNEL_BLOCK_SIZE - # Compute flat index in the flattened kv_buffer - flat_indices = block_ids * KERNEL_BLOCK_SIZE + offset_in_block + # Load the block ID from the block table + # Both strides are taken from the caller: the block table is a view owned + # elsewhere, so a unit column stride must not be assumed. + block_ids = tl.load(row_ptr + block_idx * block_table_stride_1, mask=mask) - tl.store( - page_indices + start_idx + token_offsets, - flat_indices, - mask=mask, - ) + # Compute flat index in the flattened kv_buffer + flat_indices = block_ids * KERNEL_BLOCK_SIZE + offset_in_block + + tl.store( + page_indices + start_idx + token_offsets, + flat_indices, + mask=mask, + ) class AiterMLAHelper: From 3a6963664537ed21172e2ec12e96e3a2dcd3718c Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 1 Oct 2026 13:02:31 -0400 Subject: [PATCH 207/248] [Bugfix][HiSparse] Fix a chunked-prefill preemption livelock (#59494) Signed-off-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- tests/v1/core/test_prefix_caching.py | 61 +++++++++++++++++++- vllm/v1/core/single_type_kv_cache_manager.py | 13 +++-- vllm/v1/hisparse/coordinator.py | 61 +++++++++++++------- 3 files changed, 106 insertions(+), 29 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index e4c54e880c..027bb67b18 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -1259,6 +1259,63 @@ def test_hisparse_cleanup_publishes_only_finalized_terminal_pages( assert not coordinator.has_pending_work() +@pytest.mark.parametrize("num_pages,reads_host", [(4, True), (3, False)]) +def test_hisparse_prefill_reads_host_once_it_fills_admission_window( + num_pages, reads_host +): + """A prefill switches to host reads once it fills its admission window. + + Admission caps each request's resident pages at the in-flight window, + assuming older pages move to host, but pinned pages were released only + once free blocks fell below a fixed watermark. A chunked prefill larger + than the watermark outgrew the pool first and was preempted. + """ + window_pages = 4 + manager = make_hisparse_kv_cache_manager( + 32, + 16, + enable_caching=True, + max_in_flight_tokens=window_pages * HISPARSE_BLOCK_SIZE, + ) + coordinator = get_hisparse_coordinator(manager) + tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE)) + request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert manager.allocate_slots(request, num_pages * HISPARSE_BLOCK_SIZE) is not None + pool = coordinator.gpu_pool + assert pool is not None + assert pool.get_num_free_blocks() >= coordinator.transition_watermark + + hot_manager = coordinator.hot_managers[0] + assert (request.request_id in hot_manager.hot_required) == reads_host + + +def test_hisparse_preempted_prefill_resumes_from_durable_prefix(): + """A prefill preempted mid-way must resume from the pages already on host. + + Host writes trail a chunked prefill by a chunk, and host pages were only + published once the whole computed prefix was durable. Each chunk moved + that target past the finished writes, so nothing was published before the + prefill ended: a prefill preempted each time it outgrew the GPU pool + recomputed from the start forever. + """ + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + coordinator = get_hisparse_coordinator(manager) + chunk = 2 * HISPARSE_BLOCK_SIZE + tokens = list(range(3 * chunk)) + request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert manager.allocate_slots(request, chunk) is not None + request.num_computed_tokens = chunk + first_chunk_writes = coordinator.build_offload_command().page_transfers + assert manager.allocate_slots(request, chunk) is not None + request.num_computed_tokens = 2 * chunk + acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes} + coordinator.update_spills(acks, acks) + manager.free(request) + + resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert manager.get_computed_blocks(resumed)[1] == chunk + + def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity(): """Old completions retain their pages through pressure and request-ID reuse.""" manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True) @@ -1285,7 +1342,9 @@ def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity() assert not {block.block_id for block in old_blocks[:2]} & { block.block_id for block in pressure + new_blocks } - assert all(block.block_hash is None for block in old_blocks) + # Only the page every worker finished writing is published. + assert old_blocks[0].block_hash is not None + assert all(block.block_hash is None for block in old_blocks[1:]) coordinator.update_spills({}, partial) repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256) assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index c8b9c54435..a097c9e924 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -112,7 +112,7 @@ class SingleTypeKVCacheManager(ABC): self.kv_cache_spec = kv_cache_spec self.block_pool = block_pool self.enable_caching = enable_caching - self._max_admission_blocks_per_request = max_admission_blocks_per_request + self.max_admission_blocks_per_request = max_admission_blocks_per_request # Record newly allocated block ids only when worker-side zeroing will # consume them and this manager holds a spec type that gets zeroed. self._record_new_block_ids = ( @@ -202,7 +202,7 @@ class SingleTypeKVCacheManager(ABC): model in spec decode). w/o spec decode, it is num_tokens; with spec decode, it is num_tokens - num_lookahead_tokens. apply_admission_cap: If True, clamp by `num_required_blocks` by - `_max_admission_blocks_per_request`for recycling-aware specs + `max_admission_blocks_per_request`for recycling-aware specs (SWA, chunked-local). prefill_end: The token index the request's prefill ends at, the same value the scheduler splits chunks against. Mamba reserves @@ -214,7 +214,7 @@ class SingleTypeKVCacheManager(ABC): """ num_required_blocks = cdiv(num_tokens, self.block_size) - if apply_admission_cap and self._max_admission_blocks_per_request is not None: + if apply_admission_cap and self.max_admission_blocks_per_request is not None: # Recycling-aware specs (SWA, chunked-local) cap the per-request # reservation here so admission matches the startup pool sizer # (`SlidingWindowSpec.max_admission_blocks_per_request` / its @@ -225,7 +225,7 @@ class SingleTypeKVCacheManager(ABC): # Drift between the two would re-introduce the deadlock from # issue #39734 or, worse, mid-prefill OOM. num_required_blocks = min( - num_required_blocks, self._max_admission_blocks_per_request + num_required_blocks, self.max_admission_blocks_per_request ) num_req_blocks = len(self.req_to_blocks.get(request_id, ())) @@ -2488,6 +2488,8 @@ class HiSparseHotManager(_HiSparseAuxiliaryManager): class HiSparseResidentManager(_HiSparseAuxiliaryManager): """Track GPU-resident pages for otherwise host-backed KV.""" + max_admission_blocks_per_request: int + def get_num_blocks_to_allocate( self, request_id: str, @@ -2514,8 +2516,7 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): host_pages = cdiv(num_local_computed_tokens, self.block_size) required = cdiv(num_tokens, self.block_size) if apply_admission_cap: - assert self._max_admission_blocks_per_request is not None - required = min(required, self._max_admission_blocks_per_request) + required = min(required, self.max_admission_blocks_per_request) return max(required - max(existing, host_pages), 0) def allocate_external_computed_blocks( diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index 01a6392964..1a783b506b 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -318,6 +318,19 @@ class HiSparseCoordinator: manager.has_hot(request_id) for manager in self.hot_managers ) + def _fills_admission_window(self, request_id: str) -> bool: + """Whether the request holds the resident pages admission reserved. + + Admission caps a request's resident pages at the in-flight window, + assuming older pages move to host; keeping them pinned past it lets a + chunked prefill outgrow the pool it was admitted into. + """ + return any( + len(manager.req_to_blocks.get(request_id, ())) + >= manager.max_admission_blocks_per_request + for manager in self.resident_managers + ) + def _resident_page_blocks( self, request_id: str, page_idx: int ) -> list[KVCacheBlock] | None: @@ -390,8 +403,9 @@ class HiSparseCoordinator: A request that can read from host releases every clean sealed page to the pool. One that cannot keeps its pages pinned until the shared pool - runs low, then asks for a hot region. Pages remain pinned until that region - is allocated on a subsequent scheduling pass. + runs low, or until it holds the resident pages admission reserved for + it, then asks for a hot region. Pages remain pinned until that region is + allocated on a subsequent scheduling pass. """ if not self.resident_managers: return @@ -403,7 +417,10 @@ class HiSparseCoordinator: state.pages_to_adopt = 0 if not self._can_read_from_host(request_id): assert self.gpu_pool is not None - if self.gpu_pool.get_num_free_blocks() >= self.transition_watermark: + if ( + self.gpu_pool.get_num_free_blocks() >= self.transition_watermark + and not self._fills_admission_window(request_id) + ): return for manager in self.hot_managers: manager.require_hot(request_id) @@ -453,47 +470,47 @@ class HiSparseCoordinator: *, replay_boundaries: Sequence[int], ) -> None: - """Publish host-source hashes only after their pages are durable.""" + """Publish host-source hashes of the pages that are already durable.""" manager = self.host_manager if manager is None: return - num_pages = num_computed_tokens // manager.block_size request_id = request.request_id - state = self._get_request_state(request_id) - if state.ready_prefix_pages >= num_pages: - manager.publish_blocks( - request, - num_computed_tokens, - retention_interval=retention_interval, - replay_boundaries=replay_boundaries, - ) - self._record_copies(request_id, num_computed_tokens) - state.publication = None - return - state.publication = _PendingPublication( + self._get_request_state(request_id).publication = _PendingPublication( request=request, num_computed_tokens=num_computed_tokens, - num_pages=num_pages, + num_pages=num_computed_tokens // manager.block_size, retention_interval=retention_interval, replay_boundaries=replay_boundaries, ) + self._publish_host_blocks_if_ready(request_id) def _publish_host_blocks_if_ready(self, request_id: str) -> None: + """Publish the durable prefix of the pending publication. + + Host writes trail a long prefill by about a chunk, so waiting for the + whole computed prefix would publish nothing until the prefill ends, and + a prefill preempted before then would recompute from the start. + """ state = self.request_states.get(request_id) if state is None: return publication = state.publication - if publication is None or state.ready_prefix_pages < publication.num_pages: + if publication is None: return assert self.host_manager is not None + num_tokens = min( + publication.num_computed_tokens, + state.ready_prefix_pages * self.host_manager.block_size, + ) self.host_manager.publish_blocks( publication.request, - publication.num_computed_tokens, + num_tokens, retention_interval=publication.retention_interval, replay_boundaries=publication.replay_boundaries, ) - self._record_copies(request_id, publication.num_computed_tokens) - state.publication = None + self._record_copies(request_id, num_tokens) + if state.ready_prefix_pages >= publication.num_pages: + state.publication = None def record_pending_host_import(self, request_id: str, num_tokens: int) -> None: """Note a prefix an external load is populating in host pages.""" From 083060d046a84c685fdb4b8932d95869de060543 Mon Sep 17 00:00:00 2001 From: Misha Goin Date: Thu, 1 Oct 2026 13:28:27 -0400 Subject: [PATCH 208/248] [CI/Build] Add agents auto-label rule (#59641) Signed-off-by: mgoin Co-authored-by: Claude --- .github/mergify.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/mergify.yml b/.github/mergify.yml index 5006fd798b..9151c5c0a1 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -748,3 +748,19 @@ pull_request_rules: label: add: - vllm-ir + +- name: label-agents + description: Automatically apply agents label + conditions: + - label != stale + - or: + - files~=(?:^|/)AGENTS\.md$ + - files~=^\.agents/ + - files~=^\.claude/ + # The [Agents] title tag only: a bare "agents" also matches CI, K8s and + # NIXL agents. + - title~=(?i)\[agents\] + actions: + label: + add: + - agents From e9be5323f0f864c1dc5a22a138530f3e106ebbb6 Mon Sep 17 00:00:00 2001 From: roipony Date: Thu, 1 Oct 2026 21:06:43 +0300 Subject: [PATCH 209/248] [Perf] Integrate flash-maxsim Triton kernels for late-interaction scoring (#40337) Signed-off-by: roi.pony Co-authored-by: roi.pony Co-authored-by: Taneem Ibrahim --- .../v1/worker/test_late_interaction_runner.py | 105 +++++++++ vllm/config/pooler.py | 8 + vllm/entrypoints/cli/serve.py | 3 + .../entrypoints/launchers/api_server/entry.py | 3 + vllm/entrypoints/launchers/cli_args.py | 22 +- vllm/v1/pool/flash_maxsim/__init__.py | 7 + vllm/v1/pool/flash_maxsim/_common.py | 85 ++++++++ .../pool/flash_maxsim/flash_maxsim_rerank.py | 204 ++++++++++++++++++ vllm/v1/pool/late_interaction_runner.py | 69 +++++- vllm/v1/worker/gpu/pool/pooling_runner.py | 6 +- 10 files changed, 508 insertions(+), 4 deletions(-) create mode 100644 vllm/v1/pool/flash_maxsim/__init__.py create mode 100644 vllm/v1/pool/flash_maxsim/_common.py create mode 100644 vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py diff --git a/tests/v1/worker/test_late_interaction_runner.py b/tests/v1/worker/test_late_interaction_runner.py index 054bfbdabd..4befbf72cc 100644 --- a/tests/v1/worker/test_late_interaction_runner.py +++ b/tests/v1/worker/test_late_interaction_runner.py @@ -10,6 +10,7 @@ from vllm.v1.pool.late_interaction import ( LATE_INTERACTION_MODE_CACHE_QUERY, build_late_interaction_doc_params, build_late_interaction_query_params, + compute_maxsim_score_batched, ) from vllm.v1.pool.late_interaction_runner import LateInteractionRunner @@ -152,3 +153,107 @@ def test_invalid_query_uses_raises(): req_ids=["query-req"], finished_mask=[True], ) + + +# --------------------------------------------------------------------------- +# Fused Triton scoring path (PR #40337). +# --------------------------------------------------------------------------- +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_score_flash_matches_fp64_oracle(): + """Shared-query scoring through the fused kernel must match a per-pair + fp64 MaxSim oracle on ragged doc lengths, including length-1 docs.""" + torch.manual_seed(0) + d = 128 + q = torch.randn(32, d, device="cuda", dtype=torch.float16) + doc_lengths = [180, 37, 512, 1, 300, 64, 1030, 256] + docs = [ + torch.randn(ld, d, device="cuda", dtype=torch.float16) for ld in doc_lengths + ] + + runner = LateInteractionRunner() + assert runner._flash_enabled + scores = runner._score([q] * len(docs), docs) + assert runner._flash_enabled, "kernel path must not have fallen back" + + for i, doc in enumerate(docs): + ref = (q.double() @ doc.double().T).max(dim=1).values.sum().float() + torch.testing.assert_close( + scores[i].to(torch.float32), ref, atol=5e-2, rtol=1e-3 + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_score_multi_query_uses_reference_path(): + """Distinct queries per pair route to the reference scorer and agree + with it exactly (no kernel dispatch for the N:N pattern).""" + torch.manual_seed(1) + d = 128 + queries = [torch.randn(16, d, device="cuda") for _ in range(3)] + docs = [torch.randn(ld, d, device="cuda") for ld in (50, 3, 77)] + + runner = LateInteractionRunner() + scores = runner._score(queries, docs) + ref = compute_maxsim_score_batched(queries, docs) + for s, r in zip(scores, ref): + torch.testing.assert_close(s, r) + + +def test_score_flash_failure_disables_and_falls_back(monkeypatch): + """A kernel failure must serve the batch via the reference scorer and + disable the kernel path for the rest of the process.""" + runner = LateInteractionRunner() + runner._flash_enabled = True + + def _boom(query, docs): + raise RuntimeError("synthetic kernel failure") + + monkeypatch.setattr(runner, "_score_flash_shared_query", _boom) + q = torch.randn(8, 32) + docs = [torch.randn(5, 32), torch.randn(7, 32)] + # CPU tensors already skip the kernel; force the dispatch condition. + monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True)) + scores = runner._score([q, q], docs) + assert not runner._flash_enabled + ref = compute_maxsim_score_batched([q, q], docs) + for s, r in zip(scores, ref): + torch.testing.assert_close(s, r) + + +def test_disabled_flag_uses_reference_path(): + runner = LateInteractionRunner(enable_flash=False) + assert not runner._flash_enabled + + +# --------------------------------------------------------------------------- +# Autotune config pruning. +# --------------------------------------------------------------------------- +def _prune(monkeypatch, budget: int, named_args: dict): + from vllm.v1.pool.flash_maxsim import _common + + monkeypatch.setattr(_common, "_smem_budget", lambda: budget) + return _common._prune_configs(_common._get_configs(), named_args) + + +def _est(cfg, d_pad: int) -> int: + bq, bd = cfg.kwargs["BLOCK_Q"], cfg.kwargs["BLOCK_D"] + return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4 + + +def test_prune_configs_uses_padded_dim(monkeypatch): + """d=513 pads to 1024: survivors must fit the budget at d_pad, not at + the un-padded d (which would admit ~2x-oversized tiles).""" + budget = 166_912 # A100 opt-in + survivors = _prune(monkeypatch, budget, {"Lq": 1024, "d": 513, "d_pad": 1024}) + assert survivors + assert all(_est(c, 1024) <= budget for c in survivors) + + +def test_prune_configs_no_rejected_fallback(monkeypatch): + """When nothing fits, the fallback is the single smallest-footprint + config — never a slice of rejected configs.""" + from vllm.v1.pool.flash_maxsim import _common + + survivors = _prune(monkeypatch, 10_000, {"Lq": 1024, "d_pad": 1024}) + assert len(survivors) == 1 + smallest = min(_common._get_configs(), key=lambda c: _est(c, 1024)) + assert survivors[0].kwargs == smallest.kwargs diff --git a/vllm/config/pooler.py b/vllm/config/pooler.py index a5fbcbb622..04cf323015 100644 --- a/vllm/config/pooler.py +++ b/vllm/config/pooler.py @@ -63,6 +63,14 @@ class PoolerConfig: `None` uses the pooler's default, which is `True` in most cases. """ + enable_flash_late_interaction: bool = True + """ + Whether the engine-side late-interaction scorer may use the fused + flash-maxsim Triton kernel. Disabled automatically when the API server + is started with `--no-enable-flash-late-interaction`; the reference + scorer is used instead. + """ + ## for embedding models dimensions: int | None = None """ diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index 6a8492f4c4..1f6ca2b843 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -13,6 +13,7 @@ from vllm.entrypoints.cli.types import CLISubcommand from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server from vllm.entrypoints.launchers.cli_args import ( make_arg_parser, + propagate_flash_late_interaction, validate_parsed_serve_args, ) from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor @@ -185,6 +186,7 @@ def run_headless(args: argparse.Namespace): # Create the EngineConfig. engine_args = vllm.AsyncEngineArgs.from_cli_args(args) + propagate_flash_late_interaction(args, engine_args) usage_context = UsageContext.OPENAI_API_SERVER vllm_config = engine_args.create_engine_config( usage_context=usage_context, headless=True @@ -295,6 +297,7 @@ def run_multi_api_server(args: argparse.Namespace): listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1) engine_args = vllm.AsyncEngineArgs.from_cli_args(args) + propagate_flash_late_interaction(args, engine_args) engine_args._api_process_count = num_api_servers engine_args._api_process_rank = -1 diff --git a/vllm/entrypoints/launchers/api_server/entry.py b/vllm/entrypoints/launchers/api_server/entry.py index 02837a451d..28bb696f2c 100644 --- a/vllm/entrypoints/launchers/api_server/entry.py +++ b/vllm/entrypoints/launchers/api_server/entry.py @@ -51,6 +51,9 @@ async def build_async_engine_client( # Context manager to handle engine_client lifecycle # Ensures everything is shutdown and cleaned up on error/exit engine_args = AsyncEngineArgs.from_cli_args(args) + from ..cli_args import propagate_flash_late_interaction + + propagate_flash_late_interaction(args, engine_args) if client_config: engine_args._api_process_count = client_config.get("client_count", 1) engine_args._api_process_rank = client_config.get("client_index", 0) diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index ce6692565c..3a52cf2e2b 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -363,7 +363,10 @@ class FrontendArgs(BaseFrontendArgs): """ enable_flash_late_interaction: bool = True """If set, run pooling score MaxSim on GPU in the API server process. - Can significantly improve late-interaction scoring performance.""" + Can significantly improve late-interaction scoring performance. + When disabled, the setting is also propagated to the engine's + `PoolerConfig`, so the engine-side scorer uses the reference MaxSim + path instead of the fused Triton kernel.""" @classmethod def _customize_cli_kwargs( @@ -498,3 +501,20 @@ def create_parser_for_docs() -> FlexibleArgumentParser: prog="-m vllm.entrypoints.launchers.api_server.entry" ) return make_arg_parser(parser_for_docs) + + +def propagate_flash_late_interaction(args, engine_args) -> None: + """Propagate `--no-enable-flash-late-interaction` into the engine config. + + The frontend flag alone only disables the API-server scoring path; + mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the + engine-side scorer fall back to the reference MaxSim path too. + """ + if getattr(args, "enable_flash_late_interaction", True): + return + from vllm.config.pooler import PoolerConfig + + if engine_args.pooler_config is None: + engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False) + else: + engine_args.pooler_config.enable_flash_late_interaction = False diff --git a/vllm/v1/pool/flash_maxsim/__init__.py b/vllm/v1/pool/flash_maxsim/__init__.py new file mode 100644 index 0000000000..dd5b20b102 --- /dev/null +++ b/vllm/v1/pool/flash_maxsim/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused Triton kernels for late-interaction (MaxSim) scoring.""" + +from .flash_maxsim_rerank import flash_maxsim_rerank_direct + +__all__ = ["flash_maxsim_rerank_direct"] diff --git a/vllm/v1/pool/flash_maxsim/_common.py b/vllm/v1/pool/flash_maxsim/_common.py new file mode 100644 index 0000000000..94633e743c --- /dev/null +++ b/vllm/v1/pool/flash_maxsim/_common.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared utilities for flash-maxsim kernels: padding, autotune configs, +and pruning.""" + +from vllm.triton_utils import triton + + +def _next_pow2(x: int) -> int: + return 1 << (x - 1).bit_length() + + +def _get_configs() -> list: + """Autotune grid over (BLOCK_Q, BLOCK_D); `_prune_configs` filters by + SMEM budget and Lq at runtime, so one list is safe across GPUs.""" + return [ + triton.Config({"BLOCK_Q": 16, "BLOCK_D": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_Q": 16, "BLOCK_D": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_Q": 64, "BLOCK_D": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_Q": 64, "BLOCK_D": 128}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_Q": 128, "BLOCK_D": 64}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_Q": 128, "BLOCK_D": 128}, num_warps=8, num_stages=1), + ] + + +def _smem_budget() -> int: + """Per-block opt-in shared-memory budget of the active device (bytes): + Triton driver query, else a conservative capability table.""" + try: + driver = triton.runtime.driver.active + props = driver.utils.get_device_properties(driver.get_current_device()) + return int(props["max_shared_mem"]) + except Exception: + try: + from vllm.platforms import current_platform + + cap = current_platform.get_device_capability() + if cap is not None: + if cap.major >= 9: + return 232_448 # Hopper+: 227 KB opt-in + if cap.major == 8: + return 101_376 # worst sm_8x (86/89); A100 has more + except Exception: + pass + return 98_304 # conservative pre-Ampere default + + +def _prune_configs(configs, named_args, **kwargs): + """Reject configs that exceed the GPU's shared-memory budget. + + BLOCK_Q and BLOCK_D load fp16 tiles of size [BLOCK_*, d_pad] — the + kernels index with ``tl.arange(0, d_pad)``, so the estimate must use + the padded dimension, not the real one (d=513 pads to 1024 and + nearly doubles the tile) — and the [BLOCK_Q, BLOCK_D] score tile is + fp32. Total: + (BLOCK_Q*d_pad + BLOCK_D*d_pad)*2 + BLOCK_Q*BLOCK_D*4 bytes. + """ + Lq = named_args.get("Lq", 32) + d_pad = named_args.get("d_pad") + if d_pad is None: + d_pad = _next_pow2(max(named_args.get("d", 128), 16)) + smem_limit = _smem_budget() + + def _est(cfg): + bq = cfg.kwargs["BLOCK_Q"] + bd = cfg.kwargs["BLOCK_D"] + return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4 + + pruned = [] + for cfg in configs: + if cfg.kwargs["BLOCK_Q"] > Lq * 2: + continue + if _est(cfg) > smem_limit: + continue + pruned.append(cfg) + if pruned: + return pruned + # Nothing fits the estimate (large d_pad on a small-SMEM device). + # Return the single smallest-footprint config — a deterministic + # least-bad launch attempt — never arbitrary rejected configs. + return [min(configs, key=_est)] diff --git a/vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py b/vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py new file mode 100644 index 0000000000..d25517691a --- /dev/null +++ b/vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Flash-MaxSim rerank: one query vs variable-length docs in a packed tensor. + +The kernel reads doc embeddings directly from a packed [total_tokens, d] +tensor using per-doc (offset, length) pairs — no padding, no per-doc +tensors, no materialised [B, Lq, Ld] similarity tensor. + +Entry point: + flash_maxsim_rerank_direct(Q, batch_tensor, doc_offsets, doc_lengths, max_ld) + doc_offsets[i] = start token index of doc i in batch_tensor. + doc_lengths[i] = number of tokens in doc i. + Docs can be scattered (non-contiguous) in the batch tensor. +""" + +import torch + +from vllm.triton_utils import tl, triton + +from ._common import _get_configs, _next_pow2, _prune_configs + + +def _round_to_bucket(x): + """Round up to nearest bucket: 32, 64, 128, 256, 512, 1024, 2048, 4096.""" + for b in [32, 64, 128, 256, 512, 1024, 2048, 4096]: + if x <= b: + return b + return _next_pow2(x) + + +# Kernel: shared Q, scattered D via (offset, length) per doc + + +@triton.autotune( + configs=_get_configs(), + key=["Lq_bucket", "max_Ld_bucket", "d_pad"], + prune_configs_by={"early_config_prune": _prune_configs}, +) +@triton.jit +def _maxsim_rerank_kernel( + Q_ptr, + D_ptr, + doc_offsets_ptr, + doc_lengths_ptr, + scores_ptr, + B, + Lq, + Lq_bucket: tl.constexpr, + max_Ld_bucket: tl.constexpr, + d: tl.constexpr, + d_pad: tl.constexpr, + stride_q_t, + stride_q_d, + stride_d_t, + stride_d_d, + BLOCK_Q: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """One program per doc. Q shared (read from same location). D scattered.""" + pid = tl.program_id(0) + if pid >= B: + return + + # Each doc has its own offset and length in the batch tensor. + # int64: (d_start + d_off) * stride overflows int32 past ~16.7M batch + # tokens (same widen-on-load as the pairs kernel below; the host-side + # offsets tensor stays int32, which indexes any realistic batch). + d_start = tl.load(doc_offsets_ptr + pid).to(tl.int64) + d_len = tl.load(doc_lengths_ptr + pid).to(tl.int32) + + k_off = tl.arange(0, d_pad) + k_mask = k_off < d + score_acc = tl.zeros([], dtype=tl.float32) + + # Q is shared — always read from start of Q_ptr + for q_blk in tl.static_range(0, Lq_bucket, BLOCK_Q): + q_off = q_blk + tl.arange(0, BLOCK_Q) + q_valid = q_off < Lq + + Q_block = tl.load( + Q_ptr + q_off[:, None] * stride_q_t + k_off[None, :] * stride_q_d, + mask=q_valid[:, None] & k_mask[None, :], + other=0.0, + ).to(tl.float16) + + m = tl.full([BLOCK_Q], float("-inf"), dtype=tl.float32) + + for d_blk in range(0, max_Ld_bucket, BLOCK_D): + d_off = d_blk + tl.arange(0, BLOCK_D) + d_valid = d_off < d_len + + D_block = tl.load( + D_ptr + + (d_start + d_off[:, None]) * stride_d_t + + k_off[None, :] * stride_d_d, + mask=d_valid[:, None] & k_mask[None, :], + other=0.0, + ).to(tl.float16) + + S = tl.dot(Q_block, tl.trans(D_block)) + S = tl.where(d_valid[None, :], S, float("-inf")) + m = tl.maximum(m, tl.max(S, axis=1)) + + m = tl.where(q_valid, m, 0.0) + score_acc += tl.sum(m) + + tl.store(scores_ptr + pid, score_acc) + + +# Internal helper + + +def _run_rerank_kernel(Q, D_tensor, doc_offsets, doc_lengths, max_seqlen_d): + """Common kernel launch for both rerank and rerank_direct.""" + assert Q.is_cuda, f"Q must be on CUDA, got {Q.device}" + assert D_tensor.is_cuda, f"D must be on CUDA, got {D_tensor.device}" + Lq, d = Q.shape + B = doc_offsets.shape[0] + d_pad = _next_pow2(max(d, 16)) + + Q = Q.contiguous().half() + # D_tensor is NOT copied — kernel reads from it directly + if not D_tensor.is_contiguous(): + D_tensor = D_tensor.contiguous() + if D_tensor.dtype != torch.float16: + D_tensor = D_tensor.half() + + doc_offsets = doc_offsets.to(torch.int32).contiguous() + doc_lengths = doc_lengths.to(torch.int32).contiguous() + scores = torch.empty(B, device=Q.device, dtype=torch.float32) + + if B == 0: + return scores + + # Pad Q dimension if needed (Q is small, safe to copy). + # D is NOT padded — the kernel uses k_mask to handle d < d_pad. + # This preserves zero-copy: D_tensor is read in-place. + if d < d_pad: + Q_pad = torch.zeros(Lq, d_pad, device=Q.device, dtype=Q.dtype) + Q_pad[:, :d] = Q + else: + Q_pad = Q + D_pad = D_tensor + + Lq_bucket = _round_to_bucket(Lq) + max_ld_bucket = _round_to_bucket(max_seqlen_d) + + _maxsim_rerank_kernel[(B,)]( + Q_pad, + D_pad, + doc_offsets, + doc_lengths, + scores, + B, + Lq, + Lq_bucket, + max_ld_bucket, + d, + d_pad, + Q_pad.stride(0), + Q_pad.stride(1), + D_pad.stride(0), + D_pad.stride(1), + ) + return scores + + +# Public API + + +def flash_maxsim_rerank_direct( + Q: torch.Tensor, + batch_tensor: torch.Tensor, + doc_offsets: torch.Tensor, + doc_lengths: torch.Tensor, + max_seqlen_d: int, +) -> torch.Tensor: + """TRUE zero-copy: score query against docs scattered in a batch tensor. + + The kernel reads doc embeddings directly from batch_tensor at the + positions specified by doc_offsets. No torch.stack, no torch.cat, + no copy of any kind. The batch tensor is the model's output. + + Memory for doc scoring: 0 bytes additional. + + Args: + Q: [Lq, d] — single query embedding (from cache) + batch_tensor: [total_tokens, d] — the model's projected output tensor. + Contains ALL requests' tokens (queries + docs + others). + doc_offsets: [B] int32 — start token index of each doc in batch_tensor + doc_lengths: [B] int32 — number of tokens per doc + max_seqlen_d: int — max(doc_lengths) + + Returns: + scores: [B] float32 — one MaxSim score per document + + """ + assert Q.dim() == 2, f"Q must be 2D [Lq, d], got {Q.dim()}D" + assert batch_tensor.dim() == 2, ( + f"batch_tensor must be 2D, got {batch_tensor.dim()}D" + ) + assert Q.shape[1] == batch_tensor.shape[1] + + return _run_rerank_kernel(Q, batch_tensor, doc_offsets, doc_lengths, max_seqlen_d) diff --git a/vllm/v1/pool/late_interaction_runner.py b/vllm/v1/pool/late_interaction_runner.py index da87c8f05d..7ea39f5754 100644 --- a/vllm/v1/pool/late_interaction_runner.py +++ b/vllm/v1/pool/late_interaction_runner.py @@ -4,6 +4,7 @@ from collections.abc import Iterable import torch +from vllm.logger import init_logger from vllm.pooling_params import PoolingParams from vllm.v1.outputs import PoolerOutput from vllm.v1.pool.late_interaction import ( @@ -12,17 +13,29 @@ from vllm.v1.pool.late_interaction import ( compute_maxsim_score_batched, ) +logger = init_logger(__name__) + class LateInteractionRunner: """Worker-side state and postprocessing for late-interaction scoring.""" - def __init__(self) -> None: + def __init__(self, enable_flash: bool = True) -> None: # query_key -> token embeddings for late-interaction scoring. self._query_cache: dict[str, torch.Tensor] = {} # query_key -> remaining number of docs that should use this query. self._query_uses: dict[str, int] = {} # doc request id -> query key. self._doc_query_keys: dict[str, str] = {} + # Fused Triton scoring (PoolerConfig.enable_flash_late_interaction + # gates it; a runtime kernel failure disables it for the process). + if enable_flash: + try: + from vllm.v1.pool.flash_maxsim import ( # noqa: F401 + flash_maxsim_rerank_direct, + ) + except ImportError: + enable_flash = False + self._flash_enabled = enable_flash def clear(self) -> None: self._query_cache.clear() @@ -116,7 +129,7 @@ class LateInteractionRunner: raise ValueError(f"Unsupported late-interaction mode: {mode!r}") if score_indices: - score_values = compute_maxsim_score_batched(score_queries, score_docs) + score_values = self._score(score_queries, score_docs) for i, req_id, query_key, score in zip( score_indices, score_req_ids, score_query_keys, score_values ): @@ -126,6 +139,58 @@ class LateInteractionRunner: return outputs + def _score( + self, + queries: list[torch.Tensor], + docs: list[torch.Tensor], + ) -> list[torch.Tensor]: + """Score (query_i, doc_i) pairs: the fused Triton kernel handles the + common rerank pattern (one shared query, CUDA inputs); multiple + distinct queries, CPU tensors, or a kernel failure fall back to the + reference scorer.""" + if self._flash_enabled and docs and queries[0].is_cuda: + first = queries[0] + if all(q is first for q in queries): + try: + return self._score_flash_shared_query(first, docs) + except Exception as exc: + # A persistent compile/launch failure must not take down + # requests: serve through the reference scorer and stop + # trying the kernel for the rest of the process. + self._flash_enabled = False + logger.warning( + "flash-maxsim scoring failed (%s); falling back to " + "the reference MaxSim path for this process.", + exc, + ) + return compute_maxsim_score_batched(queries, docs) + + @staticmethod + def _score_flash_shared_query( + query: torch.Tensor, + docs: list[torch.Tensor], + ) -> list[torch.Tensor]: + """One kernel launch for all docs against a single shared query: + docs pack into one [total_tokens, d] tensor (single cat, no padding) + and the kernel reads each by (offset, length) — the [B, Lq, Ld] + similarity tensor is never materialised.""" + from vllm.v1.pool.flash_maxsim import flash_maxsim_rerank_direct + + device = docs[0].device + lengths = [int(d.shape[0]) for d in docs] + offsets = [0] * len(lengths) + for i in range(1, len(lengths)): + offsets[i] = offsets[i - 1] + lengths[i - 1] + packed = docs[0] if len(docs) == 1 else torch.cat(docs, dim=0) + scores = flash_maxsim_rerank_direct( + query, + packed, + torch.tensor(offsets, device=device, dtype=torch.int32), + torch.tensor(lengths, device=device, dtype=torch.int32), + max(lengths), + ) + return list(scores.unbind(0)) + def _release_query_use(self, query_key: str) -> None: remaining = self._query_uses.get(query_key, 1) - 1 if remaining <= 0: diff --git a/vllm/v1/worker/gpu/pool/pooling_runner.py b/vllm/v1/worker/gpu/pool/pooling_runner.py index ce94764851..1a81046b17 100644 --- a/vllm/v1/worker/gpu/pool/pooling_runner.py +++ b/vllm/v1/worker/gpu/pool/pooling_runner.py @@ -51,7 +51,11 @@ class PoolingRunner: self.pooling_params: dict[int, PoolingParams] = {} self.pooling_states: dict[int, PoolingStates] = {} self.prompt_token_ids: dict[int, torch.Tensor] = {} - self.late_interaction_runner = LateInteractionRunner() + pooler_config = self.model_config.pooler_config + self.late_interaction_runner = LateInteractionRunner( + enable_flash=pooler_config is None + or getattr(pooler_config, "enable_flash_late_interaction", True) + ) @staticmethod def get_supported_tasks(model: nn.Module) -> list[PoolingTask]: From 47de9d4d049831914858404c48827837c2d915db Mon Sep 17 00:00:00 2001 From: Chanbin Lim <57864041+beenpow@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:10:54 -0700 Subject: [PATCH 210/248] [KV Connector][NIXL] Coalesce host-buffer KV copies across cache groups (#54483) Signed-off-by: beenpow <57864041+beenpow@users.noreply.github.com> Co-authored-by: Chendi.Xue --- .../kernels/benchmark_host_buffer_kv_copy.py | 256 ++++++++++++++++++ .../unit/test_nixl_connector_hma.py | 47 ++++ .../kv_connector/v1/nixl/base_worker.py | 41 +-- 3 files changed, 326 insertions(+), 18 deletions(-) create mode 100644 benchmarks/kernels/benchmark_host_buffer_kv_copy.py diff --git a/benchmarks/kernels/benchmark_host_buffer_kv_copy.py b/benchmarks/kernels/benchmark_host_buffer_kv_copy.py new file mode 100644 index 0000000000..be4051c0bd --- /dev/null +++ b/benchmarks/kernels/benchmark_host_buffer_kv_copy.py @@ -0,0 +1,256 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Benchmark the NIXL host-buffer KV copies across KV cache groups. + +`NixlBaseConnectorWorker.sync_recved_kv_to_device` / `save_kv_to_host` move a +request's blocks between the CPU transfer buffer and the device KV cache. Block +ids are unique across KV cache groups, so the per-group copies can be issued as +one. This measures what that coalescing saves: the copied bytes are identical, +only the number of launches changes. +""" + +import time + +import torch +from tabulate import tabulate + +from vllm.distributed.kv_transfer.kv_connector.utils import copy_kv_blocks +from vllm.logger import init_logger +from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE + +logger = init_logger(__name__) + + +def _make_caches( + num_layers: int, + num_blocks: int, + block_size: int, + num_heads: int, + head_size: int, + dtype: torch.dtype, + device: str, +) -> tuple[dict[str, torch.Tensor], dict[str, torch.Tensor]]: + """Host transfer buffers and device KV caches, shaped as the worker builds them.""" + shape = (num_blocks, block_size, num_heads, head_size) + host = { + f"layer.{i}": torch.randn(shape, dtype=dtype, device="cpu") + for i in range(num_layers) + } + device_caches = { + f"layer.{i}": torch.zeros(shape, dtype=dtype, device=device) + for i in range(num_layers) + } + return host, device_caches + + +def _split_into_groups(block_ids: list[int], num_groups: int) -> list[list[int]]: + """Partition block ids into groups, mirroring the global BlockPool id space.""" + per_group = len(block_ids) // num_groups + return [block_ids[i * per_group : (i + 1) * per_group] for i in range(num_groups)] + + +def _percentile(sorted_values: list[float], q: float) -> float: + return sorted_values[min(len(sorted_values) - 1, int(len(sorted_values) * q))] + + +@torch.inference_mode() +def _time_paired( + host: dict[str, torch.Tensor], + device_caches: dict[str, torch.Tensor], + group_block_ids: list[list[int]], + direction: str, + num_iters: int, +) -> tuple[list[float], list[float]]: + """Per-group and coalesced latencies, measured alternately. + + Interleaving the two variants keeps drift in machine state from landing on + one of them: each iteration times both, back to back. + """ + src, dst = (host, device_caches) if direction == "h2d" else (device_caches, host) + merged = [[b for group in group_block_ids for b in group]] + + def _run(batches: list[list[int]]) -> float: + start = time.perf_counter() + for ids in batches: + copy_kv_blocks(src, dst, ids, ids, direction) + torch.accelerator.synchronize() + return time.perf_counter() - start + + for _ in range(5): # warmup + _run(group_block_ids) + _run(merged) + + per_group: list[float] = [] + coalesced: list[float] = [] + for i in range(num_iters): + # Alternate which variant goes first so ordering cannot favour either. + if i % 2: + per_group.append(_run(group_block_ids)) + coalesced.append(_run(merged)) + else: + coalesced.append(_run(merged)) + per_group.append(_run(group_block_ids)) + return sorted(per_group), sorted(coalesced) + + +@torch.inference_mode() +def _count_device_ops( + host: dict[str, torch.Tensor], + device_caches: dict[str, torch.Tensor], + group_block_ids: list[list[int]], + direction: str, + coalesced: bool, +) -> dict[str, int]: + """Device-op counts for one request-sync, via the profiler. + + Counts do not depend on how busy the machine is, so this is the part of + the comparison that reproduces exactly. + """ + from torch.profiler import ProfilerActivity, profile + + src, dst = (host, device_caches) if direction == "h2d" else (device_caches, host) + batches = ( + [[b for group in group_block_ids for b in group]] + if coalesced + else group_block_ids + ) + + for ids in batches: # warm up allocator/autograd caches + copy_kv_blocks(src, dst, ids, ids, direction) + torch.accelerator.synchronize() + + with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA]) as prof: + for ids in batches: + copy_kv_blocks(src, dst, ids, ids, direction) + torch.accelerator.synchronize() + + counts: dict[str, int] = {} + for evt in prof.key_averages(): + if evt.count and (evt.self_device_time_total or evt.device_time_total): + counts[evt.key] = counts.get(evt.key, 0) + evt.count + return counts + + +def _report_op_counts(host, device_caches, block_ids, groups): + print("\nDevice-op counts per request-sync (profiler; independent of load):") + rows = [] + for num_groups in groups: + group_block_ids = _split_into_groups(block_ids, num_groups) + for direction in ("h2d", "d2h"): + before = _count_device_ops( + host, device_caches, group_block_ids, direction, coalesced=False + ) + after = _count_device_ops( + host, device_caches, group_block_ids, direction, coalesced=True + ) + for key in sorted(set(before) | set(after)): + b, a = before.get(key, 0), after.get(key, 0) + if b != a: + rows.append([num_groups, direction, key, b, a]) + print( + tabulate( + rows, + headers=["groups", "direction", "op", "per-group", "coalesced"], + ) + ) + + +def main(args): + dtype = STR_DTYPE_TO_TORCH_DTYPE[args.dtype] + torch.manual_seed(args.seed) + + host, device_caches = _make_caches( + num_layers=args.num_layers, + num_blocks=args.num_blocks, + block_size=args.block_size, + num_heads=args.num_heads, + head_size=args.head_size, + dtype=dtype, + device="cuda", + ) + block_ids = list(range(args.blocks_per_request)) + + rows = [] + for num_groups in args.groups: + group_block_ids = _split_into_groups(block_ids, num_groups) + for direction in ("h2d", "d2h"): + per_group, coalesced = _time_paired( + host, + device_caches, + group_block_ids, + direction, + num_iters=args.iters, + ) + # Interference only ever adds time, so the minimum is the most + # stable estimator here; the p10 spread above it says how noisy + # the machine was while sampling. + best_before = per_group[0] + best_after = coalesced[0] + spread = (_percentile(coalesced, 0.5) - best_after) / best_after + rows.append( + [ + num_groups, + direction, + best_before * 1e6, + best_after * 1e6, + (best_before - best_after) / best_before * 100.0, + spread * 100.0, + ] + ) + + print( + f"layers={args.num_layers} num_blocks={args.num_blocks} " + f"block_size={args.block_size} heads={args.num_heads} " + f"head_size={args.head_size} dtype={args.dtype} " + f"blocks/request={args.blocks_per_request} iters={args.iters}" + ) + print( + tabulate( + rows, + headers=[ + "groups", + "direction", + "per-group min (µs)", + "coalesced min (µs)", + "saved (%)", + "p50 over min (%)", + ], + floatfmt=".3f", + ) + ) + + if args.op_counts: + _report_op_counts(host, device_caches, block_ids, args.groups) + + +if __name__ == "__main__": + parser = FlexibleArgumentParser() + parser.add_argument("--num-layers", type=int, default=48) + parser.add_argument("--num-blocks", type=int, default=2048) + parser.add_argument("--block-size", type=int, choices=[16, 32], default=16) + parser.add_argument("--num-heads", type=int, default=8) + parser.add_argument("--head-size", type=int, default=128) + parser.add_argument("--blocks-per-request", type=int, default=60) + parser.add_argument( + "--groups", + type=int, + nargs="+", + default=[1, 2, 3, 6], + help="KV cache group counts to sweep (1 = non-hybrid model).", + ) + parser.add_argument( + "--dtype", + type=str, + choices=["half", "bfloat16", "float"], + default="bfloat16", + ) + parser.add_argument("--iters", type=int, default=100) + parser.add_argument( + "--op-counts", + action="store_true", + help="Also report profiler op counts, which do not vary with load.", + ) + parser.add_argument("--seed", type=int, default=0) + + main(parser.parse_args()) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 199ac2fe8c..5a9a10a57d 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -2360,3 +2360,50 @@ def test_push_write_hybrid_mla_replicates_attention(): assert spec.remote_block_ids == [[7, 8], [3]] assert call.kwargs["local_xfer_side_handle"] == local_handle assert call.kwargs["remote_xfer_side_handle"] == remote_handle + + +def _make_host_buffer_worker(copy_op): + """A worker stripped down to what the host-buffer copy paths touch.""" + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( + NixlConnectorWorker, + ) + + worker = object.__new__(NixlConnectorWorker) + worker.use_host_buffer = True + worker.copy_blocks = copy_op + return worker + + +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "group_block_ids,expected_ids", + [ + # Single group (non-hybrid model): one copy, as before. + ([[1, 2, 3]], [1, 2, 3]), + # Hybrid model: three groups collapse into a single copy. + ([[1, 2], [3, 4], [5]], [1, 2, 3, 4, 5]), + # An empty group contributes no ids. + ([[1, 2], []], [1, 2]), + ], +) +def test_sync_recved_kv_issues_one_copy_per_request(group_block_ids, expected_ids): + """h2d copies are issued once per request, not once per KV cache group.""" + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ReqMeta + + calls = [] + worker = _make_host_buffer_worker( + lambda src, dst, src_ids, dst_ids, direction: calls.append( + (src_ids, dst_ids, direction) + ) + ) + worker.host_xfer_buffers = {"layer": None} + worker.device_kv_caches = {"layer": None} + + meta = ReqMeta( + local_block_ids=group_block_ids, + local_physical_block_ids=group_block_ids, + tp_size=1, + ) + worker.sync_recved_kv_to_device("req", meta) + + assert calls == [(expected_ids, expected_ids, "h2d")] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 1e22180b3d..0eaa1aadf3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -2702,17 +2702,18 @@ class NixlBaseConnectorWorker: assert self.copy_blocks is not None local_block_ids = meta.local_physical_block_ids - # TODO (NickLucche) D2H<>H2D ops could benefit from coalescing io across groups - # The h2d block copies below are intentionally synchronous. + # Every KV cache group allocates from the same BlockPool, so block ids + # are unique across groups and the per-group copies can be issued as one. + block_ids = [block_id for group in local_block_ids for block_id in group] + # The h2d block copy below is intentionally synchronous. with gpu_sync_allowed(): - for group_block_ids in local_block_ids: - self.copy_blocks( - self.host_xfer_buffers, - self.device_kv_caches, - group_block_ids, - group_block_ids, - "h2d", - ) + self.copy_blocks( + self.host_xfer_buffers, + self.device_kv_caches, + block_ids, + block_ids, + "h2d", + ) if logger.isEnabledFor(logging.DEBUG): logger.debug( "synced recved kv of request[%s] to device kv buffer," @@ -2740,14 +2741,18 @@ class NixlBaseConnectorWorker: ",".join(map(str, meta.local_physical_block_ids)), ) # blocking - for group_block_ids in meta.local_physical_block_ids: - self.copy_blocks( - self.device_kv_caches, - self.host_xfer_buffers, - group_block_ids, - group_block_ids, - "d2h", - ) + block_ids = [ + block_id + for group in meta.local_physical_block_ids + for block_id in group + ] + self.copy_blocks( + self.device_kv_caches, + self.host_xfer_buffers, + block_ids, + block_ids, + "d2h", + ) @cached_property def _attention_kv_caches(self) -> list[torch.Tensor]: From d848c4ed3d0c576c3c24860547fa2b4ee980b37d Mon Sep 17 00:00:00 2001 From: Jaden Mathias Date: Thu, 1 Oct 2026 14:41:41 -0400 Subject: [PATCH 211/248] =?UTF-8?q?[ROCm][Triton]=20Migrate=20Kimi-K3=20ke?= =?UTF-8?q?rnels=20from=20make=5Fblock=5Fptr=20to=20tensor=20=E2=80=A6=20(?= =?UTF-8?q?#58769)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: JadenMathias --- .../kimi_k3/amd/ops/third_party/kda/chunk.py | 222 ++++------------- .../amd/ops/third_party/kda/chunk_intra.py | 224 +++++------------- .../kda/chunk_intra_token_parallel.py | 17 +- 3 files changed, 112 insertions(+), 351 deletions(-) diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py index e8b83d23ae..2e975e9f77 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk.py @@ -17,7 +17,11 @@ from vllm.third_party.flash_linear_attention.ops.chunk_delta_h import ( from vllm.third_party.flash_linear_attention.ops.cumsum import chunk_local_cumsum from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices from vllm.third_party.flash_linear_attention.ops.l2norm import l2norm_fwd -from vllm.third_party.flash_linear_attention.ops.op import exp2, log +from vllm.third_party.flash_linear_attention.ops.op import ( + exp2, + log, + make_tensor_descriptor, +) from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is_amd from vllm.triton_utils import tl, triton from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2 @@ -90,86 +94,35 @@ def recompute_w_u_fwd_kernel( T = eos - bos else: bos, eos = i_b * T, i_b * T + T - p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) - b_b = tl.load(p_b, boundary_check=(0,)).to(tl.float32) + o_b = i_t * BT + tl.arange(0, BT) + b_b = tl.load(beta + (bos + o_b) * H + i_h, mask=o_b < T, other=0).to(tl.float32) - p_A = tl.make_block_ptr( - A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) - ) - b_A = tl.load(p_A, boundary_check=(0, 1)) + desc_A = make_tensor_descriptor(A + (bos * H + i_h) * BT, [T, BT], [H * BT, 1], [BT, BT]) + b_A = desc_A.load([i_t * BT, 0]) + desc_v = make_tensor_descriptor(v + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) + desc_u = make_tensor_descriptor(u + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) for i_v in range(tl.cdiv(V, BV)): - p_v = tl.make_block_ptr( - v + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - p_u = tl.make_block_ptr( - u + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - b_v = tl.load(p_v, boundary_check=(0, 1)) + b_v = desc_v.load([i_t * BT, i_v * BV]) b_vb = (b_v * b_b[:, None]).to(b_v.dtype) b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION) - tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) + desc_u.store([i_t * BT, i_v * BV], b_u.to(desc_u.dtype)) + desc_w = make_tensor_descriptor(w + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_k = make_tensor_descriptor(k + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_gk = make_tensor_descriptor(gk + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) for i_k in range(tl.cdiv(K, BK)): - p_w = tl.make_block_ptr( - w + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_k = tl.make_block_ptr( - k + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - b_k = tl.load(p_k, boundary_check=(0, 1)) + b_k = desc_k.load([i_t * BT, i_k * BK]) b_kb = b_k * b_b[:, None] - p_gk = tl.make_block_ptr( - gk + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - b_gk = tl.load(p_gk, boundary_check=(0, 1)) + b_gk = desc_gk.load([i_t * BT, i_k * BK]) b_kb *= exp2(b_gk) if STORE_QG: - p_q = tl.make_block_ptr( - q + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_qg = tl.make_block_ptr( - qg + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - b_q = tl.load(p_q, boundary_check=(0, 1)) + desc_q = make_tensor_descriptor(q + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_qg = make_tensor_descriptor(qg + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + b_q = desc_q.load([i_t * BT, i_k * BK]) b_qg = b_q * exp2(b_gk) - tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1)) + desc_qg.store([i_t * BT, i_k * BK], b_qg.to(desc_qg.dtype)) if STORE_KG: last_idx = min(i_t * BT + BT, T) - 1 @@ -180,18 +133,11 @@ def recompute_w_u_fwd_kernel( ) b_kg = b_k * exp2(b_gn - b_gk) - p_kg = tl.make_block_ptr( - kg + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1)) + desc_kg = make_tensor_descriptor(kg + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_kg.store([i_t * BT, i_k * BK], b_kg.to(desc_kg.dtype)) b_w = tl.dot(b_A, b_kb.to(b_k.dtype)) - tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) + desc_w.store([i_t * BT, i_k * BK], b_w.to(desc_w.dtype)) def recompute_w_u_fwd( @@ -294,70 +240,32 @@ def chunk_gla_fwd_kernel_o( m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :] b_o = tl.zeros([BT, BV], dtype=tl.float32) + desc_q = make_tensor_descriptor(q + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_g = make_tensor_descriptor(g + (bos * H + i_h) * K, [T, K], [H * K, 1], [BT, BK]) + desc_h = make_tensor_descriptor(h + (i_tg * H + i_h) * K * V, [V, K], [K, 1], [BV, BK]) for i_k in range(tl.cdiv(K, BK)): - p_q = tl.make_block_ptr( - q + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_g = tl.make_block_ptr( - g + (bos * H + i_h) * K, - (T, K), - (H * K, 1), - (i_t * BT, i_k * BK), - (BT, BK), - (1, 0), - ) - p_h = tl.make_block_ptr( - h + (i_tg * H + i_h) * K * V, - (V, K), - (K, 1), - (i_v * BV, i_k * BK), - (BV, BK), - (1, 0), - ) - # [BT, BK] - b_q = tl.load(p_q, boundary_check=(0, 1)) + b_q = desc_q.load([i_t * BT, i_k * BK]) b_q = (b_q * scale).to(b_q.dtype) # [BT, BK] - b_g = tl.load(p_g, boundary_check=(0, 1)) + b_g = desc_g.load([i_t * BT, i_k * BK]) # [BT, BK] b_qg = (b_q * exp2(b_g)).to(b_q.dtype) # [BV, BK] - b_h = tl.load(p_h, boundary_check=(0, 1)) + b_h = desc_h.load([i_v * BV, i_k * BK]) # [BT, BV] if i_k >= 0: b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype)) - p_v = tl.make_block_ptr( - v + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - p_o = tl.make_block_ptr( - o + (bos * H + i_h) * V, - (T, V), - (H * V, 1), - (i_t * BT, i_v * BV), - (BT, BV), - (1, 0), - ) - p_A = tl.make_block_ptr( - A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) - ) + desc_v = make_tensor_descriptor(v + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) + desc_o = make_tensor_descriptor(o + (bos * H + i_h) * V, [T, V], [H * V, 1], [BT, BV]) + desc_A = make_tensor_descriptor(A + (bos * H + i_h) * BT, [T, BT], [H * BT, 1], [BT, BT]) # [BT, BV] - b_v = tl.load(p_v, boundary_check=(0, 1)) + b_v = desc_v.load([i_t * BT, i_v * BV]) # [BT, BT] - b_A = tl.load(p_A, boundary_check=(0, 1)) + b_A = desc_A.load([i_t * BT, 0]) b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype) b_o += tl.dot(b_A, b_v, allow_tf32=False) - tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + desc_o.store([i_t * BT, i_v * BV], b_o.to(desc_o.dtype)) def chunk_gla_fwd_o_gk( @@ -479,34 +387,13 @@ def kda_gate_chunk_cumsum_vector_kernel( i_s -= 1 - p_s = tl.make_block_ptr( - s + (bos * H + i_h) * S, - (T, S), - (H * S, 1), - (i_t * BT, i_s * BS), - (BT, BS), - (1, 0), - ) - p_o = tl.make_block_ptr( - o + (bos * H + i_h) * S, - (T, S), - (H * S, 1), - (i_t * BT, i_s * BS), - (BT, BS), - (1, 0), - ) + desc_s = make_tensor_descriptor(s + (bos * H + i_h) * S, [T, S], [H * S, 1], [BT, BS]) + desc_o = make_tensor_descriptor(o + (bos * H + i_h) * S, [T, S], [H * S, 1], [BT, BS]) - b_s = tl.load(p_s, boundary_check=(0, 1)).to(tl.float32) + b_s = desc_s.load([i_t * BT, i_s * BS]).to(tl.float32) if HAS_BIAS: - p_bias = tl.make_block_ptr( - g_bias + i_h * S, - (S,), - (1,), - (i_s * BS,), - (BS,), - (0,), - ) - b_bias = tl.load(p_bias, boundary_check=(0,)).to(tl.float32) + desc_bias = make_tensor_descriptor(g_bias + i_h * S, [S], [1], [BS]) + b_bias = desc_bias.load([i_s * BS]).to(tl.float32) b_s += b_bias[None, :] b_a = tl.exp(tl.load(A_log + i_h).to(tl.float32)) @@ -524,7 +411,7 @@ def kda_gate_chunk_cumsum_vector_kernel( # Boundary loads return zero, but bias and gate activation can make padded # rows nonzero. Padding trails valid rows, so it only affects masked stores. b_o = tl.cumsum(b_gate, axis=0) * cumsum_scale - tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + desc_o.store([i_t * BT, i_s * BS], b_o.to(desc_o.dtype)) def fused_kda_gate_chunk_cumsum( @@ -862,25 +749,10 @@ def kda_gate_fwd_kernel( stride_row = H * D stride_col = 1 - g_ptr = tl.make_block_ptr( - base=g + i_h * D, - shape=(T, D), - strides=(stride_row, stride_col), - offsets=(n_t, 0), - block_shape=(BT, BD), - order=(1, 0), - ) + desc_g = make_tensor_descriptor(g + i_h * D, [T, D], [stride_row, stride_col], [BT, BD]) + desc_y = make_tensor_descriptor(y + i_h * D, [T, D], [stride_row, stride_col], [BT, BD]) - y_ptr = tl.make_block_ptr( - base=y + i_h * D, - shape=(T, D), - strides=(stride_row, stride_col), - offsets=(n_t, 0), - block_shape=(BT, BD), - order=(1, 0), - ) - - b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32) + b_g = desc_g.load([n_t, 0]).to(tl.float32) if HAS_BIAS: n_d = tl.arange(0, BD) @@ -898,7 +770,7 @@ def kda_gate_fwd_kernel( sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled))) b_y = -b_a * sp - tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1)) + desc_y.store([n_t, 0], b_y.to(desc_y.dtype)) def fused_kda_gate( diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py index 8b9aa86f9e..0c600f3ba3 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py @@ -12,7 +12,11 @@ import torch from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.index import prepare_chunk_indices -from vllm.third_party.flash_linear_attention.ops.op import exp2, gather +from vllm.third_party.flash_linear_attention.ops.op import ( + exp2, + gather, + make_tensor_descriptor, +) from vllm.third_party.flash_linear_attention.ops.utils import is_gather_supported from vllm.triton_utils import tl, triton @@ -125,33 +129,21 @@ def chunk_kda_fwd_kernel_inter_solve_fused( ################################################################################ # off-diagonal blocks ################################################################################ + desc_qk = make_tensor_descriptor(q, [T, K], [H * K, 1], [BC, BK]) + desc_kk = make_tensor_descriptor(k, [T, K], [H * K, 1], [BC, BK]) + desc_gk = make_tensor_descriptor(g, [T, K], [HV * K, 1], [BC, BK]) for i_k in range(tl.cdiv(K, BK)): o_k = i_k * BK + tl.arange(0, BK) m_k = o_k < K - p_k0 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0) - ) - p_g0 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc0, i_k * BK), (BC, BK), (1, 0) - ) - b_k0 = tl.load(p_k0, boundary_check=(0, 1)).to(tl.float32) - b_g0 = tl.load(p_g0, boundary_check=(0, 1)).to(tl.float32) + b_k0 = desc_kk.load([i_tc0, i_k * BK]).to(tl.float32) + b_g0 = desc_gk.load([i_tc0, i_k * BK]).to(tl.float32) if i_tc1 < T: - p_q1 = tl.make_block_ptr( - q, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0) - ) - p_k1 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0) - ) - p_g1 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc1, i_k * BK), (BC, BK), (1, 0) - ) # [BC, BK] - b_q1 = tl.load(p_q1, boundary_check=(0, 1)).to(tl.float32) - b_k1 = tl.load(p_k1, boundary_check=(0, 1)).to(tl.float32) - b_g1 = tl.load(p_g1, boundary_check=(0, 1)).to(tl.float32) + b_q1 = desc_qk.load([i_tc1, i_k * BK]).to(tl.float32) + b_k1 = desc_kk.load([i_tc1, i_k * BK]).to(tl.float32) + b_g1 = desc_gk.load([i_tc1, i_k * BK]).to(tl.float32) # [BK] b_gn1 = tl.load(g + i_tc1 * HV * K + o_k, mask=m_k, other=0).to(tl.float32) # [BC, BK] @@ -163,19 +155,10 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk10 += tl.dot(b_k1 * b_gqn, b_kgt) if i_tc2 < T: - p_q2 = tl.make_block_ptr( - q, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0) - ) - p_k2 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0) - ) - p_g2 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc2, i_k * BK), (BC, BK), (1, 0) - ) # [BC, BK] - b_q2 = tl.load(p_q2, boundary_check=(0, 1)).to(tl.float32) - b_k2 = tl.load(p_k2, boundary_check=(0, 1)).to(tl.float32) - b_g2 = tl.load(p_g2, boundary_check=(0, 1)).to(tl.float32) + b_q2 = desc_qk.load([i_tc2, i_k * BK]).to(tl.float32) + b_k2 = desc_kk.load([i_tc2, i_k * BK]).to(tl.float32) + b_g2 = desc_gk.load([i_tc2, i_k * BK]).to(tl.float32) # [BK] b_gn2 = tl.load(g + i_tc2 * HV * K + o_k, mask=m_k, other=0).to( tl.float32 @@ -195,19 +178,10 @@ def chunk_kda_fwd_kernel_inter_solve_fused( b_Akk21 += tl.dot(b_kg2, b_kgt) if i_tc3 < T: - p_q3 = tl.make_block_ptr( - q, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0) - ) - p_k3 = tl.make_block_ptr( - k, (T, K), (H * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0) - ) - p_g3 = tl.make_block_ptr( - g, (T, K), (HV * K, 1), (i_tc3, i_k * BK), (BC, BK), (1, 0) - ) # [BC, BK] - b_q3 = tl.load(p_q3, boundary_check=(0, 1)).to(tl.float32) - b_k3 = tl.load(p_k3, boundary_check=(0, 1)).to(tl.float32) - b_g3 = tl.load(p_g3, boundary_check=(0, 1)).to(tl.float32) + b_q3 = desc_qk.load([i_tc3, i_k * BK]).to(tl.float32) + b_k3 = desc_kk.load([i_tc3, i_k * BK]).to(tl.float32) + b_g3 = desc_gk.load([i_tc3, i_k * BK]).to(tl.float32) # [BK] b_gn3 = tl.load(g + i_tc3 * HV * K + o_k, mask=m_k, other=0).to( tl.float32 @@ -235,83 +209,36 @@ def chunk_kda_fwd_kernel_inter_solve_fused( ################################################################################ # save off-diagonal Aqk blocks and prepare Akk ################################################################################ + desc_Aqk = make_tensor_descriptor(Aqk, [T, BT], [HV * BT, 1], [BC, BC]) + p_beta = beta + bos * HV + i_hv + o_bc = tl.arange(0, BC) if i_tc1 < T: - p_Aqk10 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0) - ) - tl.store( - p_Aqk10, (b_Aqk10 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) + desc_Aqk.store([i_tc1, 0], (b_Aqk10 * scale).to(desc_Aqk.dtype)) - p_b1 = tl.make_block_ptr( - beta + bos * HV + i_hv, (T,), (HV,), (i_tc1,), (BC,), (0,) - ) - b_b1 = tl.load(p_b1, boundary_check=(0,)).to(tl.float32) + b_b1 = tl.load(p_beta + (i_tc1 + o_bc) * HV, mask=i_tc1 + o_bc < T, other=0).to(tl.float32) b_Akk10 = b_Akk10 * b_b1[:, None] if i_tc2 < T: - p_Aqk20 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0) - ) - p_Aqk21 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0) - ) - tl.store( - p_Aqk20, (b_Aqk20 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) - tl.store( - p_Aqk21, (b_Aqk21 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) + desc_Aqk.store([i_tc2, 0], (b_Aqk20 * scale).to(desc_Aqk.dtype)) + desc_Aqk.store([i_tc2, BC], (b_Aqk21 * scale).to(desc_Aqk.dtype)) - p_b2 = tl.make_block_ptr( - beta + bos * HV + i_hv, (T,), (HV,), (i_tc2,), (BC,), (0,) - ) - b_b2 = tl.load(p_b2, boundary_check=(0,)).to(tl.float32) + b_b2 = tl.load(p_beta + (i_tc2 + o_bc) * HV, mask=i_tc2 + o_bc < T, other=0).to(tl.float32) b_Akk20 = b_Akk20 * b_b2[:, None] b_Akk21 = b_Akk21 * b_b2[:, None] if i_tc3 < T: - p_Aqk30 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0) - ) - p_Aqk31 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0) - ) - p_Aqk32 = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0) - ) - tl.store( - p_Aqk30, (b_Aqk30 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) - tl.store( - p_Aqk31, (b_Aqk31 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) - tl.store( - p_Aqk32, (b_Aqk32 * scale).to(Aqk.dtype.element_ty), boundary_check=(0, 1) - ) + desc_Aqk.store([i_tc3, 0], (b_Aqk30 * scale).to(desc_Aqk.dtype)) + desc_Aqk.store([i_tc3, BC], (b_Aqk31 * scale).to(desc_Aqk.dtype)) + desc_Aqk.store([i_tc3, 2 * BC], (b_Aqk32 * scale).to(desc_Aqk.dtype)) - p_b3 = tl.make_block_ptr( - beta + bos * HV + i_hv, (T,), (HV,), (i_tc3,), (BC,), (0,) - ) - b_b3 = tl.load(p_b3, boundary_check=(0,)).to(tl.float32) + b_b3 = tl.load(p_beta + (i_tc3 + o_bc) * HV, mask=i_tc3 + o_bc < T, other=0).to(tl.float32) b_Akk30 = b_Akk30 * b_b3[:, None] b_Akk31 = b_Akk31 * b_b3[:, None] b_Akk32 = b_Akk32 * b_b3[:, None] - p_Akk00 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc0, 0), (BC, BC), (1, 0) - ) - p_Akk11 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc1, 0), (BC, BC), (1, 0) - ) - p_Akk22 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc2, 0), (BC, BC), (1, 0) - ) - p_Akk33 = tl.make_block_ptr( - Akkd, (T, BC), (HV * BC, 1), (i_tc3, 0), (BC, BC), (1, 0) - ) - b_Ai00 = tl.load(p_Akk00, boundary_check=(0, 1)).to(tl.float32) - b_Ai11 = tl.load(p_Akk11, boundary_check=(0, 1)).to(tl.float32) - b_Ai22 = tl.load(p_Akk22, boundary_check=(0, 1)).to(tl.float32) - b_Ai33 = tl.load(p_Akk33, boundary_check=(0, 1)).to(tl.float32) + desc_Akkd = make_tensor_descriptor(Akkd, [T, BC], [HV * BC, 1], [BC, BC]) + b_Ai00 = desc_Akkd.load([i_tc0, 0]).to(tl.float32) + b_Ai11 = desc_Akkd.load([i_tc1, 0]).to(tl.float32) + b_Ai22 = desc_Akkd.load([i_tc2, 0]).to(tl.float32) + b_Ai33 = desc_Akkd.load([i_tc3, 0]).to(tl.float32) ################################################################################ # forward substitution on diagonals @@ -397,47 +324,18 @@ def chunk_kda_fwd_kernel_inter_solve_fused( # store full Akk_inv to Akk ################################################################################ - p_Akk00 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc0, 0), (BC, BC), (1, 0) - ) - p_Akk10 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc1, 0), (BC, BC), (1, 0) - ) - p_Akk11 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc1, BC), (BC, BC), (1, 0) - ) - p_Akk20 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc2, 0), (BC, BC), (1, 0) - ) - p_Akk21 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc2, BC), (BC, BC), (1, 0) - ) - p_Akk22 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc2, 2 * BC), (BC, BC), (1, 0) - ) - p_Akk30 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, 0), (BC, BC), (1, 0) - ) - p_Akk31 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, BC), (BC, BC), (1, 0) - ) - p_Akk32 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, 2 * BC), (BC, BC), (1, 0) - ) - p_Akk33 = tl.make_block_ptr( - Akk, (T, BT), (HV * BT, 1), (i_tc3, 3 * BC), (BC, BC), (1, 0) - ) + desc_Akk = make_tensor_descriptor(Akk, [T, BT], [HV * BT, 1], [BC, BC]) - tl.store(p_Akk00, b_Ai00.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk10, b_Ai10.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk11, b_Ai11.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk20, b_Ai20.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk21, b_Ai21.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk22, b_Ai22.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk30, b_Ai30.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk31, b_Ai31.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk32, b_Ai32.to(Akk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk33, b_Ai33.to(Akk.dtype.element_ty), boundary_check=(0, 1)) + desc_Akk.store([i_tc0, 0], b_Ai00.to(desc_Akk.dtype)) + desc_Akk.store([i_tc1, 0], b_Ai10.to(desc_Akk.dtype)) + desc_Akk.store([i_tc1, BC], b_Ai11.to(desc_Akk.dtype)) + desc_Akk.store([i_tc2, 0], b_Ai20.to(desc_Akk.dtype)) + desc_Akk.store([i_tc2, BC], b_Ai21.to(desc_Akk.dtype)) + desc_Akk.store([i_tc2, 2 * BC], b_Ai22.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, 0], b_Ai30.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, BC], b_Ai31.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, 2 * BC], b_Ai32.to(desc_Akk.dtype)) + desc_Akk.store([i_tc3, 3 * BC], b_Ai33.to(desc_Akk.dtype)) @triton.heuristics( @@ -505,16 +403,14 @@ def chunk_kda_fwd_kernel_intra_sub_chunk( Aqk = Aqk + (bos * HV + i_hv) * BT Akk = Akk + (bos * HV + i_hv) * BC - p_q = tl.make_block_ptr(q, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0)) - p_k = tl.make_block_ptr(k, (T, K), (H * K, 1), (i_ti, 0), (BC, BK), (1, 0)) - p_g = tl.make_block_ptr(g, (T, K), (HV * K, 1), (i_ti, 0), (BC, BK), (1, 0)) + desc_q = make_tensor_descriptor(q, [T, K], [H * K, 1], [BC, BK]) + desc_k = make_tensor_descriptor(k, [T, K], [H * K, 1], [BC, BK]) + desc_g = make_tensor_descriptor(g, [T, K], [HV * K, 1], [BC, BK]) - p_beta = tl.make_block_ptr(beta, (T,), (HV,), (i_ti,), (BC,), (0,)) - - b_q = tl.load(p_q, boundary_check=(0, 1)) - b_k = tl.load(p_k, boundary_check=(0, 1)) - b_g = tl.load(p_g, boundary_check=(0, 1)) - b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32) + b_q = desc_q.load([i_ti, 0]) + b_k = desc_k.load([i_ti, 0]) + b_g = desc_g.load([i_ti, 0]) + b_beta = tl.load(beta + o_c * HV, mask=m_c, other=0).to(tl.float32) if USE_GATHER: b_gn = gather( @@ -546,12 +442,10 @@ def chunk_kda_fwd_kernel_intra_sub_chunk( b_Aqk = tl.where(m_Aqk, b_Aqk, 0.0) b_Akk = tl.where(m_Akk, b_Akk, 0.0) - p_Aqk = tl.make_block_ptr( - Aqk, (T, BT), (HV * BT, 1), (i_ti, i_i * BC), (BC, BC), (1, 0) - ) - p_Akk = tl.make_block_ptr(Akk, (T, BC), (HV * BC, 1), (i_ti, 0), (BC, BC), (1, 0)) - tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1)) - tl.store(p_Akk, b_Akk.to(Akk.dtype.element_ty), boundary_check=(0, 1)) + desc_Aqk = make_tensor_descriptor(Aqk, [T, BT], [HV * BT, 1], [BC, BC]) + desc_Akk = make_tensor_descriptor(Akk, [T, BC], [HV * BC, 1], [BC, BC]) + desc_Aqk.store([i_ti, i_i * BC], b_Aqk.to(desc_Aqk.dtype)) + desc_Akk.store([i_ti, 0], b_Akk.to(desc_Akk.dtype)) tl.debug_barrier() @@ -566,7 +460,7 @@ def chunk_kda_fwd_kernel_intra_sub_chunk( b_a += tl.sum(b_a[:, None] * b_Ai, 0) b_Ai = tl.where((o_i == i)[:, None], b_a, b_Ai) b_Ai += m_I - tl.store(p_Akk, b_Ai.to(Akk.dtype.element_ty), boundary_check=(0, 1)) + desc_Akk.store([i_ti, 0], b_Ai.to(desc_Akk.dtype)) def chunk_kda_fwd_intra( diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py index 86654f2644..7740e64d5d 100644 --- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py +++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra_token_parallel.py @@ -12,7 +12,7 @@ import torch -from vllm.third_party.flash_linear_attention.ops.op import exp2 +from vllm.third_party.flash_linear_attention.ops.op import exp2, make_tensor_descriptor from vllm.triton_utils import tl, triton @@ -108,20 +108,15 @@ def chunk_kda_fwd_kernel_intra_token_parallel( b_k = tl.load(k + i_t * H * K + p_qk, mask=m_hk, other=0).to(tl.float32) # g: [B, T, HV, K], beta: [B, T, HV] - p_g = tl.make_block_ptr( - g + i_t * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0) - ) - p_beta = tl.make_block_ptr(beta + i_t * HV, (HV,), (1,), (i_hg * BH,), (BH,), (0,)) - b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) - b_beta = tl.load(p_beta, boundary_check=(0,)).to(tl.float32) + desc_g = make_tensor_descriptor(g + i_t * HV * K, [HV, K], [K, 1], [BH, BK]) + b_g = desc_g.load([i_hg * BH, 0]).to(tl.float32) + b_beta = tl.load(beta + i_t * HV + o_hv, mask=m_hv, other=0).to(tl.float32) b_k *= b_beta[:, None] for j in range(i_ts, min(i_t + 1, min(T, i_ts + BC))): b_kj = tl.load(k + j * H * K + p_qk, mask=m_hk, other=0).to(tl.float32) - p_gj = tl.make_block_ptr( - g + j * HV * K, (HV, K), (K, 1), (i_hg * BH, 0), (BH, BK), (1, 0) - ) - b_gj = tl.load(p_gj, boundary_check=(0, 1)).to(tl.float32) + desc_gj = make_tensor_descriptor(g + j * HV * K, [HV, K], [K, 1], [BH, BK]) + b_gj = desc_gj.load([i_hg * BH, 0]).to(tl.float32) b_kgj = tl.where(m_k[None, :], b_kj * exp2(b_g - b_gj), 0.0) b_Aqk = tl.sum(b_q * b_kgj, axis=1) * scale From 2a52715ef7ca533a0ce287e14dc1c0702bdc2e8d Mon Sep 17 00:00:00 2001 From: ai-jz <156989844+ai-jz@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:20:57 -0700 Subject: [PATCH 212/248] [Bugfix][Responses] Use standard reasoning content-part events (#59652) Signed-off-by: ai-jz Co-authored-by: ai-jz Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../entrypoints/openai/responses/conftest.py | 8 +-- .../openai/responses/test_function_call.py | 9 ++-- .../openai/responses/test_harmony.py | 5 +- .../openai/responses/test_streaming_events.py | 41 +++++++++++++++- vllm/entrypoints/openai/responses/protocol.py | 49 ------------------- .../openai/responses/streaming_events.py | 32 +++++------- 6 files changed, 59 insertions(+), 85 deletions(-) diff --git a/tests/entrypoints/openai/responses/conftest.py b/tests/entrypoints/openai/responses/conftest.py index 7ee4629df3..ce17012cdf 100644 --- a/tests/entrypoints/openai/responses/conftest.py +++ b/tests/entrypoints/openai/responses/conftest.py @@ -37,7 +37,6 @@ def pairs_of_event_types() -> dict[str, str]: "response.content_part.done": "response.content_part.added", "response.output_text.done": "response.output_text.delta", "response.reasoning_text.done": "response.reasoning_text.delta", - "response.reasoning_part.done": "response.reasoning_part.added", "response.mcp_call_arguments.done": "response.mcp_call_arguments.delta", "response.mcp_call.completed": "response.mcp_call.in_progress", "response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa: E501 @@ -250,11 +249,8 @@ def _validate_field_consistency(events: list) -> None: active_content_index = None continue - # --- content_part / reasoning_part added: sets content_index - if etype in ( - "response.content_part.added", - "response.reasoning_part.added", - ): + # --- content_part added: sets content_index + if etype == "response.content_part.added": _assert_item_fields(event, etype, active_item_id, active_output_index) content_index = getattr(event, "content_index", None) if active_content_index is None: diff --git a/tests/entrypoints/openai/responses/test_function_call.py b/tests/entrypoints/openai/responses/test_function_call.py index 006ee0f6f2..8fd26c350a 100644 --- a/tests/entrypoints/openai/responses/test_function_call.py +++ b/tests/entrypoints/openai/responses/test_function_call.py @@ -340,12 +340,12 @@ async def test_function_calling_with_streaming_types( # response.created # -> response.in_progress # -> response.output_item.added - # -> response.reasoning_part.added + # -> response.content_part.added # -> response.reasoning_text.delta # .... # -> response.reasoning_text.delta # -> response.reasoning_text.done - # -> response.reasoning_part.done + # -> response.content_part.done # -> response.output_item.done # -> response.output_item.added # -> response.content_part.added @@ -368,12 +368,12 @@ async def test_function_calling_with_streaming_types( # response.created # -> response.in_progress # -> response.output_item.added - # -> response.reasoning_part.added + # -> response.content_part.added # -> response.reasoning_text.delta # .... # -> response.reasoning_text.delta # -> response.reasoning_text.done - # -> response.reasoning_part.done + # -> response.content_part.done # -> response.output_item.done # -> response.output_item.added # -> response.content_part.added @@ -424,7 +424,6 @@ async def test_function_calling_with_streaming_types( "response.output_text.done": "response.output_text.delta", "response.content_part.done": "response.content_part.added", "response.reasoning_text.done": "response.reasoning_text.delta", - "response.reasoning_part.done": "response.reasoning_part.added", "response.function_call_arguments.done": "response.function_call_arguments.delta", # noqa } diff --git a/tests/entrypoints/openai/responses/test_harmony.py b/tests/entrypoints/openai/responses/test_harmony.py index 574f33a8f2..1732382c35 100644 --- a/tests/entrypoints/openai/responses/test_harmony.py +++ b/tests/entrypoints/openai/responses/test_harmony.py @@ -467,10 +467,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): assert event.item_id == current_item_id # Verify content indices - if event.type in [ - "response.content_part.added", - "response.reasoning_part.added", - ]: + if event.type == "response.content_part.added": assert event.content_index == current_content_index + 1 current_content_index = event.content_index elif event.type in [ diff --git a/tests/entrypoints/openai/responses/test_streaming_events.py b/tests/entrypoints/openai/responses/test_streaming_events.py index e1a8bf46f6..21fe5537bb 100644 --- a/tests/entrypoints/openai/responses/test_streaming_events.py +++ b/tests/entrypoints/openai/responses/test_streaming_events.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from openai.types.responses import ResponseFunctionWebSearch +import pytest +from openai.types.responses import ResponseFunctionWebSearch, ResponseStreamEvent from openai_harmony import Message, Role +from pydantic import TypeAdapter from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, @@ -14,6 +16,8 @@ from vllm.entrypoints.openai.responses.streaming_events import ( StreamingState, _StateType, emit_browser_tool_events, + emit_reasoning_delta_events, + emit_reasoning_done_events, split_delta, ) @@ -40,6 +44,41 @@ def test_browser_find_uses_responses_action_type(): assert event.item.action.url == "cursor:42" +@pytest.mark.parametrize("harmony", [False, True], ids=["simple", "harmony"]) +def test_reasoning_content_parts_use_sdk_events(harmony): + text = "Let me think." + if harmony: + state = StreamingState() + events = emit_reasoning_delta_events(text, state) + events.extend(emit_reasoning_done_events(text, state)) + else: + processor = SimpleStreamingEventProcessor() + events = _run_through_processor(processor, DeltaMessage(reasoning=text)) + events.extend(processor.close_current()) + + assert [event.type for event in events] == [ + "response.output_item.added", + "response.content_part.added", + "response.reasoning_text.delta", + "response.reasoning_text.done", + "response.content_part.done", + "response.output_item.done", + ] + added, part_added, delta, text_done, part_done, done = events + assert part_added.part.type == part_done.part.type == "reasoning_text" + assert part_added.part.text == "" + assert delta.delta == text_done.text == part_done.part.text == text + assert done.item.content[0].text == text + assert done.item.id == added.item.id + for event in events[1:-1]: + assert event.item_id == added.item.id + assert event.content_index == part_added.content_index + validator = TypeAdapter(ResponseStreamEvent) + for event in events: + assert event.output_index == added.output_index + validator.validate_python(event.model_dump()) + + def _make_tool_call( index: int, name: str | None = None, arguments: str | None = None ) -> DeltaToolCall: diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 7e286aa782..9c44a04b3a 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -42,9 +42,6 @@ from openai.types.responses import ( ResponseInProgressEvent as OpenAIResponseInProgressEvent, ) from openai.types.responses.response import IncompleteDetails, ToolChoice -from openai.types.responses.response_reasoning_item import ( - Content as ResponseReasoningTextContent, -) from openai.types.responses.tool import Tool from openai.types.shared import Metadata, Reasoning from openai_harmony import Message as OpenAIHarmonyMessage @@ -845,50 +842,6 @@ class ResponsesResponse(OpenAIBaseModel): ) -# TODO: this code can be removed once -# https://github.com/openai/openai-python/issues/2634 has been resolved -class ResponseReasoningPartDoneEvent(OpenAIBaseModel): - content_index: int - """The index of the content part that is done.""" - - item_id: str - """The ID of the output item that the content part was added to.""" - - output_index: int - """The index of the output item that the content part was added to.""" - - part: ResponseReasoningTextContent - """The content part that is done.""" - - sequence_number: int - """The sequence number of this event.""" - - type: Literal["response.reasoning_part.done"] - """The type of the event. Always `response.reasoning_part.done`.""" - - -# TODO: this code can be removed once -# https://github.com/openai/openai-python/issues/2634 has been resolved -class ResponseReasoningPartAddedEvent(OpenAIBaseModel): - content_index: int - """The index of the content part that is done.""" - - item_id: str - """The ID of the output item that the content part was added to.""" - - output_index: int - """The index of the output item that the content part was added to.""" - - part: ResponseReasoningTextContent - """The content part that is done.""" - - sequence_number: int - """The sequence number of this event.""" - - type: Literal["response.reasoning_part.added"] - """The type of the event. Always `response.reasoning_part.added`.""" - - # vLLM Streaming Events # Note: we override the response type with the vLLM ResponsesResponse type class ResponseCompletedEvent(OpenAIResponseCompletedEvent): @@ -913,8 +866,6 @@ StreamingResponsesResponse: TypeAlias = ( | ResponseContentPartDoneEvent | ResponseReasoningTextDeltaEvent | ResponseReasoningTextDoneEvent - | ResponseReasoningPartAddedEvent - | ResponseReasoningPartDoneEvent | ResponseCodeInterpreterCallInProgressEvent | ResponseCodeInterpreterCallCodeDeltaEvent | ResponseWebSearchCallInProgressEvent diff --git a/vllm/entrypoints/openai/responses/streaming_events.py b/vllm/entrypoints/openai/responses/streaming_events.py index ae9fc2b25e..b4731dc687 100644 --- a/vllm/entrypoints/openai/responses/streaming_events.py +++ b/vllm/entrypoints/openai/responses/streaming_events.py @@ -70,8 +70,6 @@ from vllm.entrypoints.openai.parser.harmony_utils import ( is_function_recipient, ) from vllm.entrypoints.openai.responses.protocol import ( - ResponseReasoningPartAddedEvent, - ResponseReasoningPartDoneEvent, StreamingResponsesResponse, ) from vllm.entrypoints.openai.responses.utils import ( @@ -220,16 +218,13 @@ def emit_reasoning_delta_events( ) state.current_content_index += 1 events.append( - ResponseReasoningPartAddedEvent( - type="response.reasoning_part.added", + ResponseContentPartAddedEvent( + type="response.content_part.added", sequence_number=-1, output_index=state.current_output_index, item_id=state.current_item_id, content_index=state.current_content_index, - part=ResponseReasoningTextContent( - text="", - type="reasoning_text", - ), + part={"text": "", "type": "reasoning_text"}, ) ) events.append( @@ -457,13 +452,13 @@ def emit_reasoning_done_events( ) ) events.append( - ResponseReasoningPartDoneEvent( - type="response.reasoning_part.done", + ResponseContentPartDoneEvent( + type="response.content_part.done", sequence_number=-1, item_id=state.current_item_id, output_index=state.current_output_index, content_index=state.current_content_index, - part=content, + part=content.model_dump(), ) ) events.append( @@ -969,16 +964,13 @@ def emit_simple_reasoning_open( status="in_progress", ), ), - ResponseReasoningPartAddedEvent( - type="response.reasoning_part.added", + ResponseContentPartAddedEvent( + type="response.content_part.added", sequence_number=-1, output_index=state.output_index, item_id=state.current_item_id, content_index=state.content_index, - part=ResponseReasoningTextContent( - text="", - type="reasoning_text", - ), + part={"text": "", "type": "reasoning_text"}, ), ] @@ -1016,13 +1008,13 @@ def emit_simple_reasoning_done( content_index=state.content_index, text=state.accumulated_text, ), - ResponseReasoningPartDoneEvent( - type="response.reasoning_part.done", + ResponseContentPartDoneEvent( + type="response.content_part.done", sequence_number=-1, item_id=state.current_item_id, output_index=state.output_index, content_index=state.content_index, - part=part, + part=part.model_dump(), ), ResponseOutputItemDoneEvent( type="response.output_item.done", From 7b44aad5db820f8914810ae9f238ffb871e86d44 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:32:27 +0100 Subject: [PATCH 213/248] [MyPy] Fix mypy errors in `vllm/model_executor/models/[kK]*` (#59402) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- tools/pre_commit/mypy.py | 8 --- vllm/model_executor/models/k2_horizon.py | 45 ++++++++++------ vllm/model_executor/models/kanana_v.py | 31 +++++------ vllm/model_executor/models/keye.py | 63 ++++++++++------------ vllm/model_executor/models/keye_vl1_5.py | 28 +++++----- vllm/model_executor/models/kimi_audio.py | 4 +- vllm/model_executor/models/kimi_k25.py | 18 ++++--- vllm/model_executor/models/kimi_k25_vit.py | 2 +- vllm/model_executor/models/kimi_vl.py | 5 +- vllm/transformers_utils/configs/kimi_vl.py | 2 +- 10 files changed, 106 insertions(+), 100 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 4bef23869b..a6c8e46419 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -109,10 +109,6 @@ SEPARATE_GROUPS = [ "tests/v1/worker", ] -EXCLUDE = [ - r"vllm/model_executor/models/[kK]", -] - def group_files(changed_files: list[str]) -> dict[str, list[str]]: """Group changed files into different mypy calls. @@ -124,16 +120,12 @@ def group_files(changed_files: list[str]) -> dict[str, list[str]]: A dictionary mapping file group names to lists of changed files. """ - exclude_pattern = re.compile(f"^{'|'.join(EXCLUDE)}.*") silent_pattern = re.compile(f"^({'|'.join(SILENT_GROUPS)}).*") file_groups: dict[str, list[str]] = {"": []} file_groups.update({k: [] for k in SEPARATE_GROUPS}) # Longest path first so a sub-directory is not shadowed by its parent separate_groups = sorted(SEPARATE_GROUPS, key=len, reverse=True) for changed_file in changed_files: - # Skip files which should be ignored completely - if exclude_pattern.match(changed_file): - continue # Already-fixed paths go in the default group, which runs at the # stricter follow_imports setting from pyproject.toml if silent_pattern.match(changed_file): diff --git a/vllm/model_executor/models/k2_horizon.py b/vllm/model_executor/models/k2_horizon.py index 8ef517ac32..6edb9f7b42 100644 --- a/vllm/model_executor/models/k2_horizon.py +++ b/vllm/model_executor/models/k2_horizon.py @@ -502,6 +502,7 @@ class K2HorizonSparseMoeBlock(nn.Module): self.gate.bias = nn.Parameter(self.gate.bias.float(), requires_grad=False) self.num_shared_experts = config.num_shared_experts + self.shared_experts: K2HorizonMLP | None if config.num_shared_experts > 0: self.shared_experts = K2HorizonMLP( hidden_size=config.hidden_size, @@ -644,6 +645,14 @@ class K2HorizonAttention(nn.Module): rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, ) + attn_kwargs: dict[str, Any] = ( + { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } + if dual_chunk_attention_config + else {} + ) self.attn = Attention( self.num_heads, self.head_dim, @@ -652,12 +661,7 @@ class K2HorizonAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attn_kwargs, ) self.query_key_norm = query_key_norm @@ -842,6 +846,14 @@ class K2HorizonMoVAAttention(nn.Module): rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, ) + attn_kwargs: dict[str, Any] = ( + { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } + if dual_chunk_attention_config + else {} + ) self.attn = Attention( self.num_heads, self.head_dim, @@ -850,12 +862,7 @@ class K2HorizonMoVAAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attn_kwargs, ) self.query_key_norm = query_key_norm @@ -1228,9 +1235,10 @@ class K2HorizonModel(nn.Module, EagleModelMixin): expert_params_mapping = self.get_expert_mapping() for name, loaded_weight in weights: if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name # QK norm weights if name.endswith(".self_attn.q_norm.weight") or name.endswith( @@ -1267,7 +1275,9 @@ class K2HorizonModel(nn.Module, EagleModelMixin): kv_rank ] - weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader: Callable[..., Any] = getattr( + param, "weight_loader", default_weight_loader + ) weight_loader(param, loaded_weight) loaded_params.add(name) continue @@ -1380,9 +1390,10 @@ class K2HorizonModel(nn.Module, EagleModelMixin): continue if name.endswith("scale"): # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is None: continue + name = remapped_name if name not in params_dict: continue diff --git a/vllm/model_executor/models/kanana_v.py b/vllm/model_executor/models/kanana_v.py index fd076d66e7..db47fae841 100644 --- a/vllm/model_executor/models/kanana_v.py +++ b/vllm/model_executor/models/kanana_v.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Iterable, Mapping, Sequence from functools import partial -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias, cast import numpy as np import regex as re @@ -288,7 +288,7 @@ class CustomQwen2VLVE(Qwen2VisionTransformer): """Drop-in replacement for the HF `_from_config` constructor.""" return cls(config) - def forward( + def forward( # type: ignore[override] self, pixel_values: torch.Tensor, grid_thw: torch.Tensor, @@ -331,12 +331,12 @@ class CustomQwen2VLVE(Qwen2VisionTransformer): # Pre-compute seqlens for attention backend. max_seqlen = self.compute_attn_mask_seqlen(cu_seqlens) - encoder_states = () if output_hidden_states else None + encoder_states: tuple[torch.Tensor, ...] = () for blk in self.blocks: if output_hidden_states: # Store patch-level states (S, D). - encoder_states = encoder_states + (x.squeeze(1),) + encoder_states += (x.squeeze(1),) x = blk( x, @@ -348,14 +348,15 @@ class CustomQwen2VLVE(Qwen2VisionTransformer): # Final hidden state at patch level (S, D). hidden_states = x.squeeze(1) + all_hidden_states = None if output_hidden_states: - encoder_states = encoder_states + (hidden_states,) + all_hidden_states = encoder_states + (hidden_states,) if not return_dict: - return tuple(v for v in [hidden_states, encoder_states] if v is not None) + return tuple(v for v in [hidden_states, all_hidden_states] if v is not None) return BaseModelOutput( last_hidden_state=hidden_states, - hidden_states=encoder_states, + hidden_states=all_hidden_states, ) def get_num_tokens(self) -> int: @@ -479,7 +480,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) assert isinstance(prompt_text, str) # Images - image_inputs = hf_data.get("images", []) + image_inputs = cast(list[Any], hf_data.get("images", [])) pixel_sizes = [] if not isinstance(image_inputs[0], Image.Image): image_inputs = [Image.fromarray(image) for image in image_inputs] @@ -487,9 +488,9 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) image_processor = self.info.get_hf_processor().image_processor processor_output = [image_processor(image) for image in image_inputs] pixel_values = [o["pixel_values"] for o in processor_output] - image_meta = [o["image_meta"] for o in processor_output] + image_meta_list = [o["image_meta"] for o in processor_output] # list of dict -> dict of list - image_meta = {k: [d[k] for d in image_meta] for k in image_meta[0]} + image_meta = {k: [d[k] for d in image_meta_list] for k in image_meta_list[0]} for pixel_value in pixel_values: pixel_sizes.append(pixel_value.shape[0]) @@ -545,7 +546,7 @@ class KananaVMultiModalProcessor(BaseMultiModalProcessor[KananaVProcessingInfo]) ) -> Sequence[PromptUpdate]: tokenizer = self.info.get_tokenizer() - def get_replacement(idx: int) -> Sequence[int]: + def get_replacement(idx: int) -> list[int]: out_item = out_mm_kwargs["image"][idx] image_token_thw = out_item["image_token_thw"].data assert isinstance(image_token_thw, torch.Tensor) @@ -683,7 +684,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) def _get_visual_feature_at( self, v_output: Sequence[torch.Tensor], - layer_index: int | Sequence[int], + layer_index: int | list[int] | tuple[int, ...], ) -> torch.Tensor: if isinstance(layer_index, (list, tuple)): visual_features = torch.stack(v_output, dim=1)[ @@ -696,7 +697,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) def forward_vision( self, pixel_values: torch.Tensor, - image_metas: dict | None = None, + image_metas: dict, ) -> torch.Tensor: vision_model_args = { "pixel_values": pixel_values, @@ -714,7 +715,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) def forward_projector( self, visual_features: torch.Tensor, - image_metas: dict | None = None, + image_metas: dict, ) -> torch.Tensor: visual_embeds = self.abstractor( visual_features, @@ -725,7 +726,7 @@ class KananaVForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP) def forward_and_project_vision( self, pixel_values: torch.Tensor, - image_metas: dict | None = None, + image_metas: dict, ) -> torch.Tensor: assert pixel_values is not None visual_features = self.forward_vision(pixel_values, image_metas=image_metas) diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 7bf5d79709..60d0d03b65 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -208,8 +208,8 @@ class KeyeVisionEmbeddings(nn.Module): self.num_patches = (self.image_size // self.patch_size) ** 2 self.num_positions = self.num_patches - self.cache_position_embedding = dict() - self.cache_position_count = dict() + self.cache_position_embedding: dict[tuple[int, int], torch.Tensor] = {} + self.cache_position_count: dict[tuple[int, int], int] = {} self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) self.packing_position_embedding = nn.Embedding(32768, self.embed_dim) @@ -264,7 +264,7 @@ class KeyeVisionEmbeddings(nn.Module): if len(self.cache_position_embedding) >= max_cache: min_hit_grid = min( self.cache_position_count, - key=self.cache_position_count.get, + key=self.cache_position_count.__getitem__, ) self.cache_position_count.pop(min_hit_grid) self.cache_position_embedding.pop(min_hit_grid) @@ -278,8 +278,7 @@ class KeyeVisionEmbeddings(nn.Module): self, pixel_values: torch.FloatTensor, position_ids: torch.Tensor | None = None, - image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] - | None = None, + image_grid_thw: list[tuple[int, int, int]] | None = None, interpolate_pos_encoding=False, ) -> torch.Tensor: if pixel_values.dim() == 4: @@ -408,7 +407,7 @@ class KeyeSiglipAttention(nn.Module): hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None, output_attentions: bool | None = False, - cu_seqlens: list[torch.Tensor] | None = None, + cu_seqlens: torch.Tensor | None = None, rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) @@ -417,6 +416,8 @@ class KeyeSiglipAttention(nn.Module): dim=-1, ) + if cu_seqlens is None: + raise ValueError("cu_seqlens cannot be None.") max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max() if rope_emb is None: @@ -432,8 +433,6 @@ class KeyeSiglipAttention(nn.Module): self.head_dim, ) else: - if cu_seqlens is None: - raise ValueError("cu_seqlens cannot be None when rope_emb is not None.") cos, sin = rope_emb q = q.view(*q.shape[:-1], self.num_heads, self.head_dim) k = k.view( @@ -511,7 +510,7 @@ class KeyeSiglipEncoderLayer(nn.Module): hidden_states: torch.Tensor, attention_mask: torch.Tensor, output_attentions: bool | None = False, - cu_seqlens: list[torch.Tensor] | None = None, + cu_seqlens: torch.Tensor | None = None, rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None, ) -> tuple[torch.FloatTensor]: residual = hidden_states @@ -576,13 +575,12 @@ class KeyeSiglipEncoder(nn.Module): attention_mask: torch.Tensor | None = None, output_attentions: bool | None = None, output_hidden_states: bool | None = None, - cu_seqlens: list[torch.Tensor] | None = None, - image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] - | None = None, + cu_seqlens: torch.Tensor | None = None, + image_grid_thw: list[tuple[int, int, int]] | None = None, height_position_ids: torch.Tensor | None = None, width_position_ids: torch.Tensor | None = None, use_rope: bool | None = False, - window_size: bool | None = -1, + window_size: int = -1, vision_or_text: str = "vision", ) -> BaseModelOutput: device = inputs_embeds.device @@ -662,14 +660,13 @@ class KeyeSiglipVisionTransformer(nn.Module): position_ids: torch.Tensor | None = None, height_position_ids: torch.Tensor | None = None, width_position_ids: torch.Tensor | None = None, - cu_seqlens: list[torch.Tensor] | None = None, + cu_seqlens: torch.Tensor | None = None, padding_mask: torch.Tensor | None = None, vision_return_embed_list: bool | None = False, - image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] - | None = None, + image_grid_thw: list[tuple[int, int, int]] | None = None, return_pooler_output: bool | None = True, use_rope: bool | None = False, - window_size: bool | None = -1, + window_size: int = -1, ) -> BaseModelOutputWithPooling: hidden_states = self.embeddings( pixel_values, @@ -757,12 +754,11 @@ class KeyeSiglipVisionModel(nn.Module): interpolate_pos_encoding: bool = False, position_ids: torch.Tensor | None = None, vision_return_embed_list: bool | None = False, - image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]] - | None = None, - cu_seqlens: list[torch.Tensor] | None = None, + image_grid_thw: list[tuple[int, int, int]] | None = None, + cu_seqlens: torch.Tensor | None = None, return_pooler_output: bool | None = True, use_rope: bool | None = False, - window_size: bool | None = -1, + window_size: int = -1, ) -> BaseModelOutputWithPooling: return self.vision_model( pixel_values=pixel_values, @@ -1337,7 +1333,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): def _process_video_embeds( self, video_type: Literal["video_embeds", "pixel_values_videos"], - video_grid_thw: list[torch.Tensor], + video_grid_thw: torch.Tensor, pixel_values_videos: torch.Tensor | None = None, ) -> torch.Tensor | list[torch.Tensor]: siglip_position_ids = list() @@ -1361,6 +1357,7 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): "Video embeddings are not supported for this processing path." ) else: + assert pixel_values_videos is not None pixel_values_videos = pixel_values_videos.type(self.visual.dtype) # These are host-built; concat straight into pinned buffers so # the H2D copies stay non-blocking. @@ -1535,12 +1532,11 @@ class KeyeForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: - return KeyeImageEmbeddingInputs( - type="image_embeds", - image_embeds=image_embeds, - image_grid_thw=image_grid_thw, - ) + return KeyeImageEmbeddingInputs( + type="image_embeds", + image_embeds=image_embeds, + image_grid_thw=image_grid_thw, + ) def _parse_and_validate_video_input( self, **kwargs: object @@ -1559,12 +1555,11 @@ class KeyeForConditionalGeneration( video_grid_thw=video_grid_thw, ) - if video_embeds is not None: - return KeyeVideoEmbeddingInputs( - type="video_embeds", - video_embeds=video_embeds, - video_grid_thw=video_grid_thw, - ) + return KeyeVideoEmbeddingInputs( + type="video_embeds", + video_embeds=video_embeds, + video_grid_thw=video_grid_thw, + ) def _process_video_input( self, video_input: KeyeVideoInputs diff --git a/vllm/model_executor/models/keye_vl1_5.py b/vllm/model_executor/models/keye_vl1_5.py index 3bc7f36420..fc08035fa7 100644 --- a/vllm/model_executor/models/keye_vl1_5.py +++ b/vllm/model_executor/models/keye_vl1_5.py @@ -75,7 +75,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor: def get_num_patches( grid_thw: torch.Tensor, num_frames: list[int] | torch.Tensor -) -> list[int]: +) -> torch.Tensor: """Return num_patches per video. Args: @@ -84,7 +84,7 @@ def get_num_patches( num_frames: List or tensor indicating the number of frames per video Returns: - List of ints representing the number of patches for each video + Tensor of the number of patches for each video Examples: >>> # Suppose there are 2 videos with a total of 3 grids @@ -538,12 +538,11 @@ class KeyeVL1_5ForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: - return KeyeVL1_5ImageEmbeddingInputs( - type="image_embeds", - image_embeds=image_embeds, - image_grid_thw=image_grid_thw, - ) + return KeyeVL1_5ImageEmbeddingInputs( + type="image_embeds", + image_embeds=image_embeds, + image_grid_thw=image_grid_thw, + ) def _parse_and_validate_video_input( self, **kwargs: object @@ -564,13 +563,12 @@ class KeyeVL1_5ForConditionalGeneration( num_frames=num_frames, ) - if video_embeds is not None: - return KeyeVL1_5VideoEmbeddingInputs( - type="video_embeds", - video_embeds=video_embeds, - video_grid_thw=video_grid_thw, - num_frames=num_frames, - ) + return KeyeVL1_5VideoEmbeddingInputs( + type="video_embeds", + video_embeds=video_embeds, + video_grid_thw=video_grid_thw, + num_frames=num_frames, + ) def _process_video_input( self, video_input: KeyeVL1_5VideoInputs diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index a2357836f1..9a404ce845 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -18,6 +18,7 @@ from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import PromptType, TokensPrompt from vllm.model_executor.model_loader import DefaultModelLoader from vllm.model_executor.models.interfaces import ( + MultiModalEmbeddings, SupportsMultiModal, SupportsPP, SupportsTranscription, @@ -240,6 +241,7 @@ class KimiAudioMultiModalProcessor(BaseMultiModalProcessor[KimiAudioProcessingIn # KimiAudioProcessor expects raw numpy arrays mm_data = hf_inputs.hf_data if audios := mm_data.pop("audio", []): + assert isinstance(audios, Sequence) audio_arrays = [] for aud in audios: if isinstance(aud, (tuple, list)) and len(aud) == 2: @@ -485,7 +487,7 @@ class KimiAudioForConditionalGeneration( def embed_input_ids( self, input_ids: torch.Tensor, - multimodal_embeddings: tuple[torch.Tensor, ...] | None = None, + multimodal_embeddings: MultiModalEmbeddings | None = None, *, is_multimodal: torch.Tensor | None = None, ) -> torch.Tensor: diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 81b5ec1646..2426ec61ef 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -136,9 +136,9 @@ class KimiK25ProcessingInfo(BaseProcessingInfo): # may remap token IDs vs config.json. config_token_id = hf_config.media_placeholder_token_id resolved_token_id = tokenizer.convert_tokens_to_ids("<|media_pad|>") + unk_token_id = getattr(tokenizer, "unk_token_id", None) is_valid_resolved = isinstance(resolved_token_id, int) and ( - tokenizer.unk_token_id is None - or resolved_token_id != tokenizer.unk_token_id + unk_token_id is None or resolved_token_id != unk_token_id ) if is_valid_resolved and resolved_token_id != config_token_id: logger.warning_once( @@ -165,7 +165,7 @@ class KimiK25ProcessingInfo(BaseProcessingInfo): ) self.media_tokens_calculator = image_processor.media_tokens_calculator - def get_hf_processor(self): + def get_hf_processor(self, **kwargs: object): return self.hf_processor def get_hf_config(self): @@ -191,7 +191,11 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]): ) video_chunk_dummy_item = VisionChunkVideo( - type="video_chunk", video_chunk=dummy_videos + type="video_chunk", + video_chunk=dummy_videos, + uuid=None, + prompt="", + video_idx=0, ) video_chunk_num_tokens = self.info.media_tokens_calculator( video_chunk_dummy_item @@ -204,6 +208,7 @@ class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]): width=MaxImageTokenMeta.width, num_images=1, )[0], + uuid=None, ) image_num_tokens = self.info.media_tokens_calculator(image_dummy_item) # return the larger one @@ -377,7 +382,7 @@ class KimiK25ForConditionalGeneration( ) self.media_placeholder: int = self.config.media_placeholder_token_id - def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig): + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig | None): if isinstance(quant_config, compressed_tensors.CompressedTensorsConfig): return None return quant_config @@ -392,6 +397,7 @@ class KimiK25ForConditionalGeneration( if isinstance(pixel_values, list): pixel_values = torch.cat(pixel_values, dim=0) + assert isinstance(pixel_values, torch.Tensor) if len(pixel_values.shape) == 5 or len(pixel_values.shape) == 3: pixel_values = pixel_values.reshape( @@ -612,7 +618,7 @@ class KimiK25ForConditionalGeneration( grid_thw_list = [[1, ho * kh, wo * kw] for _ in range(max_batch_size)] - ps = self.vision_tower.patch_size + ps: int | tuple[int, int] = self.vision_tower.patch_size if isinstance(ps, int): ps = (ps, ps) total_patches = max_batch_size * ho * kh * wo * kw diff --git a/vllm/model_executor/models/kimi_k25_vit.py b/vllm/model_executor/models/kimi_k25_vit.py index 72e45d82e6..ee8adb6a74 100644 --- a/vllm/model_executor/models/kimi_k25_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -804,7 +804,7 @@ class MoonViT3dPretrainedModel(nn.Module): self, grid_thw_list: list[list[int]], *, - max_batch_size: int, + max_batch_size: int | None, max_seqlen_override: int | None = None, device: torch.device, ) -> dict[str, torch.Tensor | None]: diff --git a/vllm/model_executor/models/kimi_vl.py b/vllm/model_executor/models/kimi_vl.py index 13779042b6..50444a0c61 100644 --- a/vllm/model_executor/models/kimi_vl.py +++ b/vllm/model_executor/models/kimi_vl.py @@ -268,6 +268,7 @@ class KimiVLMultiModalProcessor(BaseMultiModalProcessor[KimiVLProcessingInfo]): if isinstance(images, ImageEmbeddingItems): num_image_tokens = images.get_feature_size(item_idx) else: + assert isinstance(images, ImageProcessorItems) image_size = images.get_image_size(item_idx) num_image_tokens = self.info.get_num_image_tokens( image_width=image_size.width, @@ -317,7 +318,7 @@ class KimiVLForConditionalGeneration( assert isinstance(config.vision_config, MoonViTConfig) self.use_data_parallel = ( - model_config.multimodal_config.mm_encoder_tp_mode == "data" + model_config.get_multimodal_config().mm_encoder_tp_mode == "data" ) self.hidden_size = config.text_config.hidden_size @@ -469,7 +470,7 @@ class KimiVLForConditionalGeneration( ) grid_hws_list = [(ho * kh, wo * kw) for _ in range(max_batch_size)] - patch_size = self.config.vision_config.patch_size + patch_size: int | tuple[int, int] = self.config.vision_config.patch_size if isinstance(patch_size, int): patch_size = (patch_size, patch_size) diff --git a/vllm/transformers_utils/configs/kimi_vl.py b/vllm/transformers_utils/configs/kimi_vl.py index 5733afd668..1b2d32a263 100644 --- a/vllm/transformers_utils/configs/kimi_vl.py +++ b/vllm/transformers_utils/configs/kimi_vl.py @@ -30,7 +30,7 @@ class KimiVLConfig(PreTrainedConfig): text_config = DeepseekV2Config() elif isinstance(text_config, dict): text_config = DeepseekV2Config(**text_config) - self.text_config = text_config + self.text_config: DeepseekV2Config = text_config self.ignore_index = ignore_index self.media_placeholder_token_id = media_placeholder_token_id From 840c7c2f4bc72c7c55edb2dea6d23f9a9c9734e0 Mon Sep 17 00:00:00 2001 From: djramic Date: Thu, 1 Oct 2026 21:38:55 +0200 Subject: [PATCH 214/248] [Bugfix][Engram] Fix intermittent Triton 3.8 crash in the lookup kernel (#59639) Signed-off-by: Djordje Ramic Signed-off-by: Andreas Karatzas Co-authored-by: Andreas Karatzas Co-authored-by: Andreas Karatzas --- vllm/models/deepseek_v41/common/engram.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/vllm/models/deepseek_v41/common/engram.py b/vllm/models/deepseek_v41/common/engram.py index 8452c1ad8a..d4810ee4ea 100644 --- a/vllm/models/deepseek_v41/common/engram.py +++ b/vllm/models/deepseek_v41/common/engram.py @@ -605,6 +605,15 @@ def _engram_head_shard_weight_loader( param.data.copy_(shard) +# Branching on SORTED at runtime intermittently crashes Triton 3.8's +# RemoveLayoutConversions. +# TODO: Remove the 3.8 check once Triton ships +# https://github.com/triton-lang/triton/pull/10706. +_SORTED_IS_CONSTEXPR = tl.constexpr( + triton.__version__.startswith("3.8") and current_platform.is_rocm() +) + + @triton.jit( do_not_specialize=[ "vocab_start", @@ -613,7 +622,7 @@ def _engram_head_shard_weight_loader( "ids_stride_t", "ids_stride_h", "GRID", - "SORTED", + *(() if _SORTED_IS_CONSTEXPR else ("SORTED",)), ] ) def _engram_lookup_kernel( @@ -634,7 +643,7 @@ def _engram_lookup_kernel( QUANT_BLOCK: tl.constexpr, BLOCK_R: tl.constexpr, GRID, - SORTED, + SORTED: tl.constexpr if _SORTED_IS_CONSTEXPR else None, # type: ignore[valid-type] ): """Gather fp8 rows, apply their ue8m0 block scales, write bf16. From ccda9098cc5a91b6b6f73e0c0e697022382e01b4 Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Thu, 1 Oct 2026 21:54:26 +0200 Subject: [PATCH 215/248] [ROCm][CI] Drop two no-GPU AMD mirrors from the CPU test areas (#59593) Signed-off-by: Stefan Koncarevic Co-authored-by: Andreas Karatzas --- .buildkite/test_areas/models_basic.yaml | 10 ---------- .buildkite/test_areas/models_multimodal.yaml | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index b817b9a1fb..735735c15e 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -223,13 +223,3 @@ steps: device: cpu-small commands: - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py - mirror: - amd: - label: ":amd: (MI250) Basic Models Other" - dind: false - device: mi250_1 - no_gpu: true - working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 35 - depends_on: - - image-build-amd diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 82c77f58ef..0750d92eac 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -99,17 +99,6 @@ steps: commands: - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB parallelism: 4 - mirror: - amd: - label: ":amd: (MI250) Multimodal Processor Shard %N" - dind: false - device: mi250_1 - no_gpu: true - working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 90 - parallelism: 6 - depends_on: - - image-build-amd - label: ":nvidia: (H200 MIG 18GB) Multimodal Processor Shard %N" key: multi-modal-processor From ea84051819c2a52a535e4b6df1fd828def5880f7 Mon Sep 17 00:00:00 2001 From: Zijing Liu Date: Thu, 1 Oct 2026 14:30:58 -0700 Subject: [PATCH 216/248] [KVConnector][NIXL] Count completion notifications that arrive after KV expiry (#58875) Signed-off-by: Zijing Liu --- docs/features/nixl_connector_usage.md | 1 + .../kv_connector/unit/test_multi_connector.py | 12 ++++++++ .../kv_connector/unit/test_nixl_connector.py | 30 +++++++++++++++++++ .../unit/test_nixl_push_connector.py | 1 + .../kv_connector/v1/nixl/pull_worker.py | 1 + .../kv_connector/v1/nixl/push_worker.py | 1 + .../kv_transfer/kv_connector/v1/nixl/stats.py | 24 ++++++++++++++- 7 files changed, 69 insertions(+), 1 deletion(-) diff --git a/docs/features/nixl_connector_usage.md b/docs/features/nixl_connector_usage.md index 0d1f16e2aa..dab0faaf98 100644 --- a/docs/features/nixl_connector_usage.md +++ b/docs/features/nixl_connector_usage.md @@ -454,6 +454,7 @@ exported when NixlConnector is active: | `vllm:nixl_num_failed_transfers` | Counter | Cumulative count of failed NIXL KV-block transfers, including handshake and notification (`send_notif`) failures. These are grouped because all are sporadic lower-transport-layer events. | | `vllm:nixl_num_failed_notifications` | Counter | Cumulative count of failed notifications (`send_notif`), retained for compatibility. These failures are also included in `vllm:nixl_num_failed_transfers`; do not sum the two counters. | | `vllm:nixl_num_kv_expired_reqs` | Counter | Requests whose KV blocks expired on the prefiller before the decoder read them (tracked on the P instance). Kept separate from the failure counters above: KV expiry is an autoscaler/lease-tuning signal, not a transport issue. | +| `vllm:nixl_num_notifications_after_expiry` | Counter | Completion notifications that arrive for a request that is no longer tracked, usually because its KV lease expired first. Pull connector: counted on P, where it means the decoder may have read blocks that were already reused; each matches a "Potentially invalid KV blocks" error log. Push connector: counted on whichever instance tracks the request as neither a send nor a receive (P after the lease expired, or D after the receive completed or was abandoned); each matches an "Unrecognized request" error log. Counted per notification, not per request: each remote rank notifies separately, so with TP > 1 or heterogeneous TP one late request counts several times, and the value is not comparable to `vllm:nixl_num_kv_expired_reqs`. | !!! tip High `vllm:nixl_num_kv_expired_reqs` indicates that the prefiller's lease diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index 9d04a2ea53..e636aabfd9 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -601,6 +601,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } } @@ -626,6 +627,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], }, "MockConnector": {"mock_field": [1, 2, 3]}, } @@ -657,6 +659,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], }, } @@ -678,6 +681,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) mock_stats = MockConnectorStats(data={"mock_field": [1, 2, 3]}) @@ -709,6 +713,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) @@ -742,6 +747,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], }, "ExampleConnector": {"some_field": [1, 2, 3]}, } @@ -781,6 +787,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -798,6 +805,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -831,6 +839,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -859,6 +868,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -887,6 +897,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } @@ -918,6 +929,7 @@ class TestMultiConnectorStats: "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } ) } diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index 6be77b7b9d..07ae290219 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -1698,6 +1698,7 @@ def test_kv_connector_stats_failure_grouping(): stats.record_failed_handshake() stats.record_failed_notification() stats.record_kv_expired_req() + stats.record_notification_after_expiry() assert not stats.is_empty() # No successful transfers: latency stats are zero but the failure @@ -1706,6 +1707,7 @@ def test_kv_connector_stats_failure_grouping(): assert reduced["Num successful transfers"] == 0 assert reduced["Num failed transfers"] == 3 assert reduced["Num KV expired reqs"] == 1 + assert reduced["Num notifs after expiry"] == 1 def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): @@ -1750,6 +1752,7 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): stats.record_failed_handshake() stats.record_failed_notification() stats.record_kv_expired_req() + stats.record_notification_after_expiry() prom.observe(stats.data, engine_idx=0) def counter_value(name: str) -> float: @@ -1761,6 +1764,33 @@ def test_nixl_prom_metrics_group_handshake_with_transfer_failures(): assert counter_value("vllm:nixl_num_failed_transfers_total") == 3.0 assert counter_value("vllm:nixl_num_kv_expired_reqs_total") == 1.0 + assert counter_value("vllm:nixl_num_notifications_after_expiry_total") == 1.0 + + +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", + FakeNixlWrapper, +) +def test_notification_after_expiry_is_counted(default_vllm_config, dist_init): + vllm_config = create_vllm_config() + connector = NixlConnector( + vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16) + ) + connector.connector_worker = FakeNixlConnectorWorker( + vllm_config, connector.engine_id, hand_shake_latency=0 + ) + worker = connector.connector_worker + worker._reqs_to_process.add("known") + worker._reqs_to_send["known"] = time.perf_counter() + 10 + worker.nixl_wrapper.get_new_notifs = MagicMock( + return_value={"decode-agent": [b"known:1", b"unknown:1"]} + ) + + assert worker._get_new_notifs() == {"known"} + + stats = connector.get_kv_connector_stats() + assert isinstance(stats, NixlKVConnectorStats) + assert stats.data["num_notifications_after_expiry"] == [1] def test_multi_kv_connector_stats_aggregation(): diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index de47376117..47779c3611 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -1357,6 +1357,7 @@ class TestPushWriterNegative: assert notified == set() # Did not register anywhere. assert "never-heard-of-you" not in w._recving_transfers + assert w.xfer_stats.data["num_notifications_after_expiry"] == [1] def test_start_load_kv_with_empty_metadata_is_noop(self): """Empty metadata must not wake the writer or enqueue anything.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py index 11b7215950..4a18632eb4 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -660,6 +660,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker): req_id not in self._reqs_to_send and req_id not in self._reqs_to_process ): + self.xfer_stats.record_notification_after_expiry() logger.error( "Potentially invalid KV blocks for " "unrecognized request %s were retrieved by " diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index ec77f426c5..6d82fbcc0a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -786,6 +786,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): else: # Not tracked on either side (lease may have expired # before the notif arrived). Log and skip. + self.xfer_stats.record_notification_after_expiry() logger.error( "Unrecognized request %s notif (may have expired).", req_id, diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py index f6d0684b6b..26c4275efb 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py @@ -41,6 +41,7 @@ class NixlKVConnectorStats(KVConnectorStats): "num_failed_notifications": [], "num_failed_handshakes": [], "num_kv_expired_reqs": [], + "num_notifications_after_expiry": [], } def record_transfer(self, res: "nixlXferTelemetry"): @@ -66,6 +67,10 @@ class NixlKVConnectorStats(KVConnectorStats): """Record a request that had its KV blocks expire.""" self.data["num_kv_expired_reqs"].append(1) + def record_notification_after_expiry(self): + """Record a completion notification that arrived after its request expired.""" + self.data["num_notifications_after_expiry"].append(1) + def clone_and_reset(self) -> "NixlKVConnectorStats": old = copy.copy(self) self.reset() @@ -79,6 +84,7 @@ class NixlKVConnectorStats(KVConnectorStats): and len(self.data["num_failed_notifications"]) == 0 and len(self.data["num_failed_handshakes"]) == 0 and len(self.data["num_kv_expired_reqs"]) == 0 + and len(self.data["num_notifications_after_expiry"]) == 0 ) def aggregate(self, other: KVConnectorStats) -> KVConnectorStats: @@ -95,12 +101,14 @@ class NixlKVConnectorStats(KVConnectorStats): # and notification failures are grouped as sporadic # lower-transport-layer events, while KV expiry is reported separately # as it is an actionable autoscaler signal rather than a transport - # issue. + # issue. Notifications arriving after expiry are surfaced too, as the + # reader may have consumed KV blocks that were already reused. failure_counts = { "Num failed transfers": len(self.data["num_failed_transfers"]) + len(self.data["num_failed_handshakes"]) + len(self.data["num_failed_notifications"]), "Num KV expired reqs": len(self.data["num_kv_expired_reqs"]), + "Num notifs after expiry": len(self.data["num_notifications_after_expiry"]), } if self.num_successful_transfers == 0: # Timing / throughput stats only cover successful transfers. If @@ -262,6 +270,18 @@ class NixlPromMetrics(KVConnectorPromMetrics): counter_nixl_num_kv_expired_reqs, self.per_engine_labelvalues ) + counter_nixl_num_notifications_after_expiry = self._counter_cls( + name="vllm:nixl_num_notifications_after_expiry", + documentation="Number of completion notifications for requests that " + "were no longer tracked, usually because their KV lease expired. The " + "KV blocks may have been reused before the transfer finished. Counted " + "per notification (one per remote rank), not per request.", + labelnames=labelnames, + ) + self.counter_nixl_num_notifications_after_expiry = create_metric_per_engine( + counter_nixl_num_notifications_after_expiry, self.per_engine_labelvalues + ) + def observe(self, transfer_stats_data: dict[str, Any], engine_idx: int = 0): for prom_obj, list_item_key in zip( [ @@ -284,6 +304,7 @@ class NixlPromMetrics(KVConnectorPromMetrics): self.counter_nixl_num_failed_transfers, self.counter_nixl_num_failed_notifications, self.counter_nixl_num_kv_expired_reqs, + self.counter_nixl_num_notifications_after_expiry, ], [ # Transfer, handshake and notification failures are grouped: @@ -297,6 +318,7 @@ class NixlPromMetrics(KVConnectorPromMetrics): ), ("num_failed_notifications",), ("num_kv_expired_reqs",), + ("num_notifications_after_expiry",), ], ): for counter_item_key in counter_item_keys: From 7566d83bd35cdfd7cd3f9e521a005d2fc1ed1c33 Mon Sep 17 00:00:00 2001 From: Yongye Zhu Date: Thu, 1 Oct 2026 14:39:55 -0700 Subject: [PATCH 217/248] [Attention][MiniMax-M3] NVFP4 KV cache on the MSA sparse attention path (#59300) Signed-off-by: Yongye Zhu Co-authored-by: Claude Opus 5.5 --- CMakeLists.txt | 21 + cmake/external_projects/fmha_sm100.cmake | 16 +- ...minimax_m3_qknorm_rope_kv_insert_kernel.cu | 433 +++++++++++++++-- csrc/libtorch_stable/ops.h | 4 +- csrc/libtorch_stable/torch_bindings.cpp | 3 +- ...st_minimax_m3_msa_cutlass_sparse_decode.py | 442 +++++++++++++++++- ..._fused_minimax_m3_qknorm_rope_kv_insert.py | 230 ++++++++- vllm/_custom_ops.py | 13 +- .../minimax_m3/common/sparse_attention.py | 22 +- vllm/models/minimax_m3/nvidia/model.py | 8 +- .../nvidia/msa_cutlass_sparse_decode.py | 59 ++- .../minimax_m3/nvidia/sparse_attention_msa.py | 119 ++++- 12 files changed, 1297 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 02e098f2d9..0a5414b4bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -590,9 +590,30 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP") endif() endif() + # The fused MiniMax-M3 KV insert quantizes NVFP4 with SM100-family-only + # cvt instructions, so its SM10x/11x code is built for the family targets. + set(MINIMAX_M3_FUSED_SRC + "csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu") + list(REMOVE_ITEM VLLM_STABLE_EXT_SRC "${MINIMAX_M3_FUSED_SRC}") set_gencode_flags_for_srcs( SRCS "${VLLM_STABLE_EXT_SRC}" CUDA_ARCHS "${CUDA_ARCHS}") + if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0) + cuda_archs_loose_intersection(MINIMAX_M3_FUSED_ARCHS + "10.0f;10.7f;11.0f" "${CUDA_ARCHS}") + else() + cuda_archs_loose_intersection(MINIMAX_M3_FUSED_ARCHS + "10.0a;10.1a;10.3a" "${CUDA_ARCHS}") + endif() + foreach(_ARCH ${CUDA_ARCHS}) + if(_ARCH VERSION_LESS 10.0 OR _ARCH VERSION_GREATER_EQUAL 12.0) + list(APPEND MINIMAX_M3_FUSED_ARCHS "${_ARCH}") + endif() + endforeach() + set_gencode_flags_for_srcs( + SRCS "${MINIMAX_M3_FUSED_SRC}" + CUDA_ARCHS "${MINIMAX_M3_FUSED_ARCHS}") + list(APPEND VLLM_STABLE_EXT_SRC "${MINIMAX_M3_FUSED_SRC}") if(COOPERATIVE_TOPK_ARCHS) list(APPEND VLLM_STABLE_EXT_SRC diff --git a/cmake/external_projects/fmha_sm100.cmake b/cmake/external_projects/fmha_sm100.cmake index ae5bfec7b4..f6bb0b8ace 100644 --- a/cmake/external_projects/fmha_sm100.cmake +++ b/cmake/external_projects/fmha_sm100.cmake @@ -17,7 +17,7 @@ else() FetchContent_Declare( fmha_sm100 GIT_REPOSITORY https://github.com/vllm-project/MSA.git - GIT_TAG f355c37eb4e1413f21ee2ad8bbad25079e6bef9d + GIT_TAG be4a52aea420b6bef73be9af1a649859964790cb GIT_PROGRESS TRUE CONFIGURE_COMMAND "" BUILD_COMMAND "" @@ -39,6 +39,8 @@ install(FILES "${FMHA_SM100_PY_ROOT}/api.py" "${FMHA_SM100_PY_ROOT}/bench_utils.py" "${FMHA_SM100_PY_ROOT}/jit.py" + "${FMHA_SM100_PY_ROOT}/nvfp4_kv.py" + "${FMHA_SM100_PY_ROOT}/q8kv4_decode_adapter.py" "${FMHA_SM100_PY_ROOT}/sparse.py" "${FMHA_SM100_PY_ROOT}/sparse_fmha_adapter.py" DESTINATION vllm/third_party/fmha_sm100 @@ -51,6 +53,18 @@ install(DIRECTORY "${FMHA_SM100_PY_ROOT}/csrc/" PATTERN "*.pyc" EXCLUDE PATTERN ".git*" EXCLUDE) +install(DIRECTORY "${FMHA_SM100_PY_ROOT}/decode_q8kv4/" + DESTINATION vllm/third_party/fmha_sm100/decode_q8kv4 + COMPONENT fmha_sm100 + PATTERN "__pycache__" EXCLUDE + PATTERN "*.pyc" EXCLUDE) + +install(DIRECTORY "${FMHA_SM100_PY_ROOT}/kvouter/" + DESTINATION vllm/third_party/fmha_sm100/kvouter + COMPONENT fmha_sm100 + PATTERN "__pycache__" EXCLUDE + PATTERN "*.pyc" EXCLUDE) + install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cute/" DESTINATION vllm/third_party/fmha_sm100/cute COMPONENT fmha_sm100 diff --git a/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu b/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu index 8769ba723b..0a45af8ade 100644 --- a/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu +++ b/csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu @@ -285,6 +285,262 @@ __device__ __forceinline__ void storeScaledQElemsFp8( storeElemsFp8(dst, scaled); } +// index_q: E4M3(scalar_t(v)) with unit scale, i.e. q is rounded to the model +// dtype before the fp8 cast. On CUDA each element pair is rounded to scalar_t +// and then converted to E4M3, with one packed cvt each. +template +__device__ __forceinline__ void storeIndexQElemsFp8( + uint8_t* __restrict__ dst, float const (&elems)[kElemsPerLane]) { +#ifndef USE_ROCM + using Converter = vllm::_typeConvert; + __nv_fp8x2_storage_t out2[kElemsPerLane / 2]; + #pragma unroll + for (int i = 0; i < kElemsPerLane / 2; i++) { + auto const v = + Converter::convert(make_float2(elems[2 * i], elems[2 * i + 1])); + if constexpr (std::is_same_v) { + out2[i] = __nv_cvt_bfloat16raw2_to_fp8x2(v, __NV_SATFINITE, __NV_E4M3); + } else { + out2[i] = __nv_cvt_halfraw2_to_fp8x2(v, __NV_SATFINITE, __NV_E4M3); + } + } + *reinterpret_cast(dst) = *reinterpret_cast(out2); +#else + storeScaledQElemsFp8(dst, elems, 1.0f); +#endif +} + +// ──────────────────────────────────────────────────────────────────────────── +// NVFP4 main K/V cache store +// ──────────────────────────────────────────────────────────────────────────── +// The block encoding of reshape_and_cache_nvfp4_kernel +// (nvfp4_kv_cache_kernels.cu) in the MSA readers' head-slot page layout. The +// quantizer differs from that kernel's reciprocal-multiply math: per 16-value +// block, scale = min(448, max(E4M3(max(amax, 1e-12) / 6), 1/512)) and +// code = sign | RNE-E2M1(min(|x / scale|, 6)), with zero stored as +0, all on +// x / the global scale. +// An HND page is 2*nkv slots of [bs, 72] bytes; slot 2*head + side (K = 0, +// V = 1) holds that head's packed E2M1 data [bs, 64] followed by its E4M3 block +// scales [bs, 8]. Every head is one contiguous 2*bs*72-byte run, so a page +// split across TP ranks is a contiguous byte slice per rank. K scales are +// linear, V scales use the SM100 trtllm-gen 4x4 token-quad swizzle. One +// 16-element scale group is four lanes. SM100-family builds of this file +// (CMake gives it family-specific SM10x/11x targets) encode E2M1 with +// cvt.rn.satfinite.e2m1x2; other targets use a bit-exact software encoder. +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \ + defined(__CUDA_ARCH_FAMILY_SPECIFIC__) + #define MINIMAX_M3_NVFP4_NATIVE_CVT 1 +#else + #define MINIMAX_M3_NVFP4_NATIVE_CVT 0 +#endif + +#ifndef USE_ROCM +constexpr int kNvfp4DataBytes = kHeadDim / 2; // 64 +constexpr int kNvfp4ScaleBytes = kHeadDim / 16; // 8 +constexpr int kNvfp4LanesPerGroup = 16 / kElemsPerLane; // 4 + +// Round-to-nearest-even, saturating E2M1 code of ``x`` (matches +// cvt.rn.satfinite.e2m1x2.f32). Below 1 the grid is the 0.5-step subnormals; +// from 1 up it is fp32 rounded to one mantissa bit, whose (exponent, mantissa) +// bits are the code offset by 252 once the exponent is rebiased. +__device__ __forceinline__ uint32_t e2m1Code(float x) { + float const ax = fabsf(x); + uint32_t mag; + if (ax < 1.0f) { + mag = static_cast(__float2int_rn(2.0f * ax)); + } else { + uint32_t const bits = __float_as_uint(ax); + uint32_t const rounded = bits + 0x1FFFFFu + ((bits >> 22) & 1u); + mag = min((rounded >> 22) - 252u, 7u); + } + return mag | ((__float_as_uint(x) >> 28) & 0x8u); +} + +__device__ __forceinline__ float rcpApproxFtz(float a) { + float b; + asm("rcp.approx.ftz.f32 %0, %1;" : "=f"(b) : "f"(a)); + return b; +} + +// a / b rounded to nearest even from r = RN(1 / b): Markstein's correction of +// a * r, which is __fdiv_rn's fast path without its range-check branch. Exact +// unless a * r overflows or the residual a - q * b underflows (a within a few +// binades of the subnormals). +__device__ __forceinline__ float divRnByRcp(float a, float b, float r) { + float const q = __fmul_rn(a, r); + return __fmaf_rn(__fmaf_rn(-q, b, a), r, q); +} + +// Block scale E4M3(max(amax, 1e-12) / 6) clamped to [1/512, 448], with +// sf_scale applied to it as to the values, and its reciprocal RN(1 / sf_value). +__device__ __forceinline__ void nvfp4GroupScale(float sf_scale, float group_max, + uint8_t& sf_byte, + float& sf_value, + float& sf_rcp) { + constexpr float kRcp6 = 0x1.555556p-3f; // RN(1 / 6) + float const amax = fmaxf(group_max, 1e-12f); + // amax / 6 (RNE). amax * kRcp6 bounds it from above, and fminf picks it for + // amax = inf, where the correction is NaN. + float const amax_6 = + fminf(divRnByRcp(amax, 6.0f, kRcp6), __fmul_rn(amax, kRcp6)); + // Flooring before the rounding gives the 1/512 minimum; satfinite caps it + // at 448. + __nv_fp8_e4m3 const sf(fmaxf(sf_scale * amax_6, 0x1p-9f)); + sf_byte = sf.__x; + sf_value = float(sf); + // rcp.approx is correctly rounded on every positive E4M3 value. + sf_rcp = rcpApproxFtz(sf_value); +} + +// nvfp4GroupScale for sf_scale = 1 and a bf16/fp16 group_max. Uncorrected, +// amax * RN(1 / 6) still rounds to the E4M3 byte of amax / 6 (checked for every +// such amax: amax / 6 is too far from an E4M3 rounding boundary to cross it, +// and exact ties round back), and the 1e-12 floor lies below the 1/512 one. +__device__ __forceinline__ void nvfp4UnitGroupScale(float group_max, + uint8_t& sf_byte, + float& sf_rcp) { + constexpr float kRcp6 = 0x1.555556p-3f; // RN(1 / 6) + __nv_fp8_e4m3 const sf(fmaxf(__fmul_rn(group_max, kRcp6), 0x1p-9f)); + sf_byte = sf.__x; + sf_rcp = rcpApproxFtz(float(sf)); +} + +// (x * sf_scale) / sf_value with the E2M1 code of the IEEE quotient. The +// quotient itself is the IEEE one except below 1e-35 (code 0 either way) and +// for magnitudes over 4096, which are clamped (NaN kept) so that the division +// stays finite: 4096 / 448 > 5, so their code stays the saturated 6. +__device__ __forceinline__ float nvfp4Quotient(float x, float sf_scale, + float sf_value, float sf_rcp) { + constexpr float kMaxMag = 4096.0f; + float n = __fmul_rn(x, sf_scale); + n = fabsf(n) > kMaxMag ? copysignf(kMaxMag, n) : n; + return divRnByRcp(n, sf_value, sf_rcp); +} + +// Zero is stored as +0: clear the sign of every nibble whose magnitude is 0. +// Adding 7 to a 3-bit magnitude carries into the sign bit exactly when it is +// nonzero. +__device__ __forceinline__ uint16_t clearNegZeroNibbles(uint16_t packed) { + return packed & (((packed & 0x7777u) + 0x7777u) | 0x7777u); +} + +// Quantize one K or V head row of one token into its NVFP4 page, where +// ``scale`` is the dequantization (global) scale. Every lane of the warp must +// call this (the scale-group reductions shuffle). +template +__device__ __forceinline__ void storeNvfp4CacheElems( + uint8_t* __restrict__ page, int const kv, int const head, int const token, + int const block_size, int const laneId, float const (&elems)[kElemsPerLane], + float const scale) { + // Quantize the model-dtype value that the unfused path reads back from qkv. + using Converter = vllm::_typeConvert; + float x[kElemsPerLane]; + float group_max = 0.0f; + #pragma unroll + for (int i = 0; i < kElemsPerLane; i++) { + x[i] = static_cast(Converter::convert(elems[i])); + group_max = fmaxf(group_max, fabsf(x[i])); + } + group_max = fmaxf(group_max, __shfl_xor_sync(FINAL_MASK, group_max, 1, 32)); + group_max = fmaxf(group_max, __shfl_xor_sync(FINAL_MASK, group_max, 2, 32)); + + uint8_t sf_byte; + float sf_rcp; + float q[kElemsPerLane]; // (x / scale) / sf_value, E2M1-encoded below + if (scale == 1.0f) { + static_assert(sizeof(scalar_t) == 2, "the unit-scale path needs bf16/fp16"); + nvfp4UnitGroupScale(group_max, sf_byte, sf_rcp); + // x / sf_value truncated from x * RN(1 / sf_value). RN(1 / s) >= 1 / s for + // every E4M3 s, so a bf16/fp16 x on an E2M1 midpoint lands exactly on it + // (and the cvt rounds it to even), while any other bf16/fp16 x is too far + // from a midpoint to reach one. No clamp either: a finite product never + // truncates to inf, and inf saturates in the cvt. + #if MINIMAX_M3_NVFP4_NATIVE_CVT + float2 const r2 = make_float2(sf_rcp, sf_rcp); + #pragma unroll + for (int i = 0; i < kElemsPerLane; i += 2) { + float2 const p = __fmul2_rz(make_float2(x[i], x[i + 1]), r2); + q[i] = p.x; + q[i + 1] = p.y; + } + #else + #pragma unroll + for (int i = 0; i < kElemsPerLane; i++) q[i] = __fmul_rz(x[i], sf_rcp); + #endif + } else { + float const sf_scale = __frcp_rn(scale); // exactly 1.0f / scale + float sf_value; + nvfp4GroupScale(sf_scale, group_max, sf_byte, sf_value, sf_rcp); + // A correctly rounded division, not a reciprocal multiply, so that exact + // E2M1 midpoints round to even. + #if MINIMAX_M3_NVFP4_NATIVE_CVT + // nvfp4Quotient on element pairs, with SM100's packed FP32 instructions. + float2 const s2 = make_float2(sf_scale, sf_scale); + float2 const neg_v2 = make_float2(-sf_value, -sf_value); + float2 const r2 = make_float2(sf_rcp, sf_rcp); + #pragma unroll + for (int i = 0; i < kElemsPerLane; i += 2) { + float2 n = __fmul2_rn(make_float2(x[i], x[i + 1]), s2); + // |n| <= 4096 as in nvfp4Quotient, NaN kept. + asm("max.NaN.f32 %0, %0, 0fC5800000;\n\t" + "min.NaN.f32 %0, %0, 0f45800000;\n\t" + "max.NaN.f32 %1, %1, 0fC5800000;\n\t" + "min.NaN.f32 %1, %1, 0f45800000;" + : "+f"(n.x), "+f"(n.y)); + float2 const q0 = __fmul2_rn(n, r2); + float2 const p = __ffma2_rn(__ffma2_rn(q0, neg_v2, n), r2, q0); + q[i] = p.x; + q[i + 1] = p.y; + } + #else + #pragma unroll + for (int i = 0; i < kElemsPerLane; i++) { + q[i] = nvfp4Quotient(x[i], sf_scale, sf_value, sf_rcp); + } + #endif + } + uint16_t packed; + #if MINIMAX_M3_NVFP4_NATIVE_CVT + // Low nibble = even element; cvt packs its first source in the high nibble. + asm("{\n" + " .reg .b8 lo, hi;\n" + " cvt.rn.satfinite.e2m1x2.f32 lo, %2, %1;\n" + " cvt.rn.satfinite.e2m1x2.f32 hi, %4, %3;\n" + " mov.b16 %0, {lo, hi};\n" + "}" + : "=h"(packed) + : "f"(q[0]), "f"(q[1]), "f"(q[2]), "f"(q[3])); + #else + uint32_t codes = 0; + #pragma unroll + for (int i = 0; i < kElemsPerLane; i++) codes |= e2m1Code(q[i]) << (4 * i); + packed = static_cast(codes); + #endif + packed = clearNegZeroNibbles(packed); + + uint8_t* const slot = page + static_cast(2 * head + kv) * + block_size * + (kNvfp4DataBytes + kNvfp4ScaleBytes); + uint8_t* const data = + slot + token * kNvfp4DataBytes + laneId * (kElemsPerLane / 2); + *reinterpret_cast(data) = packed; + + if (laneId % kNvfp4LanesPerGroup == 0) { + int const group = laneId / kNvfp4LanesPerGroup; + int offset; + if (kv == 0) { + offset = token * kNvfp4ScaleBytes + group; + } else { + constexpr int kGroupsPerQuad = kNvfp4ScaleBytes / 4; + int const swizzled_t = (token / 4) * 4 + group / kGroupsPerQuad; + int const swizzled_s = (group % kGroupsPerQuad) * 4 + token % 4; + offset = swizzled_t * kNvfp4ScaleBytes + swizzled_s; + } + slot[block_size * kNvfp4DataBytes + offset] = sf_byte; + } +} +#endif // USE_ROCM + // ──────────────────────────────────────────────────────────────────────────── // Kernel // ──────────────────────────────────────────────────────────────────────────── @@ -307,9 +563,8 @@ __device__ __forceinline__ void storeScaledQElemsFp8( // branch and writes index_q/index_k outputs. Skip-index-topk reuse layers keep // kHasIndex=true but set kProcessIndex=false. template + bool kNvfp4, typename out_idx_t, bool kHasIndex, bool kInsertKV, + bool kProcessIndex, bool kFp8Idx> __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( scalar_t* __restrict__ qkv, // [N, qkv_row] in/out (packs index if sparse) scalar_t* __restrict__ q_out, // [N, nq*128] contiguous, or nullptr @@ -332,7 +587,10 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( // The content (last) dim is always innermost-contiguous (stride 1), so the // NHD/HND layout choice is captured by the head/token strides. int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token, - int64_t const kv_s_dim) { + int64_t const kv_s_dim, + // NVFP4 main cache only: per-layer K/V dequantization (global) scales. + float const* __restrict__ kv_k_scale, + float const* __restrict__ kv_v_scale) { #if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM) // _typeConvert is unavailable on pre-Ampere; the M3 kernel only // runs with bf16/fp16 inputs in practice. Discard the bf16 body there. @@ -352,8 +610,11 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( int const idx_slots = kProcessIndex ? niq + 1 : 0; int const slots_per_token = nq + nkv + v_slots + idx_slots; - int const tokenIdx = globalWarpIdx / slots_per_token; - int const slot = globalWarpIdx % slots_per_token; + // Unsigned: both are non-negative, and the signed division's sign handling + // would cost every warp. + unsigned const warp_u = globalWarpIdx, slots_u = slots_per_token; + int const tokenIdx = warp_u / slots_u; + int const slot = warp_u % slots_u; if (tokenIdx >= num_tokens) return; // Slot boundaries. @@ -420,10 +681,13 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( // flat TMA descriptor can address them as [tokens*heads, head_dim]; this // folds the de-interleaving into the store the kernel already does, instead // of a separate q.contiguous() copy. Everything else stays in place. + // Given q_fp8_out but no q_out, q is emitted only in fp8 (null store_ptr). scalar_t* store_ptr = row_ptr; if (isQ && q_out != nullptr) { store_ptr = q_out + static_cast(tokenIdx) * nq * kHeadDim + slot * kHeadDim; + } else if (isQ && q_fp8_out != nullptr) { + store_ptr = nullptr; } else if (isIQ && index_q_out != nullptr) { // bf16 index_q_out: gather here. fp8: written by the explicit fp8 store. if constexpr (!kFp8Idx) { @@ -433,6 +697,13 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( } } + // NVFP4: the per-layer global scale is not produced by the predecessor, + // so load it before the PDL wait. + float nvfp4_scale = 0.0f; + if constexpr (kNvfp4) { + if (isK || isV) nvfp4_scale = isK ? *kv_k_scale : *kv_v_scale; + } + // PDL: wait for the predecessor kernel (the qkv-projection GEMM that // produces ``qkv``) to finish before touching any global memory. No-op // when PDL is not enabled on the launch. The CUDA runtime wrapper emits @@ -451,16 +722,17 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( normAndRope(elems, laneId, eps, norm_w, do_rope, rotary_dim, cos_ptr, /*apply_norm=*/norm_w != nullptr); if constexpr (kFp8Idx) { - // index_q is e4m3 bytes; Q/K (and in-place index_k) stay scalar_t. + // index_q is E4M3(scalar_t(q)) bytes; Q/K (and in-place index_k) stay + // scalar_t. if (isIQ && index_q_out != nullptr) { - storeElemsFp8(index_q_out + - static_cast(tokenIdx) * niq * kHeadDim + - (slot - iq_begin) * kHeadDim + dim_base, - elems); - } else { + storeIndexQElemsFp8( + index_q_out + static_cast(tokenIdx) * niq * kHeadDim + + (slot - iq_begin) * kHeadDim + dim_base, + elems); + } else if (store_ptr != nullptr) { storeElems(store_ptr + dim_base, elems); } - } else { + } else if (store_ptr != nullptr) { storeElems(store_ptr + dim_base, elems); } if (isQ && q_fp8_out != nullptr) { @@ -487,6 +759,16 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( } else { storeElems(index_cache + sm * kHeadDim + dim_base, elems); } + } else if constexpr (kNvfp4) { +#ifndef USE_ROCM + // kv_cache is [num_blocks, 2*nkv, block_size, 72] bytes (HND), slot + // 2*head + side; kv_s_block is its page stride. + storeNvfp4CacheElems( + reinterpret_cast(kv_cache) + + (sm / block_size) * kv_s_block, + isK ? 0 : 1, head, static_cast(sm % block_size), block_size, + laneId, elems, nvfp4_scale); +#endif } else if (isK || isV) { // kv_cache logical shape [num_blocks, nkv, block_size, 2*head_dim]. // Paging is logical (block = sm/block_size, token = sm%block_size); @@ -515,7 +797,8 @@ __global__ void fusedMiniMaxM3QNormRopeKVInsertKernel( // ──────────────────────────────────────────────────────────────────────────── // Launch wrapper // ──────────────────────────────────────────────────────────────────────────── -template +template void launchFusedMiniMaxM3( scalar_t* qkv, scalar_t* q_out, uint8_t* q_fp8_out, void* index_q_out, scalar_t const* q_norm_w, scalar_t const* k_norm_w, @@ -527,7 +810,8 @@ void launchFusedMiniMaxM3( int const nq, int const nkv, int const niq, int const block_size, int64_t const kv_s_block, int64_t const kv_s_head, int64_t const kv_s_token, int64_t const kv_s_dim, bool const has_index, bool const insert_kv, - bool const process_index, bool const fp8_idx, cudaStream_t stream) { + bool const process_index, bool const fp8_idx, float const* kv_k_scale, + float const* kv_v_scale, cudaStream_t stream) { // Index outputs are scalar_t (bf16) or e4m3 bytes (uint8_t); reinterpret the // void* pointers per instantiation in the LAUNCH macro. // Slot count must match the kernel's compile-time gating. @@ -559,35 +843,44 @@ void launchFusedMiniMaxM3( config.attrs = attrs; config.numAttrs = (sm_version >= 90) ? 1 : 0; - #define LAUNCH(HAS_INDEX, INSERT, PROCESS_INDEX, FP8, OUT_T) \ - cudaLaunchKernelEx( \ - &config, \ - fusedMiniMaxM3QNormRopeKVInsertKernel, \ - qkv, q_out, q_fp8_out, reinterpret_cast(index_q_out), \ - q_norm_w, k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \ - slot_mapping, index_slot_mapping, kv_cache, \ - reinterpret_cast(index_cache), eps, q_fp8_inv_scale, \ - rotary_dim, num_tokens, nq, nkv, niq, block_size, kv_s_block, \ - kv_s_head, kv_s_token, kv_s_dim) -#else - // ROCm: standard kernel launch syntax (no PDL/stream serialization). - // clang-format off #define LAUNCH(HAS_INDEX, INSERT, PROCESS_INDEX, FP8, OUT_T) \ - fusedMiniMaxM3QNormRopeKVInsertKernel< \ - scalar_t, cache_t, kv_dt, OUT_T, HAS_INDEX, INSERT, PROCESS_INDEX, \ - FP8><<>>( \ + cudaLaunchKernelEx( \ + &config, \ + fusedMiniMaxM3QNormRopeKVInsertKernel, \ qkv, q_out, q_fp8_out, reinterpret_cast(index_q_out), \ q_norm_w, k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \ slot_mapping, index_slot_mapping, kv_cache, \ reinterpret_cast(index_cache), eps, q_fp8_inv_scale, \ rotary_dim, num_tokens, nq, nkv, niq, block_size, kv_s_block, \ - kv_s_head, kv_s_token, kv_s_dim) + kv_s_head, kv_s_token, kv_s_dim, kv_k_scale, kv_v_scale) +#else + // ROCm: standard kernel launch syntax (no PDL/stream serialization). + // clang-format off + #define LAUNCH(HAS_INDEX, INSERT, PROCESS_INDEX, FP8, OUT_T) \ + fusedMiniMaxM3QNormRopeKVInsertKernel< \ + scalar_t, cache_t, kv_dt, kNvfp4, OUT_T, HAS_INDEX, INSERT, \ + PROCESS_INDEX, FP8><<>>( \ + qkv, q_out, q_fp8_out, reinterpret_cast(index_q_out), \ + q_norm_w, k_norm_w, iq_norm_w, ik_norm_w, cos_sin_cache, positions, \ + slot_mapping, index_slot_mapping, kv_cache, \ + reinterpret_cast(index_cache), eps, q_fp8_inv_scale, \ + rotary_dim, num_tokens, nq, nkv, niq, block_size, kv_s_block, \ + kv_s_head, kv_s_token, kv_s_dim, kv_k_scale, kv_v_scale) // clang-format on #endif - if (has_index) { + if constexpr (kNvfp4) { + // NVFP4 main cache: sparse serving only (the host checks insert_kv). + if (!process_index) { + LAUNCH(true, true, false, false, scalar_t); + } else if (fp8_idx) { + LAUNCH(true, true, true, true, uint8_t); + } else { + LAUNCH(true, true, true, false, scalar_t); + } + } else if (has_index) { if (!process_index) { if (insert_kv) { LAUNCH(true, true, false, false, scalar_t); @@ -621,8 +914,9 @@ void launchFusedMiniMaxM3( } // namespace vllm // clang-format off -#define CALL_FUSED_MINIMAX_M3(_RAW_T, CACHE_T, KV_DTYPE) \ - vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3( \ +#define CALL_FUSED_MINIMAX_M3_IMPL(CACHE_T, KV_DTYPE, NVFP4) \ + vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3( \ reinterpret_cast(qkv.data_ptr()), \ q_out.has_value() ? reinterpret_cast(q_out->data_ptr()) : nullptr, \ q_fp8_out.has_value() \ @@ -654,7 +948,10 @@ void launchFusedMiniMaxM3( static_cast(eps), 1.0f / static_cast(q_fp8_scale), \ static_cast(rotary_dim), num_tokens, nq, nkv, niq, \ static_cast(block_size), kv_s_block, kv_s_head, kv_s_token, \ - kv_s_dim, has_index, insert_kv, process_index, fp8_idx, stream) + kv_s_dim, has_index, insert_kv, process_index, fp8_idx, kv_k_scale_ptr, \ + kv_v_scale_ptr, stream) +#define CALL_FUSED_MINIMAX_M3(_RAW_T, CACHE_T, KV_DTYPE) \ + CALL_FUSED_MINIMAX_M3_IMPL(CACHE_T, KV_DTYPE, false) // clang-format on // ──────────────────────────────────────────────────────────────────────────── @@ -681,7 +978,9 @@ void fused_minimax_m3_qknorm_rope_kv_insert( const std::string& kv_cache_dtype, bool skip_index_branch, std::optional q_fp8_out, // [N, nq*128] contiguous E4M3 - double q_fp8_scale) { + double q_fp8_scale, + std::optional kv_k_scale, // [] f32, NVFP4 only + std::optional kv_v_scale) { STD_TORCH_CHECK(qkv.is_cuda() && qkv.is_contiguous(), "qkv must be contiguous CUDA"); STD_TORCH_CHECK( @@ -721,8 +1020,10 @@ void fused_minimax_m3_qknorm_rope_kv_insert( bool const has_index = niq > 0; bool const insert_kv = kv_cache.has_value(); bool const process_index = has_index && !skip_index_branch; + bool const nvfp4_kv = kv_cache_dtype == "nvfp4"; vllm::Fp8KVCacheDataType const kv_dt = - vllm::get_fp8_kv_cache_data_type(kv_cache_dtype); + nvfp4_kv ? vllm::Fp8KVCacheDataType::kAuto + : vllm::get_fp8_kv_cache_data_type(kv_cache_dtype); int const kHeadDim = vllm::minimax_m3_fused_ops::kHeadDim; int const expected_row = (nq + 2 * nkv + (has_index ? niq + 1 : 0)) * kHeadDim; @@ -769,8 +1070,13 @@ void fused_minimax_m3_qknorm_rope_kv_insert( index_slot_mapping->numel() == slot_mapping->numel()), "index_slot_mapping must be int64 CUDA with slot_mapping length"); } - // Main attention KV cache: auto matches qkv, fp8 uses uint8 storage. - if (kv_dt == vllm::Fp8KVCacheDataType::kAuto) { + // Main attention KV cache: auto matches qkv, fp8 and nvfp4 use uint8 + // storage. + if (nvfp4_kv) { + STD_TORCH_CHECK( + kv_cache->scalar_type() == torch::headeronly::ScalarType::Byte, + "nvfp4 kv_cache must use uint8 storage"); + } else if (kv_dt == vllm::Fp8KVCacheDataType::kAuto) { STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(), "auto kv_cache dtype must match qkv"); } else { @@ -790,6 +1096,27 @@ void fused_minimax_m3_qknorm_rope_kv_insert( STD_TORCH_CHECK(kv_cache->dim() == 4 && kv_cache->stride(3) == 1, "kv_cache must be [nb,nkv,bs,2*head_dim] with contiguous " "content dim (stride(3)==1)"); + if (nvfp4_kv) { + // Packed HND pages [nb, 2*nkv, bs, 72]: slot 2*head + side is one + // contiguous [bs, 72] block, which the store addresses directly. + int64_t const full_dim = kHeadDim / 2 + kHeadDim / 16; + STD_TORCH_CHECK( + kv_cache->size(1) == 2 * nkv && kv_cache->size(2) == block_size && + kv_cache->size(3) == full_dim && + kv_cache->stride(2) == full_dim && + kv_cache->stride(1) == block_size * full_dim && + block_size % 4 == 0, + "nvfp4 kv_cache must be HND [nb, 2*num_kv_heads, block_size, 72] " + "with contiguous pages and block_size % 4 == 0"); + for (auto const* scale : {&kv_k_scale, &kv_v_scale}) { + STD_TORCH_CHECK( + scale->has_value() && (*scale)->is_cuda() && + (*scale)->scalar_type() == + torch::headeronly::ScalarType::Float && + (*scale)->numel() >= 1, + "nvfp4 kv_cache requires float32 CUDA kv_k_scale and kv_v_scale"); + } + } kv_s_block = kv_cache->stride(0); kv_s_head = kv_cache->stride(1); kv_s_token = kv_cache->stride(2); @@ -854,10 +1181,31 @@ void fused_minimax_m3_qknorm_rope_kv_insert( "fp8 index path: index_q_out must be fp8 e4m3"); } + STD_TORCH_CHECK(!nvfp4_kv || insert_kv, + "nvfp4 kv_cache_dtype requires kv_cache (insert mode)"); + float const* kv_k_scale_ptr = + nvfp4_kv ? kv_k_scale->const_data_ptr() : nullptr; + float const* kv_v_scale_ptr = + nvfp4_kv ? kv_v_scale->const_data_ptr() : nullptr; + const torch::stable::accelerator::DeviceGuard device_guard( qkv.get_device_index()); auto stream = get_current_cuda_stream(qkv.get_device_index()); + if (nvfp4_kv) { +#ifndef USE_ROCM + VLLM_STABLE_DISPATCH_HALF_TYPES( + qkv.scalar_type(), "fused_minimax_m3_qknorm_rope_kv_insert_nvfp4", [&] { + using st = scalar_t; + CALL_FUSED_MINIMAX_M3_IMPL(uint8_t, vllm::Fp8KVCacheDataType::kAuto, + true); + }); +#else + STD_TORCH_CHECK(false, "nvfp4 kv_cache is not supported on ROCm"); +#endif + return; + } + VLLM_STABLE_DISPATCH_HALF_TYPES( qkv.scalar_type(), "fused_minimax_m3_qknorm_rope_kv_insert", [&] { using st = scalar_t; @@ -867,3 +1215,4 @@ void fused_minimax_m3_qknorm_rope_kv_insert( } #undef CALL_FUSED_MINIMAX_M3 +#undef CALL_FUSED_MINIMAX_M3_IMPL diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index 6d45f99881..3266d28b1e 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -381,7 +381,9 @@ void fused_minimax_m3_qknorm_rope_kv_insert( std::optional q_out, std::optional index_q_out, const std::string& kv_cache_dtype, bool skip_index_branch, - std::optional q_fp8_out, double q_fp8_scale); + std::optional q_fp8_out, double q_fp8_scale, + std::optional kv_k_scale, + std::optional kv_v_scale); #ifdef VLLM_ENABLE_FUSED_KDA_DECODE void fused_kda_decode( diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 98e89c39d3..3d4f9ab273 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -513,7 +513,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { "Tensor!? kv_cache, Tensor!? index_cache, " "int block_size, Tensor!? q_out, Tensor!? index_q_out, " "str kv_cache_dtype, bool skip_index_branch=False, " - "Tensor!? q_fp8_out=None, float q_fp8_scale=1.0) -> ()"); + "Tensor!? q_fp8_out=None, float q_fp8_scale=1.0, " + "Tensor? kv_k_scale=None, Tensor? kv_v_scale=None) -> ()"); #ifdef VLLM_ENABLE_FUSED_KDA_DECODE ops.def( diff --git a/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py b/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py index 5c94968ddc..0007f20c5b 100644 --- a/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py +++ b/tests/kernels/attention/test_minimax_m3_msa_cutlass_sparse_decode.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Correctness tests for MiniMax M3 CUTLASS sparse decode.""" +"""Correctness tests for MiniMax M3 MSA sparse attention (CUTLASS decode and +NVFP4 KV cache).""" import math from types import SimpleNamespace @@ -8,6 +9,7 @@ from types import SimpleNamespace import pytest import torch +from tests.kernels.quantization.nvfp4_utils import dequant_nvfp4_kv_cache from vllm import _custom_ops as ops from vllm.config import AttentionConfig, CUDAGraphMode from vllm.forward_context import ForwardContext, override_forward_context @@ -18,6 +20,7 @@ from vllm.models.minimax_m3.common.sparse_attention import ( MiniMaxM3SparseBackend, MiniMaxM3SparseMetadata, MiniMaxM3SparseMetadataBuilder, + MiniMaxM3SparsePrefillMetadata, MiniMaxM3SparseTritonImpl, select_main_backend_and_impl_cls, ) @@ -28,6 +31,7 @@ from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import ( MSACutlassDecodePlanCache, msa_cutlass_sparse_decode, + nvfp4_kv_cache_views, prepare_decode_metadata, should_prepare_decode_metadata, ) @@ -36,6 +40,7 @@ from vllm.models.minimax_m3.nvidia.sparse_attention_msa import ( MiniMaxM3SparseMSADecodeMetadata, MiniMaxM3SparseMSAImpl, MiniMaxM3SparseMSAMetadataBuilder, + MiniMaxM3SparseMSANvfp4Backend, ) from vllm.platforms import current_platform from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -164,6 +169,53 @@ def test_msa_cutlass_decode_static_dispatch_requires_sm100( ) +@pytest.mark.parametrize("decode_backend", ["cutlass", "triton"]) +def test_msa_cutlass_decode_static_dispatch_nvfp4_has_no_fallback( + decode_backend: str, +) -> None: + """NVFP4 has no Triton decode, so it plans CUTLASS for any batch size.""" + assert should_prepare_decode_metadata( + 1, + DEFAULT_QUERY_LEN, + decode_backend=decode_backend, # type: ignore[arg-type] + num_q_heads=16, + num_kv_heads=1, + kv_cache_dtype="nvfp4", + page_size=BLOCK_SIZE, + topk_blocks=TOPK, + ) + + +def test_nvfp4_selects_msa_nvfp4_backend_and_requires_sm100( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend_cls, impl_cls = select_main_backend_and_impl_cls( + topk_blocks=TOPK, + kv_cache_dtype="nvfp4", + num_kv_heads=1, + ) + assert backend_cls is MiniMaxM3SparseMSANvfp4Backend + assert impl_cls is MiniMaxM3SparseMSAImpl + with pytest.raises(ValueError, match="nvfp4_4over6"): + select_main_backend_and_impl_cls( + topk_blocks=TOPK, + kv_cache_dtype="nvfp4_4over6", + num_kv_heads=1, + ) + + monkeypatch.setattr( + current_platform, + "is_device_capability_family", + lambda _: False, + ) + with pytest.raises(ValueError, match="NVFP4"): + select_main_backend_and_impl_cls( + topk_blocks=TOPK, + kv_cache_dtype="nvfp4", + num_kv_heads=1, + ) + + @pytest.mark.parametrize( ("backend", "expected"), [ @@ -639,3 +691,391 @@ def test_msa_cutlass_decode_matches_triton_with_interleaved_cache( graph.replay() current_platform.synchronize() torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02) + + +def _make_paged_layout( + seq_lens_list: list[int], +) -> tuple[torch.Tensor, int]: + """Block table over a shuffled physical page pool.""" + pages_per_request = [math.ceil(seq_len / BLOCK_SIZE) for seq_len in seq_lens_list] + num_pages = sum(pages_per_request) + block_table = torch.zeros( + len(seq_lens_list), max(pages_per_request), dtype=torch.int32, device="cuda" + ) + physical_pages = torch.randperm(num_pages, dtype=torch.int32, device="cuda") + offset = 0 + for request, request_pages in enumerate(pages_per_request): + block_table[request, :request_pages] = physical_pages[ + offset : offset + request_pages + ] + offset += request_pages + return block_table, num_pages + + +def _make_nvfp4_kv_cache( + num_pages: int, + num_kv_heads: int, + kv_cache_dtype: str, + k_scale: torch.Tensor, + v_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Write random K/V through the NVFP4 ``reshape_and_cache_flash`` writer, + whose pages hold every head's data then every head's scales per side, and + repack them into head slots. + + Returns the packed HND cache ``[pages, 2 * Hkv, 128, 72]`` and its BF16 + dequantization ``[pages, Hkv, 128, 2 * 128]`` for the Triton reference. + """ + num_slots = num_pages * BLOCK_SIZE + key = torch.randn( + num_slots, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda" + ) + value = torch.randn_like(key) + written = torch.zeros( + num_pages, + 2 * num_kv_heads, + BLOCK_SIZE, + HEAD_DIM // 2 + HEAD_DIM // 16, + dtype=torch.uint8, + device="cuda", + ) + k_cache, v_cache = written.transpose(1, 2).split(num_kv_heads, dim=-2) + ops.reshape_and_cache_flash( + key, + value, + k_cache, + v_cache, + torch.arange(num_slots, dtype=torch.int64, device="cuda"), + kv_cache_dtype, + k_scale, + v_scale, + ) + + kv_cache = torch.empty_like(written) + head_slot_views = iter(nvfp4_kv_cache_views(kv_cache)) + dequantized = [] + for side, global_scale, swizzled in ( + (written[:, :num_kv_heads], k_scale, False), + (written[:, num_kv_heads:], v_scale, True), + ): + data = side.as_strided( + (num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2), + (side.stride(0), BLOCK_SIZE * HEAD_DIM // 2, HEAD_DIM // 2, 1), + ) + scale = side.as_strided( + (num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 16), + (side.stride(0), BLOCK_SIZE * HEAD_DIM // 16, HEAD_DIM // 16, 1), + side.storage_offset() + num_kv_heads * BLOCK_SIZE * HEAD_DIM // 2, + ) + next(head_slot_views).copy_(data) + next(head_slot_views).copy_(scale) + dequantized.append( + dequant_nvfp4_kv_cache( + data, + scale, + global_scale.item(), + HEAD_DIM, + BLOCK_SIZE, + swizzled_scales=swizzled, + ) + ) + return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16) + + +def test_nvfp4_kv_cache_views_address_head_slots() -> None: + """Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head + is one contiguous run of the page: a TP rank's page is a byte slice.""" + num_kv_heads, slot_bytes = 4, BLOCK_SIZE * (HEAD_DIM // 2 + HEAD_DIM // 16) + kv_cache = torch.randint( + 0, + 256, + (3, 2 * num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2 + HEAD_DIM // 16), + dtype=torch.uint8, + ) + k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache) + runs = kv_cache.view(3, num_kv_heads, 2 * slot_bytes) + for head in range(num_kv_heads): + expected = torch.cat( + [v[:, head].flatten(1) for v in (k_data, k_sf, v_data, v_sf)], dim=1 + ) + assert torch.equal(runs[:, head], expected) + + +@pytest.mark.parametrize( + ("num_q_heads", "num_kv_heads", "num_request_pairs", "query_len", "capture"), + [ + pytest.param(64, 4, 1, 1, True, id="tp1-small-batch-query-len-1"), + pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"), + pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"), + ], +) +def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache( + num_q_heads: int, + num_kv_heads: int, + num_request_pairs: int, + query_len: int, + capture: bool, +) -> None: + """CUTLASS decode reads vLLM's packed NVFP4 pages, including swizzled V + scales and device global scales, as their exact dequantized values.""" + torch.manual_seed(0) + seq_lens_list = [257, 513] * num_request_pairs + seq_lens_cpu = torch.tensor(seq_lens_list, dtype=torch.int32) + seq_lens = seq_lens_cpu.cuda() + block_table, num_pages = _make_paged_layout(seq_lens_list) + k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda") + v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda") + kv_cache, kv_reference = _make_nvfp4_kv_cache( + num_pages, num_kv_heads, "nvfp4", k_scale, v_scale + ) + + num_query_tokens = len(seq_lens_list) * query_len + query = torch.randn( + num_query_tokens, num_q_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda" + ) + query_fp8 = query.to(torch.float8_e4m3fn) + topk = _make_topk(seq_lens_list, num_kv_heads, query_len) + expected = torch.empty_like(query) + minimax_m3_sparse_attn_decode( + query_fp8.to(torch.bfloat16), + kv_reference, + topk.transpose(0, 1), + block_table, + seq_lens, + num_kv_heads, + SM_SCALE, + expected, + query_len, + ) + + assert should_prepare_decode_metadata( + len(seq_lens_list), + query_len, + decode_backend="triton", + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + kv_cache_dtype="nvfp4", + page_size=BLOCK_SIZE, + topk_blocks=TOPK, + ) + metadata = prepare_decode_metadata( + block_table, + seq_lens, + seq_lens_cpu, + query_len, + num_q_heads=num_q_heads, + num_kv_heads=num_kv_heads, + page_size=BLOCK_SIZE, + topk_blocks=TOPK, + ) + actual = torch.zeros_like(query) + + def run() -> None: + msa_cutlass_sparse_decode( + query_fp8, + kv_cache, + topk, + actual, + metadata, + scale=SM_SCALE, + q_scale_float=1.0, + k_scale_float=1.0, + v_scale_float=1.0, + k_scale=k_scale, + v_scale=v_scale, + ) + + if capture: + run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + actual.zero_() + graph.replay() + else: + run() + current_platform.synchronize() + torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05) + + +@pytest.mark.parametrize( + ("num_q_heads", "num_kv_heads"), + [pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")], +) +def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache( + num_q_heads: int, + num_kv_heads: int, +) -> None: + """MSA prefill reads NVFP4 pages as their exact dequantized values, + including chunked prefill over cached context, and takes q from its fp8 + copy (the only one the fused insert writes on this path).""" + torch.manual_seed(0) + q_lens = [300, 128, 77] + seq_lens_list = [300, 640, 1100] + block_table, num_pages = _make_paged_layout(seq_lens_list) + k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda") + v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda") + kv_cache, kv_reference = _make_nvfp4_kv_cache( + num_pages, num_kv_heads, "nvfp4", k_scale, v_scale + ) + + total_q = sum(q_lens) + seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda") + context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda") + cu_seqlens_q = torch.zeros(len(q_lens) + 1, dtype=torch.int32, device="cuda") + cu_seqlens_q[1:] = torch.tensor(q_lens, device="cuda").cumsum(0) + cu_seqlens_k = torch.zeros_like(cu_seqlens_q) + cu_seqlens_k[1:] = seq_lens.cumsum(0) + prefill = MiniMaxM3SparsePrefillMetadata( + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + seq_lens=seq_lens, + context_lens=context_lens, + block_table=block_table, + max_query_len=max(q_lens), + max_seq_len=max(seq_lens_list), + total_kv_blocks=sum(math.ceil(n / BLOCK_SIZE) for n in seq_lens_list), + ) + metadata = MiniMaxM3SparseMetadata( + seq_lens=seq_lens, + max_seq_len=max(seq_lens_list), + slot_mapping=torch.empty(total_q, dtype=torch.int64, device="cuda"), + num_actual_tokens=total_q, + num_decodes=0, + num_decode_tokens=0, + num_prefills=len(q_lens), + num_prefill_tokens=total_q, + prefill=prefill, + ) + + # Every causally visible page (<= TOPK per query), ending with the local one. + topk = torch.full( + (total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda" + ) + token = 0 + for q_len, seq_len in zip(q_lens, seq_lens_list): + for local_q in range(q_len): + visible = (seq_len - q_len + local_q) // BLOCK_SIZE + 1 + assert visible <= TOPK + topk[token, :, :visible] = torch.arange(visible, device="cuda") + token += 1 + + layer_name = "model.layers.0.self_attn.attn" + q_scale = 0.5 + layer = SimpleNamespace( + layer_name=layer_name, + topk_indices_buffer=topk, + _q_scale_float=q_scale, + _k_scale=k_scale, + _v_scale=v_scale, + ) + query_fp8 = ( + torch.randn( + total_q, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda" + ) + / q_scale + ).to(torch.float8_e4m3fn) + query = query_fp8.to(torch.bfloat16) * q_scale + common = dict( + num_heads=num_q_heads, + head_size=HEAD_DIM, + scale=SM_SCALE, + num_kv_heads=num_kv_heads, + topk_blocks=TOPK, + sparse_block_size=BLOCK_SIZE, + ) + forward_context = ForwardContext( + no_compile_layers={}, + attn_metadata={layer_name: metadata}, + slot_mapping={}, + ) + with override_forward_context(forward_context): + expected = MiniMaxM3SparseTritonImpl(**common).forward( + layer, query, kv_reference, torch.empty_like(query) + ) + actual = MiniMaxM3SparseMSAImpl(**common, kv_cache_dtype="nvfp4").forward( + layer, + torch.full_like(query, float("nan")), + kv_cache, + torch.zeros_like(query), + query_fp8=query_fp8, + ) + current_platform.synchronize() + torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05) + + +def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None: + """With CUTLASS decode on but not planned for the batch, the Triton decode + fallback attends with the dequantized fp8 q, the only q written.""" + torch.manual_seed(0) + num_q_heads, num_kv_heads, q_scale = 16, 1, 0.5 + seq_lens_list = [257, 513, 129] + seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda") + block_table, num_pages = _make_paged_layout(seq_lens_list) + kv_cache = ( + torch.randn( + num_pages, + num_kv_heads, + BLOCK_SIZE, + 2 * HEAD_DIM, + dtype=torch.bfloat16, + device="cuda", + ) + * 0.25 + ).to(torch.float8_e4m3fn) + num_tokens = len(seq_lens_list) + metadata = MiniMaxM3SparseMetadata( + seq_lens=seq_lens, + max_seq_len=max(seq_lens_list), + slot_mapping=torch.empty(num_tokens, dtype=torch.int64, device="cuda"), + num_actual_tokens=num_tokens, + num_decodes=num_tokens, + num_decode_tokens=num_tokens, + num_prefills=0, + num_prefill_tokens=0, + decode=MiniMaxM3SparseMSADecodeMetadata( + seq_lens=seq_lens, + block_table=block_table, + decode_query_len=1, + msa_cutlass=None, + ), + ) + layer_name = "model.layers.0.self_attn.attn" + layer = SimpleNamespace( + layer_name=layer_name, + topk_indices_buffer=_make_topk(seq_lens_list, num_kv_heads, 1), + _q_scale_float=q_scale, + ) + query_fp8 = torch.randn( + num_tokens, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda" + ).to(torch.float8_e4m3fn) + query = query_fp8.to(torch.bfloat16) * q_scale + common = dict( + num_heads=num_q_heads, + head_size=HEAD_DIM, + scale=SM_SCALE, + num_kv_heads=num_kv_heads, + kv_cache_dtype="fp8", + topk_blocks=TOPK, + sparse_block_size=BLOCK_SIZE, + ) + impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass") + assert impl.use_cutlass_decode + forward_context = ForwardContext( + no_compile_layers={}, + attn_metadata={layer_name: metadata}, + slot_mapping={}, + ) + with override_forward_context(forward_context): + expected = MiniMaxM3SparseTritonImpl(**common).forward( + layer, query, kv_cache, torch.empty_like(query) + ) + actual = impl.forward( + layer, + torch.full_like(query, float("nan")), + kv_cache, + torch.zeros_like(query), + query_fp8=query_fp8, + ) + current_platform.synchronize() + torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02) diff --git a/tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py b/tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py index 0bc3c5692a..16a9be307b 100644 --- a/tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py +++ b/tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py @@ -320,6 +320,38 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype): index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0 ) + # Without q_out, q is emitted only in fp8: the same bytes, while qkv keeps + # its raw q and every other output matches the run above. + outputs = [qkv_orig.clone(), torch.empty_like(q_fp8), torch.empty_like(index_q)] + caches = [torch.zeros_like(kv_cache), torch.zeros_like(index_cache)] + ops.fused_minimax_m3_qknorm_rope_kv_insert( + outputs[0], + q_w, + k_w, + cos_sin, + positions, + num_heads, + num_kv_heads, + ROTARY_DIM, + eps, + iq_w, + ik_w, + num_idx_heads, + slot_mapping, + index_slot_mapping, + *caches, + block_size, + None, + outputs[2], + kv_cache_dtype, + q_fp8_out=outputs[1], + q_fp8_scale=0.5, + ) + for actual, expected in zip( + outputs + caches, [qkv, q_fp8, index_q, kv_cache, index_cache] + ): + assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8)) + @pytest.mark.parametrize("num_tokens", [1, 64, 513]) @pytest.mark.parametrize("block_size", [16, 64]) @@ -441,8 +473,8 @@ def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype): # ── Test 4: fp8 (e4m3) index outputs ───────────────────────────────────────── # The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v + # q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run -# (the index dtype must not perturb the main branch), and (2) the e4m3 index -# outputs dequantize close to the bf16 reference. +# (the index dtype must not perturb the main branch), and (2) index_q is the +# e4m3 cast of the bf16 run's index_q, and the index-K cache is within fp8 ulp. @pytest.mark.skipif( @@ -528,6 +560,196 @@ def test_sparse_full_fp8_index(num_tokens, block_size): torch.testing.assert_close(qkv_fp, qkv_bf, rtol=0, atol=0) torch.testing.assert_close(kvc_fp, kvc_bf, rtol=0, atol=0) - # (2) Dequantized e4m3 index outputs match the bf16 reference within fp8 ulp. - torch.testing.assert_close(iq_fp.float(), iq_bf.float(), rtol=0.13, atol=0.05) + # (2) index_q is e4m3(bf16(q)); the index-K cache is within fp8 ulp. + assert torch.equal( + iq_fp.view(torch.uint8), iq_bf.to(torch.float8_e4m3fn).view(torch.uint8) + ) torch.testing.assert_close(idxc_fp.float(), idxc_bf.float(), rtol=0.13, atol=0.05) + + +# ── Test 5: sparse mode with an NVFP4 main KV cache ────────────────────────── +# The NVFP4 inserts quantize x * sf_scale, where sf_scale = 1 / the global +# scale: per 16-value block, +# scale = min(448, max(E4M3(max(amax, 1e-12) / 6), 1/512)) and +# code = sign | RNE E2M1 of min(|x / scale|, 6), with zero stored as +0. + +E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0) + + +def _div(a, b): + # Division by a Python scalar would become a reciprocal multiply. + return a / torch.as_tensor(b, dtype=a.dtype, device=a.device) + + +def nvfp4_ref(x, sf_scale): + """Returns the E2M1 codes [..., 128] and E4M3 scale bytes [..., 8] of x.""" + blocks = x.float().unflatten(-1, (-1, 16)) + amax = blocks.abs().amax(-1, keepdim=True).clamp_min(1e-12) + scale = (sf_scale * _div(amax, 6.0)).clamp(max=448.0) + scale = scale.to(torch.float8_e4m3fn).float().clamp(min=2.0**-9) + q = _div(blocks * sf_scale, scale) + # Alternating > / >= rounds exact midpoints to the even code. + mag = sum( + (q.abs() >= m) if i % 2 else (q.abs() > m) for i, m in enumerate(E2M1_MIDPOINTS) + ).to(torch.uint8) + codes = torch.where(mag > 0, mag | (torch.signbit(q).to(torch.uint8) << 3), 0) + scale_bytes = scale.squeeze(-1).to(torch.float8_e4m3fn).view(torch.uint8) + return codes.flatten(-2), scale_bytes + + +def nvfp4_pages(sides, slot_mapping, num_blocks, block_size): + """Expected NVFP4 pages [nb, len(sides) * nkv, bs, 72] for the K (then V) + side given as (x [N, nkv, 128], sf_scale). Slot h * len(sides) + side is + head h's packed E2M1 data [bs, 64] (low nibble = even element), then its + E4M3 scales [bs, 8]: linear for K, 4x4 token-quad swizzled for V.""" + nkv, ns, device = sides[0][0].shape[1], HEAD_DIM // 16, slot_mapping.device + row_bytes, data_bytes = HEAD_DIM // 2 + ns, block_size * HEAD_DIM // 2 + pages = torch.zeros( + num_blocks, + nkv, + len(sides), + block_size * row_bytes, + dtype=torch.uint8, + device=device, + ) + valid = slot_mapping >= 0 + blk, tok = slot_mapping[valid] // block_size, slot_mapping[valid] % block_size + g, t = torch.arange(ns, device=device), tok[:, None] + scale_offsets = (t * ns + g, ((t // 4) * 4 + g // 2) * ns + (g % 2) * 4 + t % 4) + for side, (x, sf_scale) in enumerate(sides): + codes, scales = nvfp4_ref(x[valid], sf_scale) + data = pages[:, :, side, :data_bytes].view(num_blocks, nkv, block_size, -1) + data[blk, :, tok] = codes[..., 0::2] | (codes[..., 1::2] << 4) + scale_region = pages[:, :, side, data_bytes:] + for h in range(nkv): + scale_region[blk[:, None], h, scale_offsets[side]] = scales[:, h] + return pages.view(num_blocks, -1, block_size, row_bytes) + + +def nvfp4_edge_values(num_rows, global_scale): + """[num_rows, 128] bf16 whose 16-value blocks, divided by global_scale, hit + the quantizer's corner cases: exact E2M1 midpoints under non-power-of-2 scales + (blocks 0-1), amax under the 1/512 scale floor (2), all -0 (3), and + saturation with |x| > 4096 up to bf16 max (4) or with inf (5).""" + g = torch.Generator().manual_seed(0) + x = torch.randn(num_rows, 8, 16, generator=g) + sign = torch.where(torch.rand(x.shape, generator=g) < 0.5, -1.0, 1.0) + steps = torch.tensor(E2M1_MIDPOINTS + (0.5, 1.0, 3.0)) + mant = 1 + torch.randint(1, 8, (num_rows, 2, 1), generator=g) / 8 + scale = mant * 2.0 ** torch.randint(-6, 3, (num_rows, 2, 1), generator=g) + x[:, :2] = steps[torch.randint(len(steps), (num_rows, 2, 16), generator=g)] * scale + x[:, :2, :1] = 6 * scale # amax = 6 * scale, so the block scale is `scale` + x[:, 2] = torch.rand(num_rows, 16, generator=g) * 2.0**-8 + x[:, 4] *= 5000 + x = x * sign * global_scale + x[:, 3] = -0.0 + x[:, 4, 0] = sign[:, 4, 0] * torch.finfo(torch.bfloat16).max + x[:, 5, 0] = sign[:, 5, 0] * float("inf") + return x.flatten(1).to(torch.bfloat16) + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10, + reason="NVFP4 KV caches need SM100", +) +@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513]) +@pytest.mark.parametrize("num_kv_heads", [1, 4]) +@pytest.mark.parametrize("block_size", [16, 128]) +@pytest.mark.parametrize("kv_scales", [(0.5, 2.0), (1.0, 1.0)]) +def test_sparse_full_nvfp4(num_tokens, num_kv_heads, block_size, kv_scales): + """The fused NVFP4 insert writes the reference quantizer's bytes (packed + E2M1, linear K / swizzled V E4M3 block scales) for the kernel's own + post-RoPE k and for a raw v that hits the quantizer's corner cases, and + skips padded slots. Unit global scales take the kernel's separate fast + path.""" + kv_cache_dtype = "nvfp4" + torch.manual_seed(5) + device, dtype, eps = "cuda", torch.bfloat16, 1e-6 + base, max_pos = 5_000_000.0, 4096 + num_heads, num_idx_heads = 4 * num_kv_heads, 1 + + weights = [ + torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1 for _ in range(4) + ] + cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device) + positions = torch.randint( + 0, max_pos, (num_tokens,), dtype=torch.int64, device=device + ) + qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM + iqsz = num_idx_heads * HEAD_DIM + qkv = torch.randn( + num_tokens, qsz + 2 * kvsz + iqsz + HEAD_DIM, dtype=dtype, device=device + ) + k_scale, v_scale = ( + torch.tensor(s, dtype=torch.float32, device=device) for s in kv_scales + ) + qkv[:, qsz + kvsz : qsz + 2 * kvsz] = nvfp4_edge_values( + num_tokens * num_kv_heads, v_scale.item() + ).view(num_tokens, kvsz) + qkv_orig = qkv.clone() + + num_blocks = (num_tokens + block_size - 1) // block_size + 1 + kv_cache = torch.zeros( + num_blocks, + 2 * num_kv_heads, + block_size, + HEAD_DIM // 2 + HEAD_DIM // 16, + dtype=torch.uint8, + device=device, + ) + index_cache = torch.zeros( + num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device + ) + slot_mapping = torch.randperm( + num_blocks * block_size, dtype=torch.int64, device=device + )[:num_tokens] + slot_mapping[::5] = -1 # padded / unscheduled tokens are not written + index_slot_mapping = torch.randperm( + num_blocks * block_size, dtype=torch.int64, device=device + )[:num_tokens] + + ops.fused_minimax_m3_qknorm_rope_kv_insert( + qkv, + *weights[:2], + cos_sin, + positions, + num_heads, + num_kv_heads, + ROTARY_DIM, + eps, + *weights[2:], + num_idx_heads, + slot_mapping, + index_slot_mapping, + kv_cache, + index_cache, + block_size, + torch.empty(num_tokens, qsz, dtype=dtype, device=device), + torch.empty(num_tokens, iqsz, dtype=dtype, device=device), + kv_cache_dtype, + kv_k_scale=k_scale, + kv_v_scale=v_scale, + ) + + _, k_out, v_out, _, index_k = qkv.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1) + torch.testing.assert_close( + v_out, qkv_orig.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)[2] + ) + expected_kv_cache = nvfp4_pages( + [ + (k_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / k_scale), + (v_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / v_scale), + ], + slot_mapping, + num_blocks, + block_size, + ) + assert torch.equal(kv_cache, expected_kv_cache), ( + f"{int((kv_cache != expected_kv_cache).sum())} NVFP4 cache bytes differ" + ) + + expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM) + expected_index_cache[index_slot_mapping] = index_k + torch.testing.assert_close( + index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0 + ) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index 51a810650f..eef1183697 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -2587,6 +2587,8 @@ def fused_minimax_m3_qknorm_rope_kv_insert( skip_index_branch: bool = False, q_fp8_out: torch.Tensor | None = None, q_fp8_scale: float = 1.0, + kv_k_scale: torch.Tensor | None = None, + kv_v_scale: torch.Tensor | None = None, ) -> None: """Fused MiniMax-M3 attention pre-processing (in-place). @@ -2610,7 +2612,14 @@ def fused_minimax_m3_qknorm_rope_kv_insert( attention's flat TMA descriptor. If ``q_fp8_out`` is given, the same normalized q is also written in FP8 - E4M3 using ``q_fp8_scale`` as its dequantization scale. + E4M3 using ``q_fp8_scale`` as its dequantization scale. Without ``q_out``, + q is then written only in FP8 and the q slice of ``qkv`` is left as is. + + ``kv_cache_dtype="nvfp4"`` quantizes k/v into the packed HND + ``[num_blocks, 2 * num_kv_heads, block_size, 72]`` uint8 cache using the + device dequantization scales ``kv_k_scale``/``kv_v_scale``. Slot + ``2 * head + side`` (K = 0, V = 1) holds that head's E2M1 data followed by + its E4M3 block scales, so every head is one contiguous run of the page. When ``skip_index_branch`` is true, sparse rows still keep their packed ``[index_q | index_k]`` tail, but the kernel only processes the main q/k/v @@ -2641,6 +2650,8 @@ def fused_minimax_m3_qknorm_rope_kv_insert( skip_index_branch, q_fp8_out, q_fp8_scale, + kv_k_scale, + kv_v_scale, ) diff --git a/vllm/models/minimax_m3/common/sparse_attention.py b/vllm/models/minimax_m3/common/sparse_attention.py index 75aa90100c..61791dc5ff 100644 --- a/vllm/models/minimax_m3/common/sparse_attention.py +++ b/vllm/models/minimax_m3/common/sparse_attention.py @@ -56,6 +56,7 @@ from vllm.v1.attention.backends.utils import ( from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheLayout, + get_kv_quant_mode, is_quantized_kv_cache, ) @@ -531,10 +532,10 @@ def select_main_backend_and_impl_cls( """Pick the main attention backend and implementation. Blackwell (SM100) uses the MSA attend for supported top-k block counts - when the KV cache is BF16 or FP8 E4M3; MI355 uses AITER sparse PA + when the KV cache is BF16, FP8 E4M3 or NVFP4; MI355 uses AITER sparse PA with shuffle KV cache layout; Other platforms and FP8 E5M2 fall - back to Triton. The MSA modules are imported lazily to avoid import errors - on unsupported platforms. + back to Triton. NVFP4 is MSA-only. The MSA modules are imported lazily to + avoid import errors on unsupported platforms. """ use_aiter_sparse_pa = minimax_m3_use_aiter_sparse_pa( num_kv_heads, emits_sparse_block_table=emits_sparse_block_table @@ -545,6 +546,18 @@ def select_main_backend_and_impl_cls( and topk_blocks in (4, 8, 16, 32) and kv_cache_dtype != "fp8_e5m2" ) + use_nvfp4 = get_kv_quant_mode(kv_cache_dtype).is_nvfp4 + if use_nvfp4 and kv_cache_dtype != "nvfp4": + raise ValueError( + "MiniMax M3 sparse attention supports only the 'nvfp4' NVFP4 KV " + f"cache dtype, got {kv_cache_dtype}" + ) + if use_nvfp4 and (use_aiter_sparse_pa or not use_msa): + raise ValueError( + "MiniMax M3 sparse attention supports an NVFP4 KV cache only with " + f"the SM100 MSA backend (kv_cache_dtype={kv_cache_dtype}, " + f"topk_blocks={topk_blocks})" + ) selected = ( "AITER_SPARSE_PA" if use_aiter_sparse_pa else ("MSA" if use_msa else "Triton") ) @@ -565,8 +578,11 @@ def select_main_backend_and_impl_cls( from vllm.models.minimax_m3.nvidia.sparse_attention_msa import ( MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl, + MiniMaxM3SparseMSANvfp4Backend, ) + if use_nvfp4: + return MiniMaxM3SparseMSANvfp4Backend, MiniMaxM3SparseMSAImpl return MiniMaxM3SparseMSABackend, MiniMaxM3SparseMSAImpl return MiniMaxM3SparseBackend, MiniMaxM3SparseTritonImpl diff --git a/vllm/models/minimax_m3/nvidia/model.py b/vllm/models/minimax_m3/nvidia/model.py index 1c0138946f..06c28a7a38 100644 --- a/vllm/models/minimax_m3/nvidia/model.py +++ b/vllm/models/minimax_m3/nvidia/model.py @@ -500,6 +500,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): # of wrapping the generic Attention module. Keep the same runtime scale # attributes so FP8 KV reads can honor vLLM's per-layer descale contract. set_default_quant_scales(self, register_buffer=True) + # The fused insert quantizes k/v into NVFP4 pages with these scales. + self.use_nvfp4_kv = get_kv_quant_mode(self.kv_cache_dtype).is_nvfp4 # Indexer side-cache dtype, mirroring --kv-cache-dtype for the main # cache (--attention-config '{"indexer_kv_dtype": ...}'). self.indexer_kv_dtype = vllm_config.attention_config.resolve_indexer_kv_dtype( @@ -611,6 +613,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): main_slot_mapping = fwd_slot_mapping[self.layer_name] index_slot_mapping = fwd_slot_mapping[self.indexer.index_cache.prefix] q = qkv.new_empty((num_tokens, self.q_size)) + # With query_fp8, q is emitted only in fp8; the attend dequantizes it + # into ``q`` for the tokens that run on bf16-query kernels. query_fp8 = self._allocate_query_fp8(qkv) # index_q matches the index-K cache dtype (e4m3 for the fp8 score path); # the fused kernel emits fp8 directly when this buffer is e4m3. @@ -636,11 +640,13 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): self.kv_cache, self.indexer.index_cache.kv_cache, self.kv_cache.size(2), # paged-cache block size - q, + q if query_fp8 is None else None, index_q, self.kv_cache_dtype, q_fp8_out=query_fp8, q_fp8_scale=self._q_scale_float, + kv_k_scale=self._k_scale if self.use_nvfp4_kv else None, + kv_v_scale=self._v_scale if self.use_nvfp4_kv else None, ) output = torch.empty_like(q) diff --git a/vllm/models/minimax_m3/nvidia/msa_cutlass_sparse_decode.py b/vllm/models/minimax_m3/nvidia/msa_cutlass_sparse_decode.py index 04313af107..c52dee413d 100644 --- a/vllm/models/minimax_m3/nvidia/msa_cutlass_sparse_decode.py +++ b/vllm/models/minimax_m3/nvidia/msa_cutlass_sparse_decode.py @@ -12,6 +12,7 @@ import torch from vllm.config.attention import MiniMaxM3MSADecodeBackend from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.v1.kv_cache_interface import get_kv_quant_mode _MAX_NUM_Q_HEADS = 64 _MAX_NUM_KV_HEADS = 4 @@ -23,9 +24,36 @@ _TOPK = 16 _MAX_QUERY_HEAD_ROWS = 65536 _MAX_DECODE_QUERY_LEN = 32 # Kernel benchmarks put the CUTLASS crossover at 16 requests for TP1 and TP4. +# NVFP4 has no Triton fallback, so it takes CUTLASS at every batch size. _MIN_CUTLASS_BATCH_SIZE = 16 +def is_nvfp4_kv_cache(kv_cache_dtype: str) -> bool: + return get_kv_quant_mode(kv_cache_dtype).is_nvfp4 + + +def nvfp4_kv_cache_views( + kv_cache: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Return the K data, K scale, V data and V scale views of an NVFP4 cache. + + ``kv_cache`` is uint8 ``[pages, 2 * Hkv, page_size, 72]``; slot + ``2 * head + side`` (K = 0, V = 1) is that head's E2M1 data + ``[page_size, 64]`` followed by its E4M3 block scales ``[page_size, 8]``, + the one page layout ``fmha_sm100`` reads. + """ + from vllm.third_party.fmha_sm100.nvfp4_kv import nvfp4_head_slot_views + + return nvfp4_head_slot_views(*nvfp4_kv_cache_slots(kv_cache)) + + +def nvfp4_kv_cache_slots( + kv_cache: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Return the per-head K and V slots of an NVFP4 cache.""" + return kv_cache[:, 0::2], kv_cache[:, 1::2] + + @dataclass class MSACutlassDecodeMetadata: plan: Any @@ -91,6 +119,9 @@ class MSACutlassDecodePlanCache: use_fp8_kvcache=True, split_prefill_decode=False, device=device, + # The Q8KV4 route keeps plan-time lengths and its own page offsets, + # which this cached plan does not refresh. + decode_backend="kv_mode3", ) plan_info = plan[3] @@ -214,11 +245,12 @@ def supports_cutlass_sparse_decode( topk_blocks: int, ) -> bool: """Return whether static model geometry supports CUTLASS sparse decode.""" + nvfp4 = is_nvfp4_kv_cache(kv_cache_dtype) return ( - decode_backend == "cutlass" + (decode_backend == "cutlass" or nvfp4) and current_platform.is_cuda() and current_platform.is_device_capability_family(100) - and kv_cache_dtype in ("fp8", "fp8_e4m3") + and (kv_cache_dtype in ("fp8", "fp8_e4m3") or nvfp4) and _supported_head_geometry(num_q_heads, num_kv_heads) and page_size == _PAGE_SIZE and topk_blocks == _TOPK @@ -248,7 +280,7 @@ def should_prepare_decode_metadata( topk_blocks=topk_blocks, ) and 1 <= decode_query_len <= _MAX_DECODE_QUERY_LEN - and batch_size >= _MIN_CUTLASS_BATCH_SIZE + and (batch_size >= _MIN_CUTLASS_BATCH_SIZE or is_nvfp4_kv_cache(kv_cache_dtype)) and total_q * num_q_heads <= _MAX_QUERY_HEAD_ROWS ) @@ -292,9 +324,22 @@ def msa_cutlass_sparse_decode( q_scale_float: float, k_scale_float: float, v_scale_float: float, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, ) -> None: - """Run CUTLASS sparse decode with metadata prepared by the MSA builder.""" - key, value = kv_cache.split(_HEAD_DIM, dim=-1) + """Run CUTLASS sparse decode with metadata prepared by the MSA builder. + + NVFP4 caches are uint8 head-slot pages (see ``nvfp4_kv_cache_views``) and + take the device global scales ``k_scale``/``v_scale``. + """ + if kv_cache.dtype == torch.uint8: + assert k_scale is not None and v_scale is not None + key, value = nvfp4_kv_cache_slots(kv_cache) + k_scale_arg: float | torch.Tensor = k_scale + v_scale_arg: float | torch.Tensor = v_scale + else: + key, value = kv_cache.split(_HEAD_DIM, dim=-1) + k_scale_arg, v_scale_arg = k_scale_float, v_scale_float from vllm.third_party.fmha_sm100.api import fmha_sm100 @@ -310,7 +355,7 @@ def msa_cutlass_sparse_decode( output_o=True, sm_scale=scale, q_scale=q_scale_float, - k_scale=k_scale_float, - v_scale=v_scale_float, + k_scale=k_scale_arg, + v_scale=v_scale_arg, o_scale=1.0, ) diff --git a/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py b/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py index d29e1346bd..5aa23b4ac8 100644 --- a/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py +++ b/vllm/models/minimax_m3/nvidia/sparse_attention_msa.py @@ -4,15 +4,18 @@ Prefill attends with ``fmha_sm100`` (``build_k2q_csr`` + ``sparse_atten_func``). Decode uses Triton split-K by default, with an opt-in CUTLASS ``fmha_sm100`` -path for regular decode and speculative verification. +path for regular decode and speculative verification. NVFP4 KV caches read the +vLLM packed pages directly in both prefill and CUTLASS decode. """ -from dataclasses import dataclass +from dataclasses import dataclass, replace +from typing import ClassVar import torch from vllm.config import VllmConfig from vllm.config.attention import MiniMaxM3MSADecodeBackend +from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.models.minimax_m3.common.ops.sparse_attn import ( @@ -29,20 +32,33 @@ from vllm.models.minimax_m3.common.sparse_attention import ( from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import ( MSACutlassDecodeMetadata, MSACutlassDecodePlanCache, + is_nvfp4_kv_cache, msa_cutlass_sparse_decode, + nvfp4_kv_cache_views, prepare_decode_metadata, should_prepare_decode_metadata, supports_cutlass_sparse_decode, ) +from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim from vllm.v1.attention.backend import ( AttentionLayer, CommonAttentionMetadata, ) -from vllm.v1.kv_cache_interface import AttentionSpec +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout logger = init_logger(__name__) +def _dequantize_query( + query_fp8: torch.Tensor, q_scale: float, out: torch.Tensor +) -> torch.Tensor: + """Write ``query_fp8 * q_scale`` into ``out`` (E4M3 is exact in BF16).""" + out.copy_(query_fp8) + if q_scale != 1.0: + out.mul_(q_scale) + return out + + class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend): """MiniMax M3 backend with NVIDIA MSA-specific decode metadata.""" @@ -51,6 +67,30 @@ class MiniMaxM3SparseMSABackend(MiniMaxM3SparseBackend): return MiniMaxM3SparseMSAMetadataBuilder +class MiniMaxM3SparseMSANvfp4Backend(MiniMaxM3SparseMSABackend): + """MSA backend over HND NVFP4 KV cache pages in per-head K/V slots.""" + + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = ["nvfp4"] + + @classmethod + def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec: + """Store each head's K and V as two slots of packed data + block scales + (see ``nvfp4_kv_cache_views``).""" + if spec.state_content_bytes is not None or not spec.kv_quant_mode.is_nvfp4: + return spec + return replace( + spec, + num_head_slots=2 * spec.num_kv_heads, + state_content_bytes=nvfp4_kv_cache_full_dim(spec.head_size) + * get_dtype_size(spec.dtype), + ) + + @classmethod + def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...] | None: + # Every head's K/V slots are one contiguous run of the page. + return (KVCacheLayout.LBHNC,) + + class MiniMaxM3SparseCutlassBackend(MiniMaxM3SparseMSABackend): """Attention-backend alias selecting CUTLASS MSA sparse decode.""" @@ -166,6 +206,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): topk_blocks=topk_blocks, sparse_block_size=sparse_block_size, ) + self.use_nvfp4_kv = is_nvfp4_kv_cache(kv_cache_dtype) self.use_cutlass_decode = supports_cutlass_sparse_decode( decode_backend=msa_decode_backend, num_q_heads=self.num_heads, @@ -174,6 +215,12 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): page_size=self.block_size, topk_blocks=self.topk_blocks, ) + if self.use_nvfp4_kv and not self.use_cutlass_decode: + raise ValueError( + "MiniMax M3 NVFP4 KV cache requires CUTLASS MSA sparse decode " + f"(num_heads={self.num_heads}, num_kv_heads={self.num_kv_heads}, " + f"block_size={self.block_size}, topk_blocks={self.topk_blocks})" + ) logger.info_once( "MiniMax M3 MSA sparse decode selected %s", "CUTLASS" if self.use_cutlass_decode else "Triton", @@ -218,9 +265,17 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): hd = self.head_size q = query[:num_tokens].view(-1, self.num_heads, hd) out = output[:num_tokens].view(-1, self.num_heads, hd) - kv_cache = ( - kv_cache.view(self.kv_cache_fp8_dtype) if self.use_fp8_kv else kv_cache + # Given query_fp8, it is the only q the fused insert wrote: CUTLASS + # decode reads it directly, and the bf16-query kernels read it + # dequantized into ``query``. + q_fp8 = ( + None + if query_fp8 is None + else query_fp8[:num_tokens].view(-1, self.num_heads, hd) ) + q_scale_float = getattr(layer, "_q_scale_float", 1.0) + if self.use_fp8_kv and not self.use_nvfp4_kv: + kv_cache = kv_cache.view(self.kv_cache_fp8_dtype) k_scale = getattr(layer, "_k_scale", None) if self.use_fp8_kv else None v_scale = getattr(layer, "_v_scale", None) if self.use_fp8_kv else None @@ -234,21 +289,31 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): else None ) if self.use_cutlass_decode and msa_metadata is not None: - assert query_fp8 is not None + assert q_fp8 is not None msa_cutlass_sparse_decode( - query_fp8[:nd].view(-1, self.num_heads, hd), + q_fp8[:nd], kv_cache, topk[:nd], out[:nd], msa_metadata, scale=self.scale, - q_scale_float=getattr(layer, "_q_scale_float", 1.0), + q_scale_float=q_scale_float, k_scale_float=getattr(layer, "_k_scale_float", 1.0), v_scale_float=getattr(layer, "_v_scale_float", 1.0), + k_scale=k_scale if self.use_nvfp4_kv else None, + v_scale=v_scale if self.use_nvfp4_kv else None, + ) + elif self.use_nvfp4_kv: + raise RuntimeError( + "MiniMax M3 NVFP4 KV cache has no Triton decode fallback; " + f"CUTLASS decode was not planned for {main_md.num_decodes} " + f"requests with query length {d.decode_query_len}" ) else: minimax_m3_sparse_attn_decode( - q[:nd], + q[:nd] + if q_fp8 is None + else _dequantize_query(q_fp8[:nd], q_scale_float, q[:nd]), kv_cache, topk[:nd].transpose(0, 1), d.block_table, @@ -266,6 +331,7 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): from vllm.third_party.fmha_sm100.sparse import ( build_k2q_csr, sparse_atten_func, + sparse_atten_nvfp4_kv_func, ) p = main_md.prefill @@ -273,8 +339,11 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): # [H, prefill, MK] transposed view; build_k2q_csr consumes the # strided view directly (topK stays innermost-contiguous). prefill_topk = topk[nd:num_tokens].transpose(0, 1) - qp = q[nd:] - k_cache, v_cache = kv_cache.split(self.head_size, dim=-1) + qp = ( + q[nd:] + if q_fp8 is None + else _dequantize_query(q_fp8[nd:], q_scale_float, q[nd:]) + ) k2q_row_ptr, k2q_q_indices, schedule = build_k2q_csr( prefill_topk, p.cu_seqlens_q, @@ -287,6 +356,34 @@ class MiniMaxM3SparseMSAImpl(MiniMaxM3SparseImpl): qhead_per_kv=qp.shape[1] // self.num_kv_heads, return_schedule=True, ) + if self.use_nvfp4_kv: + k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache) + sparse_atten_nvfp4_kv_func( + qp, + k_data, + v_data, + k_sf, + v_sf, + k_scale, + v_scale, + k2q_row_ptr, + k2q_q_indices, + self.topk_blocks, + cu_seqlens_q=p.cu_seqlens_q, + cu_seqlens_k=p.cu_seqlens_k, + max_seqlen_q=p.max_query_len, + max_seqlen_k=p.max_seq_len, + blk_kv=SPARSE_BLOCK_SIZE, + causal=True, + softmax_scale=self.scale, + page_table=p.block_table, + seqused_k=p.seq_lens, + schedule=schedule, + out=out[nd:], + kv_layout="vllm", + ) + return output + k_cache, v_cache = kv_cache.split(self.head_size, dim=-1) sparse_atten_func( qp, k_cache, From ca65eb67d995f5401c575ec90cba523757127ec7 Mon Sep 17 00:00:00 2001 From: Matej Sirovatka <54212263+S1ro1@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:48:18 -0700 Subject: [PATCH 218/248] [Perf][HiSparse] Avoid repeated prefix scans and residency updates (#57930) Signed-off-by: S1ro1 Signed-off-by: Lucas Wilkinson Signed-off-by: Matthew Bonanni Co-authored-by: Lucas Wilkinson Co-authored-by: Claude Fable 5.1 Co-authored-by: Matthew Bonanni --- tests/v1/core/test_prefix_caching.py | 180 +++++++++++++----- .../unit/test_hisparse_connector.py | 53 ++++++ .../kv_connector/v1/hisparse/connector.py | 38 ++-- vllm/v1/core/single_type_kv_cache_manager.py | 18 -- vllm/v1/hisparse/coordinator.py | 30 ++- 5 files changed, 236 insertions(+), 83 deletions(-) diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 027bb67b18..3972fe5230 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -168,6 +168,7 @@ def make_hisparse_kv_cache_config( host_num_blocks: int, *, transfer_device_cache: bool = False, + num_resident_groups: int = 1, ) -> KVCacheConfig: source_spec = FullAttentionSpec( block_size=HISPARSE_BLOCK_SIZE, @@ -210,6 +211,13 @@ def make_hisparse_kv_cache_config( enable_kv_transfer=False, ) ) + for group_idx in range(1, num_resident_groups): + groups.extend( + replace( + group, layer_names=[f"{name}_{group_idx}" for name in group.layer_names] + ) + for group in groups[2:4] + ) return KVCacheConfig( num_blocks=num_blocks, hisparse_host_num_blocks=host_num_blocks, @@ -225,6 +233,7 @@ def make_hisparse_kv_cache_manager( max_model_len: int = 128, max_in_flight_tokens: int | None = None, enable_caching: bool = False, + num_prefill_lookahead: int = 0, **config_kwargs, ) -> KVCacheManager: config = make_hisparse_kv_cache_config(num_blocks, host_num_blocks, **config_kwargs) @@ -234,9 +243,53 @@ def make_hisparse_kv_cache_manager( max_in_flight_tokens=max_in_flight_tokens, enable_caching=enable_caching, hash_block_size=HISPARSE_BLOCK_SIZE, + num_prefill_lookahead=num_prefill_lookahead, ) +def _allocate_scheduled( + manager: KVCacheManager, + request: Request, + num_new_tokens: int, + *, + num_new_computed_tokens: int = 0, + num_external_computed_tokens: int = 0, + **kwargs, +) -> KVCacheBlocks | None: + """``allocate_slots`` plus the per-step residency work the connector runs.""" + blocks = manager.allocate_slots( + request, + num_new_tokens, + num_new_computed_tokens=num_new_computed_tokens, + num_external_computed_tokens=num_external_computed_tokens, + **kwargs, + ) + if blocks is None or kwargs.get("delay_cache_blocks"): + return blocks + num_computed_tokens = min( + request.num_computed_tokens + + num_new_computed_tokens + + num_external_computed_tokens, + manager.max_model_len, + ) + num_tokens = min(num_computed_tokens + num_new_tokens, request.num_tokens) + get_hisparse_coordinator(manager).advance_scheduled( + [(request.request_id, num_tokens)] + ) + return blocks + + +def test_hisparse_does_not_write_back_reprefillable_tokens(): + """Multi-module MTP re-prefills the last tokens, so they are not final.""" + manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3) + request = make_request( + "request", list(range(2 * HISPARSE_BLOCK_SIZE)), HISPARSE_BLOCK_SIZE, sha256 + ) + assert _allocate_scheduled(manager, request, 2 * HISPARSE_BLOCK_SIZE) + transfers = get_hisparse_coordinator(manager).build_offload_command().page_transfers + assert len(transfers) == 1 + + def test_hisparse_builds_dma_row_mirrors_across_pages(): manager = make_hisparse_kv_cache_manager(32, 16) request = make_request( @@ -245,7 +298,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages(): HISPARSE_BLOCK_SIZE, sha256, ) - assert manager.allocate_slots(request, num_new_tokens=32) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None coordinator = get_hisparse_coordinator(manager) resident_blocks = coordinator.resident_managers[0].req_to_blocks[request.request_id] host_blocks = coordinator.host_manager.req_to_blocks[request.request_id] @@ -308,14 +361,14 @@ def test_hisparse_reports_when_context_is_fully_resident(): manager = make_hisparse_kv_cache_manager(32, 16) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None coordinator = get_hisparse_coordinator(manager) scheduled = ((request.request_id, len(tokens) - 1, 1),) assert coordinator.all_context_pages_resident(scheduled) for hot_manager in coordinator.hot_managers: hot_manager.require_hot(request.request_id) - assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None _publish_hisparse_pages(manager) pool = manager.block_pool pool.get_new_blocks(pool.get_num_free_blocks()) @@ -333,7 +386,9 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): ) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers spill_counts = {spill.transfer_id: 1 for spill in spills} get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts) @@ -348,7 +403,8 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload(): assert num_local == 2 * HISPARSE_BLOCK_SIZE assert [len(group_blocks) for group_blocks in blocks.blocks] == [3, 2, 0, 0] - allocated = manager.allocate_slots( + allocated = _allocate_scheduled( + manager, resumed, num_new_tokens=1, num_new_computed_tokens=num_local, @@ -379,7 +435,9 @@ def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix(): ) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) spills = get_hisparse_coordinator(manager).build_offload_command().page_transfers spill_counts = {spill.transfer_id: 1 for spill in spills} get_hisparse_coordinator(manager).update_spills(spill_counts, spill_counts) @@ -410,7 +468,7 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv( manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(64)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=64) is not None + assert _allocate_scheduled(manager, original, num_new_tokens=64) is not None _publish_hisparse_pages(manager) host, indexer, _, _ = manager.get_blocks(original.request_id).blocks missing_indexer = indexer[1].block_id @@ -457,7 +515,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv( assert asynchronous == (external > 0) assert [len(group) for group in blocks.blocks] == [3, 1, 0, 0] assert ( - manager.allocate_slots( + _allocate_scheduled( + manager, resumed, int(external == 0), delay_cache_blocks=True, @@ -486,7 +545,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv( def allocate_external_prefix( manager: KVCacheManager, request: Request, num_tokens: int ) -> KVCacheBlocks | None: - return manager.allocate_slots( + return _allocate_scheduled( + manager, request, num_new_tokens=0, num_external_computed_tokens=num_tokens, @@ -538,7 +598,7 @@ def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens): scheduler._mark_prefix_replay = MethodType(Scheduler._mark_prefix_replay, scheduler) Scheduler._update_waiting_for_remote_kv(scheduler, request) assert request.num_tokens - request.num_computed_tokens == 1 - assert manager.allocate_slots(request, num_new_tokens=1) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)]) # A pending restore must not expose uninitialized GPU copies to prefix hits. assert all(resident[-1] not in copies for copies in coordinator.copies.values()) @@ -583,15 +643,15 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach 32, 32, max_model_len=256, enable_caching=enable_caching ) first = make_request("first", list(range(160)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(first, num_new_tokens=128) is not None + assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None _publish_hisparse_pages(manager) first.num_computed_tokens = 128 pool = manager.block_pool held = pool.get_new_blocks(pool.get_num_free_blocks() - 3) - assert manager.allocate_slots(first, num_new_tokens=1) is not None + assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256) - manager.allocate_slots(second, num_new_tokens=16) + _allocate_scheduled(manager, second, num_new_tokens=16) coordinator = get_hisparse_coordinator(manager) scheduled = (("first", 128, 1),) assert coordinator.all_context_pages_resident(scheduled) @@ -600,7 +660,7 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach # Once another allocation finishes, A can acquire its hot buffer and # safely release its clean pages for other requests to reuse. pool.free_blocks(held) - assert manager.allocate_slots(first, num_new_tokens=1) is not None + assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None assert all(m.has_hot("first") for m in coordinator.hot_managers) pool.get_new_blocks(pool.get_num_free_blocks()) assert not coordinator.all_context_pages_resident(scheduled) @@ -611,21 +671,24 @@ def test_hisparse_full_pool_keeps_pages_pinned_until_preemption(): """Without room for a hot buffer, admission must defer instead of losing KV.""" manager = make_hisparse_kv_cache_manager(18, 18, max_model_len=160) first = make_request("first", list(range(128)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(first, num_new_tokens=128) is not None + assert _allocate_scheduled(manager, first, num_new_tokens=128) is not None _publish_hisparse_pages(manager) assert manager.block_pool.get_num_free_blocks() == 1 first.num_computed_tokens = 128 - assert manager.allocate_slots(first, num_new_tokens=1) is None + assert _allocate_scheduled(manager, first, num_new_tokens=1) is None assert get_hisparse_coordinator(manager).all_context_pages_resident( (("first", 127, 1),) ) manager.free(first) second = make_request("second", [1000] * 16, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(second, num_new_tokens=16) is not None + assert _allocate_scheduled(manager, second, num_new_tokens=16) is not None -def test_hisparse_materializes_prefix_without_allocating_hot_blocks(): +@pytest.mark.parametrize("num_resident_groups", [1, 3]) +def test_hisparse_materializes_prefix_without_allocating_hot_blocks( + num_resident_groups, +): """A host prefix becomes visible only when every page is durable. Completing a later page first must not expose a prefix with a hole. @@ -634,10 +697,11 @@ def test_hisparse_materializes_prefix_without_allocating_hot_blocks(): 32, 16, enable_caching=True, + num_resident_groups=num_resident_groups, ) tokens = list(range(2 * HISPARSE_BLOCK_SIZE)) request = make_request("resident", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None blocks = manager.get_block_ids(request.request_id) assert len(blocks[2]) == 2 assert blocks[3] == [] @@ -675,19 +739,21 @@ def test_hisparse_materializes_prefix_without_allocating_hot_blocks(): assert blocks[3] == [] -def test_hisparse_materialization_respects_per_step_spill_budget(): +@pytest.mark.parametrize("num_resident_groups", [1, 3]) +def test_hisparse_materialization_respects_per_step_spill_budget(num_resident_groups): """Prefix publication must not bypass the configured spill batch limit.""" manager = make_hisparse_kv_cache_manager( 32, 16, enable_caching=True, + num_resident_groups=num_resident_groups, ) coordinator = get_hisparse_coordinator(manager) coordinator.max_spill_pages = 1 tokens = list(range(2 * HISPARSE_BLOCK_SIZE)) request = make_request("bounded", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None first = coordinator.build_offload_command().page_transfers assert len(first) == 1 @@ -705,7 +771,9 @@ def test_hisparse_deferred_free_retains_host_blocks_until_fence(): host_free = host_pool.get_num_free_blocks() device_free = manager.block_pool.get_num_free_blocks() request = make_request("free", list(range(16)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, 16, delay_cache_blocks=True) is not None + assert ( + _allocate_scheduled(manager, request, 16, delay_cache_blocks=True) is not None + ) blocks = manager.pop_blocks_for_free(request) assert host_pool.get_num_free_blocks() < host_free assert blocks and any(host_pool.blocks[b.block_id] is b for b in blocks) @@ -903,7 +971,7 @@ def test_hisparse_host_exhaustion_defers_allocation(enable_caching): """Pages are only allocated with host backing, so every one can be reclaimed.""" manager = make_hisparse_kv_cache_manager(32, 5, enable_caching=enable_caching) donor = make_request("donor", [99] * 16, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(donor, 16) is not None + assert _allocate_scheduled(manager, donor, 16) is not None request = make_request("resident", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) free_gpu = manager.block_pool.get_num_free_blocks() assert manager.allocate_slots(request, 64) is None @@ -916,7 +984,7 @@ def test_hisparse_host_exhaustion_defers_allocation(enable_caching): assert manager.allocate_slots(request, 64) is None _publish_hisparse_pages(manager) assert not coordinator.has_pending_reclamation() - assert manager.allocate_slots(request, 64) is not None + assert _allocate_scheduled(manager, request, 64) is not None host = coordinator.host_manager assert host is not None assert not any(b.is_null for b in host.req_to_blocks[request.request_id]) @@ -974,7 +1042,7 @@ def test_hisparse_recomputation_does_not_complete_failed_host_import(): # allocation reaches the original import boundary before the worker runs. request.num_computed_tokens = 16 manager.cache_blocks(request, 16) - assert manager.allocate_slots(request, 16) is not None + assert _allocate_scheduled(manager, request, 16) is not None state = get_hisparse_coordinator(manager).request_states.get(request.request_id) assert state is None or 1 not in state.valid_pages @@ -995,12 +1063,12 @@ def test_hisparse_resident_request_can_grow_without_hot_capacity(): """Running resident history must not be mistaken for a new host-prefix hit.""" manager = make_hisparse_kv_cache_manager(8, 16) request = make_request("resident", list(range(48)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, num_new_tokens=32) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=32) is not None request.num_computed_tokens = 32 # Three free blocks fit the next indexer/resident pages, but not a hot region. assert manager.block_pool.get_num_free_blocks() == 3 - assert manager.allocate_slots(request, num_new_tokens=1) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=1) is not None assert manager.get_block_ids(request.request_id)[3] == [] @@ -1050,7 +1118,7 @@ def test_hisparse_events_report_host_and_device_placement(): enable_kv_cache_events=True, ) request = make_request("request", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, 64) is not None + assert _allocate_scheduled(manager, request, 64) is not None _publish_hisparse_pages(manager) stored = [ @@ -1113,7 +1181,9 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages(): manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) _publish_hisparse_pages(manager) original_resident_ids = [ block.block_id for block in manager.get_blocks("original").blocks[2] @@ -1123,7 +1193,8 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages(): resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) computed, num_computed, _ = manager.get_computed_blocks(resumed) assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert manager.allocate_slots( + assert _allocate_scheduled( + manager, resumed, num_new_tokens=len(tokens) - num_computed, num_new_computed_tokens=num_computed, @@ -1154,7 +1225,9 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies( manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) _publish_hisparse_pages(manager) copy_ids = [block.block_id for block in manager.get_blocks("original").blocks[2]] manager.free(original) @@ -1164,7 +1237,8 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies( resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) computed, num_computed, _ = manager.get_computed_blocks(resumed) assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert manager.allocate_slots( + assert _allocate_scheduled( + manager, resumed, num_new_tokens=len(tokens) - num_computed, num_new_computed_tokens=num_computed, @@ -1190,14 +1264,17 @@ def test_hisparse_host_backed_request_accepts_local_prefix_hit(): manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) _publish_hisparse_pages(manager) manager.free(original) resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) computed, num_computed, _ = manager.get_computed_blocks(resumed) assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert manager.allocate_slots( + assert _allocate_scheduled( + manager, resumed, num_new_tokens=len(tokens) - num_computed, num_new_computed_tokens=num_computed, @@ -1211,7 +1288,9 @@ def test_hisparse_finished_request_leaves_free_copies(): manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) _publish_hisparse_pages(manager) manager.free(original) @@ -1241,7 +1320,9 @@ def test_hisparse_cleanup_publishes_only_finalized_terminal_pages( coordinator = get_hisparse_coordinator(manager) request = make_request("terminal", list(range(33)), HISPARSE_BLOCK_SIZE, sha256) request.spec_token_ids = list(range(33, 49)) - assert manager.allocate_slots(request, 49, num_lookahead_tokens=16) is not None + assert ( + _allocate_scheduled(manager, request, 49, num_lookahead_tokens=16) is not None + ) host_blocks = list(manager.get_blocks(request.request_id).blocks[0]) command = coordinator.build_offload_command() counts = {transfer.transfer_id: 1 for transfer in command.page_transfers} @@ -1280,7 +1361,10 @@ def test_hisparse_prefill_reads_host_once_it_fills_admission_window( coordinator = get_hisparse_coordinator(manager) tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE)) request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, num_pages * HISPARSE_BLOCK_SIZE) is not None + assert ( + _allocate_scheduled(manager, request, num_pages * HISPARSE_BLOCK_SIZE) + is not None + ) pool = coordinator.gpu_pool assert pool is not None assert pool.get_num_free_blocks() >= coordinator.transition_watermark @@ -1303,10 +1387,10 @@ def test_hisparse_preempted_prefill_resumes_from_durable_prefix(): chunk = 2 * HISPARSE_BLOCK_SIZE tokens = list(range(3 * chunk)) request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, chunk) is not None + assert _allocate_scheduled(manager, request, chunk) is not None request.num_computed_tokens = chunk first_chunk_writes = coordinator.build_offload_command().page_transfers - assert manager.allocate_slots(request, chunk) is not None + assert _allocate_scheduled(manager, request, chunk) is not None request.num_computed_tokens = 2 * chunk acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes} coordinator.update_spills(acks, acks) @@ -1323,7 +1407,7 @@ def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity() host_pool = coordinator.get_host_block_pool() tokens = list(range(2 * HISPARSE_BLOCK_SIZE + 1)) request = make_request("reused", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, len(tokens)) is not None + assert _allocate_scheduled(manager, request, len(tokens)) is not None old_blocks = list(manager.get_blocks(request.request_id).blocks[0]) command = coordinator.build_offload_command() counts = {transfer.transfer_id: 2 for transfer in command.page_transfers} @@ -1336,7 +1420,7 @@ def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity() replacement = make_request( "reused", list(range(100, 117)), HISPARSE_BLOCK_SIZE, sha256 ) - assert manager.allocate_slots(replacement, 17) is not None + assert _allocate_scheduled(manager, replacement, 17) is not None new_blocks = list(manager.get_blocks(replacement.request_id).blocks[0]) pressure = host_pool.get_new_blocks(host_pool.get_num_free_blocks()) assert not {block.block_id for block in old_blocks[:2]} & { @@ -1363,7 +1447,7 @@ def test_hisparse_reset_prefix_cache_drops_copies(): manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, num_new_tokens=len(tokens)) is not None + assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None _publish_hisparse_pages(manager) manager.free(request) @@ -1378,7 +1462,9 @@ def test_hisparse_reused_copy_is_not_adopted(): manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(original, num_new_tokens=len(tokens)) is not None + assert ( + _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) is not None + ) _publish_hisparse_pages(manager) indexer_blocks = list(manager.get_blocks("original").blocks[1]) manager.free(original) @@ -1391,7 +1477,8 @@ def test_hisparse_reused_copy_is_not_adopted(): resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) computed, num_computed, _ = manager.get_computed_blocks(resumed) assert num_computed == 3 * HISPARSE_BLOCK_SIZE - assert manager.allocate_slots( + assert _allocate_scheduled( + manager, resumed, num_new_tokens=len(tokens) - num_computed, num_new_computed_tokens=num_computed, @@ -5291,7 +5378,8 @@ def test_can_fit_full_sequence_hisparse_caps_resident_pages(): ) assert ( - manager.allocate_slots( + _allocate_scheduled( + manager, request, num_new_tokens=HISPARSE_BLOCK_SIZE, full_sequence_must_fit=True, @@ -6151,7 +6239,7 @@ def test_device_eviction_keeps_host_prefix(): """Device eviction must not target a same-numbered host block.""" manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) request = make_request("producer", list(range(64)), HISPARSE_BLOCK_SIZE, sha256) - assert manager.allocate_slots(request, 64) is not None + assert _allocate_scheduled(manager, request, 64) is not None _publish_hisparse_pages(manager) host, indexer, _, _ = manager.get_blocks(request.request_id).blocks device_block = indexer[0] diff --git a/tests/v1/kv_connector/unit/test_hisparse_connector.py b/tests/v1/kv_connector/unit/test_hisparse_connector.py index 6c4e79efbe..384460fa2c 100644 --- a/tests/v1/kv_connector/unit/test_hisparse_connector.py +++ b/tests/v1/kv_connector/unit/test_hisparse_connector.py @@ -114,3 +114,56 @@ def test_full_graph_step_prepares_host_mirror_outside_model(): worker._enqueue_row_dma.assert_called_once_with((0,), ready_event=None) runtime.invalidate_written_slots.assert_called_once() + + +def test_scheduled_prefix_hit_publishes_adopted_copies(): + """Copies adopted after scheduling must reach the worker's block table.""" + from tests.v1.core.test_prefix_caching import ( + HISPARSE_BLOCK_SIZE, + _allocate_scheduled, + _publish_hisparse_pages, + make_hisparse_kv_cache_manager, + make_request, + sha256, + ) + from vllm.v1.core.kv_cache_utils import init_none_hash + + init_none_hash(sha256) + manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True) + tokens = list(range(4 * HISPARSE_BLOCK_SIZE)) + original = make_request("original", tokens, HISPARSE_BLOCK_SIZE, sha256) + assert _allocate_scheduled(manager, original, num_new_tokens=len(tokens)) + _publish_hisparse_pages(manager) + copy_ids = [block.block_id for block in manager.get_blocks("original").blocks[2]] + manager.free(original) + + resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256) + computed, num_computed, _ = manager.get_computed_blocks(resumed) + num_new_tokens = len(tokens) - num_computed + assert manager.allocate_slots( + resumed, + num_new_tokens=num_new_tokens, + num_new_computed_tokens=num_computed, + new_computed_blocks=computed, + ) + scheduler = HiSparseConnectorScheduler(async_speculative=False) + scheduler.bind_coordinator(get_hisparse_coordinator(manager)) + scheduler.requests[resumed.request_id] = resumed + scheduler_output = SimpleNamespace( + scheduled_new_reqs=[ + SimpleNamespace( + req_id=resumed.request_id, + num_computed_tokens=num_computed, + block_ids=manager.get_block_ids(resumed.request_id), + ) + ], + scheduled_cached_reqs=SimpleNamespace( + req_ids=[], num_computed_tokens=[], new_block_ids=[] + ), + num_scheduled_tokens={resumed.request_id: num_new_tokens}, + ) + + scheduler.build_connector_meta(scheduler_output) + + resident_ids = scheduler_output.block_table_updates[resumed.request_id][2] + assert resident_ids[:3] == copy_ids[:3] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py index 5acb7e3bbd..dabf9095db 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/connector.py @@ -106,21 +106,6 @@ class HiSparseConnectorScheduler: # HiSparse rebinds worker state every step, so the load must start # before the forward rather than being deferred to post-forward. scheduler_output.has_sync_kv_loads = True - scheduler_output.block_table_updates = ( - self.coordinator.take_block_table_updates() or None - ) - command = self.coordinator.build_offload_command() - host_block_copies = self.coordinator.take_host_block_copies() - source_group_id = self.coordinator.host_group_id - assert source_group_id is not None - source_block_ids = [ - block_id - for request in scheduler_output.scheduled_new_reqs - for block_id in request.block_ids[source_group_id] - ] - for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids: - if new_block_ids is not None: - source_block_ids.extend(new_block_ids[source_group_id]) num_computed_tokens = { request.req_id: request.num_computed_tokens for request in scheduler_output.scheduled_new_reqs @@ -141,6 +126,29 @@ class HiSparseConnectorScheduler: scheduler_output.num_scheduled_tokens.items() ) ) + # Runs first: it plans this step's transfers and table updates. + self.coordinator.advance_scheduled( + ( + request_id, + min(start + count, self.requests[request_id].num_tokens), + ) + for request_id, start, count in scheduled_requests + ) + scheduler_output.block_table_updates = ( + self.coordinator.take_block_table_updates() or None + ) + command = self.coordinator.build_offload_command() + host_block_copies = self.coordinator.take_host_block_copies() + source_group_id = self.coordinator.host_group_id + assert source_group_id is not None + source_block_ids = [ + block_id + for request in scheduler_output.scheduled_new_reqs + for block_id in request.block_ids[source_group_id] + ] + for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids: + if new_block_ids is not None: + source_block_ids.extend(new_block_ids[source_group_id]) row_mirrors = {} for request_id, scheduled_start, scheduled_count in scheduled_requests: mirror_start = scheduled_start diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index a097c9e924..7041600f8b 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -2366,12 +2366,6 @@ class _HiSparseAuxiliaryManager(SingleTypeKVCacheManager): coordinator: "HiSparseCoordinator | None" = None - def __init__(self, kv_cache_spec: KVCacheSpec, **kwargs) -> None: - # Never prefix-cached, but the per-step ``cache_blocks`` hook is where - # residency work runs, so stay opted in regardless of prefix caching. - kwargs["enable_caching"] = True - super().__init__(kv_cache_spec, **kwargs) - def cache_blocks( self, request: Request, @@ -2551,18 +2545,6 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager): assert self.coordinator is not None self.coordinator.commit_computed_blocks(request_id, num_host_pages) - def cache_blocks( - self, - request: Request, - num_tokens: int, - retention_interval: int | None = None, - *, - replay_boundaries: Sequence[int], - ) -> None: - assert self.coordinator is not None - self.coordinator.plan_prefix_materialization(request.request_id, num_tokens) - self.coordinator.update_residency(request.request_id) - def allocate_new_blocks( self, request_id: str, num_tokens: int, num_tokens_main_model: int ) -> list[KVCacheBlock]: diff --git a/vllm/v1/hisparse/coordinator.py b/vllm/v1/hisparse/coordinator.py index 1a783b506b..bc9bff1a8b 100644 --- a/vllm/v1/hisparse/coordinator.py +++ b/vllm/v1/hisparse/coordinator.py @@ -99,9 +99,12 @@ class HiSparseCoordinator: kv_cache_config: KVCacheConfig, managers: tuple[SingleTypeKVCacheManager, ...], max_model_len: int, + *, + num_reprefillable_tokens: int, ) -> None: self.managers = managers self.max_model_len = max_model_len + self.num_reprefillable_tokens = num_reprefillable_tokens groups = kv_cache_config.kv_cache_groups resident_managers: list[HiSparseResidentManager] = [] @@ -264,6 +267,7 @@ class HiSparseCoordinator: if manager.adopt_resident_page(request_id, host_idx, block): manager.block_pool.touch([block]) state.pinned_clean.add(host_idx) + self.block_table_updates.add(request_id) def _record_copies(self, request_id: str, num_computed_tokens: int) -> None: """Index the GPU copies of just-published host blocks for later hits.""" @@ -431,6 +435,22 @@ class HiSparseCoordinator: # Host publication and spills # ------------------------------------------------------------------ + def advance_scheduled(self, requests: Iterable[tuple[str, int]]) -> None: + """Run the per-step residency work for each scheduled request. + + ``requests`` pairs a request id with the number of tokens computed + once the step completes. As in ``KVCacheCoordinator.cache_blocks``, the + last ``num_reprefillable_tokens`` are not final yet and are not written + back. + """ + if not self.resident_managers: + return + for request_id, num_tokens in requests: + self.plan_prefix_materialization( + request_id, max(0, num_tokens - self.num_reprefillable_tokens) + ) + self.update_residency(request_id) + def plan_prefix_materialization( self, request_id: str, num_computed_tokens: int ) -> None: @@ -445,9 +465,8 @@ class HiSparseCoordinator: ) state = self._get_request_state(request_id) budget = max(self.max_spill_pages - len(self.spills_to_send), 0) - for page_idx in range(num_pages): - if page_idx < importing_pages: - continue + first_page = max(state.ready_prefix_pages, importing_pages) + for page_idx in range(first_page, num_pages): if budget == 0: break if page_idx in state.pending_pages: @@ -852,7 +871,10 @@ def get_hisparse_coordinator( assert isinstance(coordinator, HiSparseCoordinator) return coordinator coordinator = HiSparseCoordinator( - kv_cache_manager.kv_cache_config, managers, kv_cache_manager.max_model_len + kv_cache_manager.kv_cache_config, + managers, + kv_cache_manager.max_model_len, + num_reprefillable_tokens=kv_cache_manager.coordinator.num_reprefillable_tokens, ) if coordinator.host_manager is None: raise ValueError("No HiSparse cache group is configured.") From a5105dba05616d74dde5d40267fc500a4422b3db Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Thu, 1 Oct 2026 18:18:37 -0400 Subject: [PATCH 219/248] [Frontend] Port Step-3.5 parsers to the streaming parser engine (#59321) Signed-off-by: sfeng33 <4florafeng@gmail.com> --- docs/features/reasoning_outputs.md | 1 + docs/features/tool_calling.md | 12 + tests/parser/engine/test_step3p5.py | 122 ++ tests/parser/engine/trace_builder.py | 15 + .../test_step3p5_reasoning_parser.py | 18 +- .../tool_parsers/test_step3p5_tool_parser.py | 69 +- vllm/parser/engine/registered_adapters.py | 6 + vllm/parser/step3p5.py | 49 + vllm/reasoning/__init__.py | 4 +- .../step3p5_engine_reasoning_parser.py | 6 + vllm/reasoning/step3p5_reasoning_parser.py | 121 -- vllm/tool_parsers/step3p5_tool_parser.py | 1460 +---------------- 12 files changed, 236 insertions(+), 1647 deletions(-) create mode 100644 tests/parser/engine/test_step3p5.py create mode 100644 vllm/parser/step3p5.py create mode 100644 vllm/reasoning/step3p5_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/step3p5_reasoning_parser.py diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md index 16f2a675f8..53f0d16245 100644 --- a/docs/features/reasoning_outputs.md +++ b/docs/features/reasoning_outputs.md @@ -29,6 +29,7 @@ vLLM currently supports the following reasoning models: | [MiniMax-M2](https://huggingface.co/MiniMaxAI/MiniMax-M2) | `minimax_m2_append_think` | `json`, `regex` | ✅ | | [Qwen3 series](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) | `qwen3` | `json`, `regex` | ✅ | | [QwQ-32B](https://huggingface.co/Qwen/QwQ-32B) | `deepseek_r1` | `json`, `regex` | ✅ | +| [Step-3.5 / Step-3.7 Flash](https://huggingface.co/stepfun-ai/Step-3.5-Flash) | `step3p5` | `json`, `regex` | ✅ | !!! note IBM Granite 3.2 and DeepSeek-V3.1 reasoning is disabled by default; to enable it, you must also pass `thinking=True` in your `chat_template_kwargs`. diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 6527eccd09..bbbc4d6bcd 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -469,6 +469,18 @@ Use the checkpoint's chat template with `--tool-call-parser mimo --reasoning-par For schema-constrained tool arguments, set `strict: true` inside each tool's `function`, or use `--tool-strict-level parameter`. The parser supports `auto`, `required`, named tools, and `parallel_tool_calls=False`, preserving leading/trailing whitespace in string arguments. +### Step-3.5 and Step-3.7 Models (`step3p5`) + +Supported models: + +* `stepfun-ai/Step-3.5-Flash` +* `stepfun-ai/Step-3.5-Flash-FP8` +* `stepfun-ai/Step-3.7-Flash` + +Flags: `--tool-call-parser step3p5 --reasoning-parser step3p5 --enable-auto-tool-choice` + +The parser supports `auto`, `required`, and named tools. + ### Olmo 3 Models (`olmo3`) Olmo 3 models output tool calls in a format that is very similar to the one expected by the `pythonic` parser (see below), with a few differences. Each tool call is a pythonic string, but the parallel tool calls are newline-delimited, and the calls are wrapped within XML tags as `..`. In addition, the parser also allows JSON boolean and null literals (`true`, `false`, and `null`) in addition to the pythonic ones (`True`, `False`, and `None`). diff --git a/tests/parser/engine/test_step3p5.py b/tests/parser/engine/test_step3p5.py new file mode 100644 index 0000000000..68f4dc55ef --- /dev/null +++ b/tests/parser/engine/test_step3p5.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based Step-3.5 parser. + +Step-3.5 reuses the Qwen3 grammar, which ``test_qwen3.py`` covers. These tests +cover what differs: thinking is always on, the newline the model emits before +```` is dropped, prior-turn reasoning does not end the current turn, +and the ``step3p5`` names resolve to the engine adapters. +""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming +from vllm.parser.engine.registered_adapters import ( + Step3p5ParserReasoningAdapter, + Step3p5ParserToolAdapter, +) +from vllm.parser.step3p5 import Step3p5Parser +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager + +_THINK_START_ID = 50 +_THINK_END_ID = 51 +_IM_START_ID = 70 +_IM_END_ID = 71 +_TEXT_ID = 100 + +_STEP3P5_VOCAB = { + "": _THINK_START_ID, + "": _THINK_END_ID, + "": 60, + "": 61, + "<|im_start|>": _IM_START_ID, + "<|im_end|>": _IM_END_ID, +} + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_STEP3P5_VOCAB) + + +@pytest.fixture +def parser(mock_tokenizer): + return Step3p5Parser(mock_tokenizer) + + +def test_parser_names_resolve_to_engine_adapters(): + tool_parser_cls = ToolParserManager.get_tool_parser("step3p5") + assert issubclass(tool_parser_cls, Step3p5ParserToolAdapter) + assert tool_parser_cls.structural_tag_model == "qwen_3_coder" + reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser("step3p5") + assert reasoning_parser_cls is Step3p5ParserReasoningAdapter + + +def test_enable_thinking_false_is_ignored(mock_tokenizer): + # The chat template always prefills "\n" and has no thinking switch. + parser = Step3p5Parser( + mock_tokenizer, chat_template_kwargs={"enable_thinking": False} + ) + assert parser.extract_reasoning("plananswer", None) == ("plan", "answer") + assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID, _THINK_START_ID]) + + +def test_newline_before_think_end_dropped(parser): + text = "Line one.\nLine two.\n\nThe answer." + assert parser.extract_reasoning(text, None) == ( + "Line one.\nLine two.", + "\nThe answer.", + ) + + +def test_newline_before_think_end_dropped_streaming(parser): + reasoning, content = simulate_reasoning_streaming( + parser, + ["Line one.\n", "Line two.\n", "", "\nThe answer."], + [(_TEXT_ID,), (_TEXT_ID,), (_THINK_END_ID,), (_TEXT_ID,)], + ) + assert reasoning == "Line one.\nLine two." + assert content == "\nThe answer." + + +class TestPriorTurnReasoning: + """The template replays earlier assistant turns as + ``\\n...\\n\\n``, so a ```` from history must not end + reasoning for the new turn (#34211).""" + + _HISTORY = [ + _IM_START_ID, + _TEXT_ID, + _THINK_START_ID, + _TEXT_ID, + _THINK_END_ID, + _TEXT_ID, + _IM_END_ID, + ] + + def test_prior_turn_think_end_not_end(self, parser): + assert not parser.is_reasoning_end([*self._HISTORY, _IM_START_ID, _TEXT_ID]) + + def test_think_end_in_current_turn_is_end(self, parser): + assert parser.is_reasoning_end( + [*self._HISTORY, _IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID] + ) + + +def test_reasoning_then_tool_call(parser, mock_request): + text = ( + "Need the weather.\n\n\n\n" + "\nTokyo\n\n\n" + ) + reasoning, content, tool_calls = parser.parse( + text, mock_request, enable_auto_tools=True + ) + assert reasoning == "Need the weather." + assert content is None + assert tool_calls is not None + assert tool_calls[0].name == "get_weather" + assert json.loads(tool_calls[0].arguments) == {"city": "Tokyo"} diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 321ce6b9e0..3fa6e05078 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -46,6 +46,7 @@ from vllm.parser.engine.registered_adapters import ( Plamo3Parser, Qwen3Parser, SeedOssParser, + Step3p5Parser, ) # ── Data structures ────────────────────────────────────────────────── @@ -449,6 +450,19 @@ def _build_mimo(scenario: Scenario, validate: bool = True) -> Sample: ) +# ── Step-3.5 (Qwen3 XML, trailing reasoning whitespace stripped) ──── + + +def _build_step3p5(scenario: Scenario, validate: bool = True) -> Sample: + return _build_qwen3( + scenario, + name="step3p5", + parser_cls=Step3p5Parser, + strip_trailing_ws=True, + validate=validate, + ) + + # ── MiniMax M2 (XML invoke format, starts in REASONING) ────────────── _MINIMAX_M2_VOCAB: dict[str, int] = { @@ -1268,6 +1282,7 @@ _BUILDERS: dict[str, Any] = { "kimi_k2": _build_kimi_k2, "qwen3": _build_qwen3, "mimo": _build_mimo, + "step3p5": _build_step3p5, "inkling": _build_inkling, "plamo3": _build_plamo3, } diff --git a/tests/reasoning/test_step3p5_reasoning_parser.py b/tests/reasoning/test_step3p5_reasoning_parser.py index a65ac080f2..0d72250f1c 100644 --- a/tests/reasoning/test_step3p5_reasoning_parser.py +++ b/tests/reasoning/test_step3p5_reasoning_parser.py @@ -4,8 +4,8 @@ import pytest from tests.reasoning.utils import run_reasoning_extraction +from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter from vllm.reasoning import ReasoningParserManager -from vllm.reasoning.step3p5_reasoning_parser import Step3p5ReasoningParser from vllm.tokenizers import get_tokenizer parser_name = "step3p5" @@ -112,7 +112,7 @@ EMPTY_STREAMING = { } NEW_LINE = { "output": "\nThis is a reasoning section\nThis is the rest", - "reasoning": "This is a reasoning section", + "reasoning": "\nThis is a reasoning section", "content": "\nThis is the rest", "is_reasoning_end": True, } @@ -126,7 +126,7 @@ NEW_LINE_STREAMING = { NEW_LINE_STREAMING_COMPLEX_CONTENT = { "output": "\n This is a \n reasoning section\n\n\n\n\nThis is the rest", - "reasoning": "\n This is a \n reasoning section\n\n", + "reasoning": "\n This is a \n reasoning section", "content": "\n\nThis is the rest", "is_reasoning_end": True, } @@ -285,7 +285,7 @@ def test_reasoning( step3p5_tokenizer.convert_tokens_to_string([token]) for token in output ] parser = ReasoningParserManager.get_reasoning_parser(parser_name)(step3p5_tokenizer) - assert isinstance(parser, Step3p5ReasoningParser) + assert isinstance(parser, Step3p5ParserReasoningAdapter) reasoning, content = run_reasoning_extraction( parser, output_tokens, streaming=streaming @@ -307,9 +307,10 @@ def test_reasoning( current_ids: list[int] = [] for token_id in output_ids: current_ids.append(token_id) - if token_id == parser.end_token_id: + if token_id in parser.reasoning_end_token_ids: assert parser.is_reasoning_end_streaming(current_ids, [token_id]) - assert parser.is_reasoning_end_streaming(output_ids, output_ids) + if param_dict["is_reasoning_end"]: + assert parser.is_reasoning_end_streaming(output_ids, output_ids) break assert not parser.is_reasoning_end_streaming(current_ids, [token_id]) else: @@ -333,6 +334,5 @@ def test_reasoning( step3p5_tokenizer.tokenize(param_dict["content"]) ) assert content_ids == expected_content_ids - else: - content_ids = parser.extract_content_ids(output_ids) - assert content_ids == [] + elif param_dict["is_reasoning_end"]: + assert parser.extract_content_ids(output_ids) == [] diff --git a/tests/tool_parsers/test_step3p5_tool_parser.py b/tests/tool_parsers/test_step3p5_tool_parser.py index 8291c62319..1e3111f5d3 100644 --- a/tests/tool_parsers/test_step3p5_tool_parser.py +++ b/tests/tool_parsers/test_step3p5_tool_parser.py @@ -847,7 +847,7 @@ TX assert chunk.tool_calls[0].function.name == "get_current_weather" assert chunk.tool_calls[0].type == "function" # Empty initially - assert chunk.tool_calls[0].function.arguments == "" + assert not chunk.tool_calls[0].function.arguments break assert header_found @@ -867,43 +867,6 @@ TX assert parsed_args["state"] == "TX" -def test_extract_tool_calls_complex_type_with_single_quote(step3p5_tokenizer): - """Test parameter type conversion based on tool schema.""" - tools = [ - ChatCompletionToolsParam( - type="function", - function={ - "name": "test_types", - "parameters": { - "type": "object", - "properties": { - "int_param": {"type": "integer"}, - "float_param": {"type": "float"}, - "bool_param": {"type": "boolean"}, - "str_param": {"type": "string"}, - "obj_param": {"type": "object"}, - }, - }, - }, - ) - ] - - model_output = """ - - -{'key': 'value'} - - -""" - - parser = Step3p5ToolParser(step3p5_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - extracted_tool_calls = parser.extract_tool_calls(model_output, request=request) - - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments) - assert args["obj_param"] == {"key": "value"} - - def test_extract_tool_calls_streaming_mixed_content_and_multiple_tool_calls( step3p5_tool_parser, step3p5_tokenizer, sample_tools ): @@ -1016,8 +979,8 @@ def test_extract_tool_calls_non_streaming_mixed_content_and_multiple_tool_calls( """Test non-streaming extraction with mixed content and multiple tool calls. Scenario: Model outputs "hello" + complete tool call + "hi" + complete tool call. - Expected: "hello" as content, first tool call parsed (index=0), "hi" as content, - second tool call parsed (index=1) + Expected: "hello" as content, first tool call parsed (index=0), second tool call + parsed (index=1). Non-streaming content stops at the first tool call. """ # Model output: hello + complete tool call + hi + complete tool call model_output = """hello @@ -1066,17 +1029,7 @@ rectangle assert args_dict_1["dimensions"]["width"] == 10 assert args_dict_1["dimensions"]["height"] == 5 - # Verify content: should contain "hello", "hi" - assert extracted_tool_calls.content is not None - assert "hello" in extracted_tool_calls.content, "Should contain 'hello' as content" - assert "hi" in extracted_tool_calls.content, "Should contain 'hi' as content" - - # Verify the order: hello should come first, then hi - hello_index = extracted_tool_calls.content.find("hello") - hi_index = extracted_tool_calls.content.find("hi") - - assert hello_index >= 0, "'hello' should be in content" - assert hi_index > hello_index, "'hi' should come after 'hello'" + assert extracted_tool_calls.content == "hello" # Verify that tool call tags are NOT in the content assert "" not in extracted_tool_calls.content, ( @@ -1155,12 +1108,14 @@ rectangle request=request, ) - # Process delta result - if delta_result: - if delta_result.content: - other_content += delta_result.content - if delta_result.tool_calls: - for tool_call in delta_result.tool_calls: + # Serving flushes the parser at the end of the stream. + for delta in (delta_result, step3p5_tool_parser.finish_streaming()): + if not delta: + continue + if delta.content: + other_content += delta.content + if delta.tool_calls: + for tool_call in delta.tool_calls: idx = tool_call.index if idx not in tool_states: tool_states[idx] = { diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 3c445c44c9..31295869c5 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -24,6 +24,7 @@ from vllm.parser.nemotron_v3 import NemotronV3Parser from vllm.parser.plamo3 import Plamo3Parser from vllm.parser.qwen3 import Qwen3Parser from vllm.parser.seed_oss import SeedOssParser +from vllm.parser.step3p5 import Step3p5Parser ( DeepSeekV32ParserReasoningAdapter, @@ -80,6 +81,11 @@ from vllm.parser.seed_oss import SeedOssParser SeedOssParserToolAdapter, ) = make_adapters(SeedOssParser) +( + Step3p5ParserReasoningAdapter, + Step3p5ParserToolAdapter, +) = make_adapters(Step3p5Parser) + ( Glm47MoeParserReasoningAdapter, Glm47MoeParserToolAdapter, diff --git a/vllm/parser/step3p5.py b/vllm/parser/step3p5.py new file mode 100644 index 0000000000..65b3212d1a --- /dev/null +++ b/vllm/parser/step3p5.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Step-3.5 / Step-3.7 parser for tool calls and reasoning. + +Both models use the Qwen3 XML tool-call grammar and ```` reasoning +unchanged. Two things differ from :class:`Qwen3Parser`: + +* The chat template always prefills ``\\n`` and has no switch to turn + thinking off, so ``enable_thinking`` is ignored. +* The model closes reasoning with ``\\n\\n``, so trailing reasoning + whitespace is stripped. +""" + +from __future__ import annotations + +import functools +from dataclasses import replace +from typing import TYPE_CHECKING + +from vllm.parser.engine.parser_engine_config import ParserEngineConfig +from vllm.parser.qwen3 import CHATML_TURN_BOUNDARIES, Qwen3Parser, qwen3_config + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + + +@functools.cache +def step3p5_config() -> ParserEngineConfig: + return replace( + qwen3_config( + thinking=True, + name="step3p5", + turn_boundary_tokens=CHATML_TURN_BOUNDARIES, + ), + strip_trailing_reasoning_whitespace=True, + ) + + +class Step3p5Parser(Qwen3Parser): + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("parser_engine_config", step3p5_config()) + super().__init__(tokenizer, tools, **kwargs) + self.thinking_enabled = True diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 736a4fcdeb..9a4413cbd9 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -157,8 +157,8 @@ _REASONING_PARSERS_TO_REGISTER = { "Step3ReasoningParser", ), "step3p5": ( - "step3p5_reasoning_parser", - "Step3p5ReasoningParser", + "step3p5_engine_reasoning_parser", + "Step3p5ParserReasoningAdapter", ), "inkling": ( "inkling_reasoning_parser", diff --git a/vllm/reasoning/step3p5_engine_reasoning_parser.py b/vllm/reasoning/step3p5_engine_reasoning_parser.py new file mode 100644 index 0000000000..25e9546a1b --- /dev/null +++ b/vllm/reasoning/step3p5_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import Step3p5ParserReasoningAdapter + +__all__ = ["Step3p5ParserReasoningAdapter"] diff --git a/vllm/reasoning/step3p5_reasoning_parser.py b/vllm/reasoning/step3p5_reasoning_parser.py deleted file mode 100644 index 5969ca5228..0000000000 --- a/vllm/reasoning/step3p5_reasoning_parser.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.generate.base.protocol import DeltaMessage -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser -from vllm.tokenizers import TokenizerLike - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - - -class Step3p5ReasoningParser(BaseThinkingReasoningParser): - """Reasoning parser for Step3p5 model. - - Step3p5 uses the ... format, but it tends to emit an extra - newline immediately before and/or after the token. This parser - trims the newline before in extract_reasoning*. - - Reasoning end detection doesn't wait for an extra token after so - structured output can constrain the first post-think token. - """ - - @property - def start_token(self) -> str: - return "" - - @property - def end_token(self) -> str: - return "" - - def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - # Used to hold a trailing "\n" from reasoning content so we can decide - # whether it is immediately before . - self._pending_reasoning_newline = False - - def extract_reasoning( - self, - model_output: str, - request: "ChatCompletionRequest | ResponsesRequest", - ) -> tuple[str | None, str | None]: - reasoning, content = super().extract_reasoning(model_output, request) - if reasoning is not None: - reasoning = reasoning.removesuffix("\n") - return reasoning or None, content or None - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - ret = super().extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - ) - - if ret is None: - return None - - # Compatibility path for models that don't generate the start token: - # treat everything before as reasoning and everything after - # as content. - if ( - self.start_token_id not in previous_token_ids - and self.start_token_id not in delta_token_ids - ): - if self.end_token_id in delta_token_ids: - end_index = delta_text.find(self.end_token) - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - ret = DeltaMessage(reasoning=reasoning, content=content or None) - elif self.end_token_id in previous_token_ids: - ret = DeltaMessage(content=delta_text) - else: - ret = DeltaMessage(reasoning=delta_text) - - reasoning_to_output = ret.reasoning - content_to_output = ret.content - - # Reasoning: handle the newline immediately before . - if reasoning_to_output is not None: - if self._pending_reasoning_newline: - reasoning_to_output = "\n" + reasoning_to_output - self._pending_reasoning_newline = False - - if reasoning_to_output.endswith("\n"): - reasoning_to_output = reasoning_to_output.removesuffix("\n") - if self.end_token in delta_text: - # Trailing "\n" is right before , drop it. - self._pending_reasoning_newline = False - else: - # Hold the trailing "\n" until we know whether follows. - self._pending_reasoning_newline = True - - # Content: handle the newline immediately after . - if content_to_output is not None: - # If we have content, reasoning must have ended. - self._pending_reasoning_newline = False - - if self.end_token in delta_text and content_to_output.startswith("\n"): - content_to_output = content_to_output.removeprefix("\n") - - reasoning_to_output = reasoning_to_output or None - content_to_output = content_to_output or None - if reasoning_to_output is None and content_to_output is None: - return None - - return DeltaMessage(reasoning=reasoning_to_output, content=content_to_output) diff --git a/vllm/tool_parsers/step3p5_tool_parser.py b/vllm/tool_parsers/step3p5_tool_parser.py index ca0d86a9e0..9d964ce314 100644 --- a/vllm/tool_parsers/step3p5_tool_parser.py +++ b/vllm/tool_parsers/step3p5_tool_parser.py @@ -1,1464 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import json -from collections.abc import Sequence -from typing import Any -from xml.parsers.expat import ParserCreate -import regex as re +from vllm.parser.engine.registered_adapters import Step3p5ParserToolAdapter -from vllm.entrypoints.chat_utils import make_tool_call_id -from vllm.entrypoints.generate.base.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool, ToolParser -from vllm.tool_parsers.utils import ( - find_tool_name, - find_tool_properties, - safe_literal_eval, -) -logger = init_logger(__name__) - - -class StreamingXMLToolCallParser: - """Simplified streaming XML tool call parser - Supports streaming input, parsing, and output - """ - - def __init__(self): - self.reset_streaming_state() - - # Tool configuration information - self.tools: list[Tool] | None = None - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - self.function_start_token: str = " DeltaMessage: - """Parse single streaming XML chunk and return Delta response - This is the actual streaming interface that receives chunks - one by one and maintains internal state - - Args: - xml_chunk: Single XML chunk string - Returns: - DeltaMessage: Contains delta information generated by this chunk, - returns empty response if no complete elements - - """ - # Record delta count before processing - initial_delta_count = len(self.deltas) - entry_call_id = self.current_call_id - entry_tool_call_index = self.tool_call_index - - self.streaming_buffer += xml_chunk - - found_elements = self._process_complete_xml_elements() - - fallback_call_id = None - if entry_call_id is not None: - if ( - self.current_call_id == entry_call_id - and self.tool_call_index == entry_tool_call_index - ): - fallback_call_id = entry_call_id - elif ( - self.current_call_id is not None - and self.tool_call_index == entry_tool_call_index + 1 - ): - fallback_call_id = self.current_call_id - - if found_elements: - # If complete elements found, check if end events were missed - # some tags may not have been triggered - try: - new_deltas = self.deltas[initial_delta_count:] - # If this chunk contains - # but didn't generate '}', then complete it - if ( - fallback_call_id is not None - and self.function_end_token in xml_chunk - ): - # - Added '}' (non-empty parameter ending) - # - Added '{}' (empty parameter function) - has_function_close = any( - ( - td.tool_calls - and any( - ( - tc.function - and tc.id == fallback_call_id - and isinstance(tc.function.arguments, str) - and (tc.function.arguments in ("}", "{}")) - ) - for tc in td.tool_calls - ) - ) - for td in new_deltas - ) - if not has_function_close: - # Close potentially unclosed element - if self.current_param_name: - self._end_element("parameter") - if self.current_function_name: - self._end_element("function") - # If this chunk contains - # but didn't generate final empty delta, then complete it - if ( - fallback_call_id is not None - and self.tool_call_end_token in xml_chunk - ): - has_toolcall_close = any( - ( - td.tool_calls - and any( - ( - tc.type == "function" - and tc.function - and tc.function.arguments == "" - and tc.id == fallback_call_id - ) - for tc in td.tool_calls - ) - ) - for td in new_deltas - ) - if not has_toolcall_close: - # Close potentially unclosed element - if self.current_param_name: - self._end_element("parameter") - if self.current_function_name: - self._end_element("function") - self._end_element("tool_call") - except Exception as e: - logger.warning("Error with fallback parsing: %s", e) - # Merge newly generated deltas into single response - result_delta = self._merge_new_deltas_to_single_response( - initial_delta_count - ) - return result_delta - else: - # No complete elements, check if there's unoutput text content - if self.text_content_buffer and self.tool_call_index == 0: - # Has text content but no tool_call yet, output text content - text_delta = DeltaMessage(content=self.text_content_buffer) - self._emit_delta(text_delta) - # Clear buffer to avoid duplicate output - self.text_content_buffer = "" - return text_delta - - # If this chunk contains end tags but wasn't triggered by parser, - # manually complete end events - # Only execute when still on the same call as when entered, - # to prevent accidentally closing new calls - # in multi scenarios - if fallback_call_id is not None and ( - self.function_end_token in xml_chunk - or self.tool_call_end_token in xml_chunk - ): - # Close potentially unclosed element - if self.current_param_name: - self._end_element("parameter") - if self.function_end_token in xml_chunk and self.current_function_name: - self._end_element("function") - if self.tool_call_end_token in xml_chunk: - self._end_element("tool_call") - # Return the merged delta result generated by this fallback - result_delta = self._merge_new_deltas_to_single_response( - initial_delta_count - ) - return result_delta - - # No complete elements, return empty response - return DeltaMessage(content=None) - - def _escape_xml_special_chars(self, text: str) -> str: - """Escape XML special characters - Args: - text: Original text - Returns: - Escaped text - """ - xml_escapes = { - "&": "&", - "<": "<", - ">": ">", - '"': """, - "'": "'", - } - - for char, escape in xml_escapes.items(): - text = text.replace(char, escape) - - return text - - def _process_complete_xml_elements(self) -> bool: - """Process complete XML elements in buffer. - - Returns: - bool: Whether complete elements were found and processed - - """ - found_any = False - - while self.last_processed_pos < len(self.streaming_buffer): - # Find next complete xml element - element, end_pos = self._find_next_complete_element(self.last_processed_pos) - if element is None: - # No complete element found, wait for more data - break - - # Check if this element should be skipped - if self._should_skip_element(element): - self.last_processed_pos = end_pos - continue - - # Found complete XML element, process it - try: - preprocessed_element = self._preprocess_xml_chunk(element) - # Check if this is the first tool_call start - if ( - ( - preprocessed_element.strip().startswith("") - or preprocessed_element.strip().startswith("") - and self.tool_call_index > 0 - and self.current_call_id - and self.current_function_name - ): - # Reset parser state but preserve generated deltas - if self.current_param_name: - self._end_element("parameter") - if self.current_function_open: - self._end_element("function") - # Output final tool_call tail delta - final_delta = DeltaMessage( - role=None, - content=None, - reasoning=None, - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments=""), - ) - ], - ) - self._emit_delta(final_delta) - # Reset XML parser and current call state - self._reset_xml_parser_after_tool_call() - # Parse preprocessed element - self.parser.Parse(preprocessed_element, False) - found_any = True - - except Exception as e: - logger.warning("Error when parsing XML elements: %s", e) - - # Update processed position - self.last_processed_pos = end_pos - - return found_any - - def _fix_incomplete_tag_in_chunk(self, chunk: str) -> str: - """Fallback: fix incomplete ) - Examples: , - Also handles missing = cases: -> , - -> - Only fixes tags that pass validation (parameter exists in tool definition) - """ - # First, handle missing = cases for function tags - chunk = self._fix_missing_equals_in_function_tag(chunk) - - for tag_type in ["parameter", "function"]: - pattern = f"<{tag_type}=" - if pattern not in chunk: - continue - - start_idx = chunk.find(pattern) - after_tag = chunk[start_idx:] - gt_pos = after_tag.find(">") - lt_pos = after_tag.find("<", len(pattern)) - - # Skip if already well-formed - if ( - gt_pos != -1 - and (lt_pos == -1 or gt_pos < lt_pos) - and pattern in after_tag[:gt_pos] - ): - continue - - # Extract tag name (stop at space, newline, or <) - content = chunk[start_idx + len(pattern) :] - end_pos = next( - (i for i, ch in enumerate(content) if ch in (" ", "\n", "<")), - len(content), - ) - tag_name = content[:end_pos] - - if not tag_name: - continue - - # Remove duplicate prefix: ", 1 - ) - - return chunk - - def _fix_missing_equals_in_function_tag(self, chunk: str) -> str: - """Fix missing = in function tags: or - Examples: - -> - -> - Only fixes if function name exists in tool definition - """ - # already correct - if " (with space/newline but no =) - pattern1 = r"" - match1 = re.search(pattern1, chunk) - if match1: - func_name = match1.group(1).strip() - # must validate function name exists before fixing - if func_name and self._validate_function_name(func_name): - original = match1.group(0) - fixed = f"" - chunk = chunk.replace(original, fixed, 1) - return chunk - - # Pattern 2: (no space, no =) - # only match " - match2 = re.search(pattern2, chunk) - if match2: - func_name = match2.group(1).strip() - # must validate function name exists before fixing - if func_name and self._validate_function_name(func_name): - original = match2.group(0) - fixed = f"" - chunk = chunk.replace(original, fixed, 1) - return chunk - - return chunk - - def _validate_function_name(self, func_name: str) -> bool: - """Check if function name exists in tool definitions.""" - return find_tool_name(self.tools, func_name) - - def _validate_parameter_name(self, param_name: str) -> bool: - """Check if parameter exists in current function's tool definition.""" - if not self.tools or not self.current_function_name: - return True - properties = find_tool_properties(self.tools, self.current_function_name) - return not properties or param_name in properties - - def _should_skip_element(self, element: str) -> bool: - """Determine whether an element should be skipped. - - Args: - element: Element to evaluate - - Returns: - bool: True means should skip, False means should process - - """ - # If it's a tool_call XML tag, don't skip - if ( - element.startswith(self.tool_call_start_token) - or element.startswith(self.function_start_token) - or element.startswith(self.parameter_start_token) - ): - return False - - # If currently not parsing tool calls and not blank, - # collect this text instead of skipping - # Only process other XML elements after tool_call appears, - # otherwise treat as plain text - if self.current_call_id is None and element: - # Collect text content to buffer - self.text_content_buffer += element - return True # Still skip, but content has been collected - - # If currently parsing tool calls, - # this might be parameter value, don't skip - if self.current_call_id is not None: - return False - - # Skip blank content - return not element - - def _find_next_complete_element(self, start_pos: int) -> tuple[str | None, int]: - """Find next complete XML element from specified position. - - Args: - start_pos: Position to start searching - - Returns: - (Complete element string, element end position), - returns (None, start_pos) if no complete element found - - """ - buffer = self.streaming_buffer[start_pos:] - - if not buffer: - return None, start_pos - - if buffer.startswith("<"): - # Check if this is an incomplete parameter/function tag - # e.g., " not in buffer.split("\n")[0] - ) - is_incomplete_func = ( - buffer.startswith("" not in buffer.split("\n")[0] - ) - - if is_incomplete_param or is_incomplete_func: - # Find the corresponding closing tag - tag_type = "parameter" if is_incomplete_param else "function" - closing_tag = f"" - closing_pos = buffer.find(closing_tag) - - if closing_pos != -1: - # Found closing tag, return complete element including closing tag - complete_element = buffer[: closing_pos + len(closing_tag)] - return complete_element, start_pos + closing_pos + len(closing_tag) - - # Need to ensure no new < appears, - # find the nearest one between < and > - tag_end = buffer.find("<", 1) - tag_end2 = buffer.find(">", 1) - if tag_end != -1 and tag_end2 != -1: - # Next nearest is < - if tag_end < tag_end2: - return buffer[:tag_end], start_pos + tag_end - # Next nearest is >, means found XML element - else: - return buffer[: tag_end2 + 1], start_pos + tag_end2 + 1 - elif tag_end != -1: - return buffer[:tag_end], start_pos + tag_end - elif tag_end2 != -1: - return buffer[: tag_end2 + 1], start_pos + tag_end2 + 1 - else: - # If currently not parsing tool calls (entering a tool_call), - # check if starts with or - if buffer == ""[: len(buffer)]: - # Might be start of , wait for more data - return None, start_pos - elif ( - buffer.startswith(" DeltaMessage: - """Merge newly generated deltas from this processing - into a single DeltaMessage - - Args: - initial_count: Delta count before processing - - Returns: - Merged DeltaMessage containing all newly generated delta information - - """ - if len(self.deltas) <= initial_count: - return DeltaMessage(content=None) - - # Get newly generated deltas - new_deltas = self.deltas[initial_count:] - - if len(new_deltas) == 1: - # Only one new delta, return directly - return new_deltas[0] - - # Merge multiple new deltas - merged_tool_calls: list[DeltaToolCall] = [] - merged_content: str = "" - - for delta in new_deltas: - if delta.content: - merged_content += delta.content - if delta.tool_calls: - # For tool_calls, we need to intelligently merge arguments - for tool_call in delta.tool_calls: - # Find if there's already a tool_call with the same call_id - existing_call = None - for existing in merged_tool_calls: - if existing.id == tool_call.id: - existing_call = existing - break - - if existing_call and existing_call.function: - # Merge to existing tool_call - if tool_call.function and tool_call.function.name: - existing_call.function.name = tool_call.function.name - if ( - tool_call.function - and tool_call.function.arguments is not None - ): - if existing_call.function.arguments is None: - existing_call.function.arguments = "" - - # For streaming JSON parameters, - # simply concatenate in order - new_args = tool_call.function.arguments - existing_call.function.arguments += new_args - if tool_call.type: - existing_call.type = tool_call.type - else: - # Add new tool_call - merged_tool_calls.append(tool_call) - - return DeltaMessage( - content=merged_content if merged_content else None, - tool_calls=merged_tool_calls, - ) - - def _preprocess_xml_chunk(self, chunk: str) -> str: - """Preprocess XML chunk, handle non-standard formats, - and escape special characters - - Args: - chunk: Original XML chunk - - Returns: - Processed XML chunk - - """ - # Check if this is a tool_call related element - is_tool_call = False - if chunk.startswith(self.tool_call_start_token) or chunk.startswith( - self.tool_call_end_token - ): - is_tool_call = True - # Check for function tags (including malformed ones without =) - # , , , - if ( - chunk.startswith(self.function_start_token) - or chunk.startswith(self.function_end_token) - or chunk.startswith(" - # This handles cases like: format -> - processed = re.sub(r"]+)>", r'', chunk) - # Handle format -> - processed = re.sub(r"]+)>", r'', processed) - - original_chunk = chunk - # If in parameter value accumulation mode - if self._pre_inside_parameter: - # Parameter end: output accumulated raw text - # safely then return - if processed.startswith(""): - body_text = self._pre_param_buffer - # Trigger deferred parsing mode - # literal_eval+json output in end_element - self.defer_current_parameter = True - self.deferred_param_raw_value = body_text - # Clean up state - self._pre_inside_parameter = False - self._pre_param_buffer = "" - self._pre_current_param_name = None - safe_text = self._escape_xml_special_chars(body_text) - return f"{safe_text}" - else: - # If this is the first block of content after entering parameter - # evaluate if deferred parsing is needed; - # If not needed, exit accumulation mode - # and pass through directly - if self._pre_param_buffer == "": - # Get current parameter type - param_type = ( - self._get_param_type(self._pre_current_param_name) - if self._pre_current_param_name - else "string" - ) - # Only these types need deferred parsing to - # handle Python literals containing single quotes - is_object_type = param_type in ["object"] - is_complex_type = ( - param_type in ["array", "arr", "sequence"] - or param_type.startswith("dict") - or param_type.startswith("list") - ) - - # Only delay when contains container symbols - # and has single quotes and is complex type - has_container_hint = ( - ("[" in original_chunk) - or ("{" in original_chunk) - or ("(" in original_chunk) - ) - - # Determine if deferred parsing is needed - need_defer = False - if is_complex_type: - # Complex type, always need deferred parsing - need_defer = True - elif ( - is_object_type - and has_container_hint - and ("'" in original_chunk) - ): - # Object type with container symbols - # and single quotes, need deferred parsing - need_defer = True - - if not need_defer: - # No need for deferred parsing, - # exit parameter mode directly - self._pre_inside_parameter = False - return self._escape_xml_special_chars(original_chunk) - self._pre_param_buffer += original_chunk - return "" - - # Parameter start: enable accumulation - if processed.startswith("', processed) - if m: - self._pre_current_param_name = m.group(1) - self._pre_inside_parameter = True - self._pre_param_buffer = "" - return processed - - # If processed doesn't contain special_token, escape processed - # This is because XML parsing encounters special characters - # and reports errors, so escaping is needed - if not is_tool_call: - processed = self._escape_xml_special_chars(processed) - return processed - - def _emit_delta(self, delta: DeltaMessage): - """Emit Delta response (streaming output).""" - self.deltas.append(delta) - - def _auto_close_open_parameter_if_needed(self, incoming_tag: str | None = None): - """Before starting to process new elements, - if there are unclosed tags from before, - automatically complete their endings to the parser. - - If there are unclosed parameters, - it's equivalent to feeding `` - - When about to start a new function or tool_call, - if there are unclosed functions, complete ``. - - When about to start a new tool_call, - if there are unclosed tool_calls, complete ``. - """ - # First close unclosed parameters - if self.current_param_name: - self._end_element("parameter") - - # If about to start new function or tool_call, - # and there are unclosed functions, close function first - if incoming_tag in ("function", "tool_call") and self.current_function_name: - self._end_element("function") - - # If about to start new tool_call, - # and there are unclosed tool_calls, close tool_call first - if incoming_tag == "tool_call" and self.current_call_id: - self._end_element("tool_call") - - def _start_element(self, name: str, attrs: dict[str, str]): - """Handle XML start element events.""" - if name == "root": - return - - if name == "tool_call": - # Before opening new tool_call, - # automatically complete previous unclosed tags - self._auto_close_open_parameter_if_needed("tool_call") - - self.parameters = {} - self.current_call_id = make_tool_call_id() - self.current_param_is_first = True - self.tool_call_index += 1 - elif name.startswith("function") or (name == "function"): - # If missing tool_call, manually complete - if not self.current_call_id: - self._start_element("tool_call", {}) - # Before opening new function, - # automatically complete previous unclosed tags (parameter/function) - self._auto_close_open_parameter_if_needed("function") - function_name = self._extract_function_name(name, attrs) - self.current_function_name = function_name - self.current_function_open = True - if function_name: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall( - name=function_name, arguments="" - ), - ) - ] - ) - self._emit_delta(delta) - elif name.startswith("parameter") or (name == "parameter"): - # If previous parameter hasn't ended normally, - # complete its end first, then start new parameter - self._auto_close_open_parameter_if_needed("parameter") - param_name = self._extract_parameter_name(name, attrs) - self.current_param_name = param_name - self.current_param_value = "" - self.current_param_value_converted = "" - self.start_quote_emitted = False # Reset start quote flag - - # Only output parameter name and colon, - # don't output quotes - # decide after parameter value type is determined - if param_name: - if not self.parameters: - # First parameter - # start JSON, only output parameter name and colon - json_start = f'{{"{param_name}": ' - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall( - name=None, arguments=json_start - ), - ) - ] - ) - self._emit_delta(delta) - self.current_param_is_first = True - else: - # Subsequent parameters - # add comma and parameter name, no quotes - json_continue = f', "{param_name}": ' - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall( - name=None, arguments=json_continue - ), - ) - ] - ) - self._emit_delta(delta) - self.current_param_is_first = False - - def _char_data(self, data: str): - """Handle XML character data events.""" - if data and self.current_param_name: - # If preprocessing stage determines deferred parsing is needed, - # only cache character data, no streaming output - if self.defer_current_parameter: - original_data = data - if self.should_emit_end_newline: - original_data = "\n" + original_data - self.should_emit_end_newline = False - if original_data.endswith("\n"): - self.should_emit_end_newline = True - original_data = original_data[:-1] - self.current_param_value += original_data - return - - param_type = self._get_param_type(self.current_param_name) - - # Check if this is the first time receiving data for this parameter - # If this is the first packet of data and starts with \n, remove \n - if not self.current_param_value and data.startswith("\n"): - data = data[1:] - - # Output start quote for string type (if not already output) - if ( - param_type in ["string", "str", "text", "varchar", "char", "enum"] - and not self.start_quote_emitted - ): - quote_delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments='"'), - ) - ] - ) - self._emit_delta(quote_delta) - self.start_quote_emitted = True - - if not data: - return - - original_data = data - # Delay output of trailing newline - if self.should_emit_end_newline: - original_data = "\n" + original_data - self.should_emit_end_newline = False - if original_data.endswith("\n"): - self.should_emit_end_newline = True - original_data = original_data[:-1] - self.current_param_value += original_data - - # convert parameter value by param_type - converted_value = self._convert_param_value( - self.current_param_value, param_type - ) - output_data = self._convert_for_json_streaming(converted_value, param_type) - - delta_data = output_data[len(self.current_param_value_converted) :] - self.current_param_value_converted = output_data - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments=delta_data), - ) - ] - ) - self._emit_delta(delta) - - def _end_element(self, name: str): - """Handle XML end element events.""" - if name == "root": - return - - # If function or tool_call ends and there are still unclosed parameters, - # complete parameter end first - if ( - name.startswith("function") or name == "function" or name == "tool_call" - ) and self.current_param_name: - self._auto_close_open_parameter_if_needed() - - if ( - name.startswith("parameter") or name == "parameter" - ) and self.current_param_name: - # End current parameter - param_name = self.current_param_name - param_value = self.current_param_value - - # If in deferred parsing mode, - # perform overall parsing on raw content - # accumulated in preprocessing stage and output once - if self.defer_current_parameter: - raw_text = ( - self.deferred_param_raw_value - if self.deferred_param_raw_value - else param_value - ) - parsed_value = None - output_arguments = None - try: - # If previously delayed trailing newline, - # add it back before parsing - if self.should_emit_end_newline: - raw_for_parse = raw_text + "\n" - else: - raw_for_parse = raw_text - parsed_value = safe_literal_eval(raw_for_parse) - output_arguments = json.dumps(parsed_value, ensure_ascii=False) - except Exception: - # Fallback: output as string as-is - output_arguments = json.dumps(raw_text, ensure_ascii=False) - parsed_value = raw_text - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall( - name=None, arguments=output_arguments - ), - ) - ] - ) - self._emit_delta(delta) - - # Clean up and store - self.should_emit_end_newline = False - self.parameters[param_name] = parsed_value - self.current_param_name = None - self.current_param_value = "" - self.current_param_value_converted = "" - self.start_quote_emitted = False - self.defer_current_parameter = False - self.deferred_param_raw_value = "" - return - - param_type = self._get_param_type(param_name) - - # convert complete parameter value by param_type - converted_value = self._convert_param_value(param_value, param_type) - - # Decide whether to add end quote based on parameter type - if param_type in ["string", "str", "text", "varchar", "char", "enum"]: - # For empty string parameters, need special handling - if not param_value and not self.start_quote_emitted: - # No start quote output, - # directly output complete empty string - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments='""'), - ) - ] - ) - self._emit_delta(delta) - else: - # Non-empty parameter value, output end quote - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments='"'), - ) - ] - ) - self._emit_delta(delta) - - self.should_emit_end_newline = False - # Store converted value - self.parameters[param_name] = converted_value - self.current_param_name = None - self.current_param_value = "" - self.current_param_value_converted = "" - self.start_quote_emitted = False - - elif name.startswith("function") or name == "function": - # if there are parameters, close JSON object - if self.parameters: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments="}"), - ) - ] - ) - self._emit_delta(delta) - # return empty object - else: - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments="{}"), - ) - ] - ) - self._emit_delta(delta) - self.current_function_open = False - self.current_function_name = ( - None # Clear function name to prevent duplicate closing - ) - - elif name == "tool_call": - # Before ending tool_call, - # ensure function is closed to complete missing right brace - if self.current_function_open: - # If there are still unclosed parameters, close them first - if self.current_param_name: - self._end_element("parameter") - # Close function, ensure output '}' or '{}' - self._end_element("function") - # Final Delta - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.tool_call_index - 1, - id=self.current_call_id, - type="function", - function=DeltaFunctionCall(name=None, arguments=""), - ) - ] - ) - self._emit_delta(delta) - - # Check if there's text content to output (between tool_calls) - if self.text_content_buffer.strip(): - text_delta = DeltaMessage(content=self.text_content_buffer) - self._emit_delta(text_delta) - - self._reset_xml_parser_after_tool_call() - - def setup_parser(self): - """Set up XML parser event handlers.""" - self.parser.buffer_text = True - self.parser.StartElementHandler = self._start_element - self.parser.EndElementHandler = self._end_element - self.parser.CharacterDataHandler = self._char_data - - def set_tools(self, tools: list[Tool] | None): - """Set tool configuration information.""" - self.tools = tools - - def _extract_function_name(self, name: str, attrs: dict[str, str]) -> str | None: - """Extract function name from various formats.""" - if attrs and "name" in attrs: - return attrs["name"] - - if "=" in name: - parts = name.split("=", 1) - if len(parts) == 2 and parts[0] == "function": - return parts[1] - - return None - - def _extract_parameter_name(self, name: str, attrs: dict[str, str]) -> str | None: - """Extract parameter name from various formats.""" - if attrs and "name" in attrs: - return attrs["name"] - - if "=" in name: - parts = name.split("=", 1) - if len(parts) == 2 and parts[0] == "parameter": - return parts[1] - - return None - - def _get_param_type(self, param_name: str) -> str: - """Get parameter type based on tool configuration, defaults to string - Args: - param_name: Parameter name - - Returns: - Parameter type - - """ - if not self.tools or not self.current_function_name: - return "string" - properties = find_tool_properties(self.tools, self.current_function_name) - param_config = properties.get(param_name) - if isinstance(param_config, dict): - return self.repair_param_type(str(param_config.get("type", "string"))) - return "string" - - def repair_param_type(self, param_type: str) -> str: - """Repair unknown parameter types by treating them as string - Args: - param_type: Parameter type - - Returns: - Repaired parameter type - - """ - if ( - param_type in ["string", "str", "text", "varchar", "char", "enum"] - or param_type.startswith("int") - or param_type.startswith("uint") - or param_type.startswith("long") - or param_type.startswith("short") - or param_type.startswith("unsigned") - or param_type.startswith("num") - or param_type.startswith("float") - or param_type in ["boolean", "bool", "binary"] - or ( - param_type in ["object", "array", "arr", "sequence"] - or param_type.startswith("dict") - or param_type.startswith("list") - ) - ): - return param_type - else: - return "string" - - def _convert_param_value(self, param_value: str, param_type: str) -> Any: - """Convert value based on parameter type - Args: - param_value: Parameter value - param_type: Parameter type - - Returns: - Converted value - - """ - if param_value.lower() == "null": - return None - - param_type = param_type.strip().lower() - if param_type in ["string", "str", "text", "varchar", "char", "enum"]: - return param_value - elif ( - param_type.startswith("int") - or param_type.startswith("uint") - or param_type.startswith("long") - or param_type.startswith("short") - or param_type.startswith("unsigned") - ): - try: - return int(param_value) - except (ValueError, TypeError): - logger.warning( - "Parsed value '%s' is not an integer, degenerating to string.", - param_value, - ) - return param_value - elif param_type.startswith("num") or param_type.startswith("float"): - try: - float_param_value: float = float(param_value) - return ( - float_param_value - if float_param_value - int(float_param_value) != 0 - else int(float_param_value) - ) - except (ValueError, TypeError): - logger.warning( - "Parsed value '%s' is not a float, degenerating to string.", - param_value, - ) - return param_value - elif param_type in ["boolean", "bool", "binary"]: - param_value = param_value.lower() - return param_value == "true" - else: - return param_value - - def _convert_for_json_streaming(self, converted_value: Any, param_type: str) -> str: - """Convert converted_value based on - whether it's empty and if type is string - Args: - converted_value: Converted value - param_type: Parameter type - - Returns: - Converted string for streaming output - - """ - # Check if value is empty, but exclude numeric 0 - if converted_value is None or converted_value == "": - return "" - - if param_type in ["string", "str", "text", "varchar", "char", "enum"]: - # String type, remove double quotes - return json.dumps(converted_value, ensure_ascii=False)[1:-1] - else: - # Non-string type, return complete JSON string - if not isinstance(converted_value, str): - return json.dumps(converted_value, ensure_ascii=False) - else: - return converted_value - - def _reset_xml_parser_after_tool_call(self): - """Each tool_call is treated as a separate XML document, - so we need to reset the parser after each tool_call. - """ - # recreate XML parser - self.parser = ParserCreate() - self.setup_parser() - - # Reset current tool_call state - if self.current_call_id: - self.last_completed_call_id = self.current_call_id - self.current_call_id = None - self.current_function_name = None - self.current_function_open = False - self.parameters = {} - self.current_param_name = None - self.current_param_value = "" - self.current_param_value_converted = "" - self.current_param_is_first = False - self.should_emit_end_newline = False - self.start_quote_emitted = False - self.text_content_buffer = "" - - # Reset preprocessing and deferred parsing state - self._pre_inside_parameter = False - self._pre_param_buffer = "" - self._pre_current_param_name = None - self.defer_current_parameter = False - self.deferred_param_raw_value = "" - - -class Step3p5ToolParser(ToolParser): - # Step-3.5/3.7 chat templates use the Qwen3-Coder XML tool-call format. +class Step3p5ToolParser(Step3p5ParserToolAdapter): # type: ignore[valid-type, misc] structural_tag_model = "qwen_3_coder" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - self.parser = StreamingXMLToolCallParser() - - # Add missing attributes for compatibility with serving_chat.py - self.prev_tool_call_arr: list[dict] = [] - self.streamed_args_for_tool: list[str] = [] - - logger.info( - "vLLM Successfully import tool parser %s !", self.__class__.__name__ - ) - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - self.parser.reset_streaming_state() - # Reset tool call tracking arrays for new extraction - self.prev_tool_call_arr = [] - self.streamed_args_for_tool = [] - self.parser.set_tools(self.tools) - result = self.parser.parse_single_streaming_chunks(model_output) - if not result.tool_calls: - return ExtractedToolCallInformation( - tool_calls=[], - tools_called=False, - content=result.content, - ) - else: - tool_calls = [] - for tool_call in result.tool_calls: - if tool_call.function and tool_call.function.name: - tool_calls.append( - ToolCall( - id=tool_call.id, - type=tool_call.type, - function=FunctionCall( - name=tool_call.function.name, - arguments=tool_call.function.arguments, - ), - ) - ) - - # Update tool call tracking arrays for compatibility - tool_index = ( - tool_call.index - if tool_call.index is not None - else len(self.prev_tool_call_arr) - 1 - ) - - # Ensure we have enough entries in our tracking arrays - while len(self.prev_tool_call_arr) <= tool_index: - self.prev_tool_call_arr.append({"name": "", "arguments": ""}) - while len(self.streamed_args_for_tool) <= tool_index: - self.streamed_args_for_tool.append("") - - # Update tool call information - self.prev_tool_call_arr[tool_index]["name"] = ( - tool_call.function.name - ) - self.prev_tool_call_arr[tool_index]["arguments"] = ( - tool_call.function.arguments - ) - - # Update streamed arguments - if tool_call.function.arguments: - self.streamed_args_for_tool[tool_index] = ( - tool_call.function.arguments - ) - - return ExtractedToolCallInformation( - tool_calls=tool_calls, - tools_called=len(tool_calls) > 0, - content=result.content, - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if not previous_text: - self.parser.reset_streaming_state() - # Reset tool call tracking arrays for new streaming session - self.prev_tool_call_arr = [] - self.streamed_args_for_tool = [] - self.parser.set_tools(self.tools) - - # Model sometimes outputs separately causing delta_text to be empty. - # If there were tool_calls before and all current tool_calls have ended, - # return an empty tool_call for outer streaming output - # to correctly output tool_call field - if not delta_text and delta_token_ids: - open_calls = current_text.count( - self.parser.tool_call_start_token - ) - current_text.count(self.parser.tool_call_end_token) - if ( - open_calls == 0 - and self.parser.tool_call_index > 0 - or not self.parser.tool_call_index - and current_text - ): - return DeltaMessage(content="") - return None - - # Parse the delta text and get the result - result = self.parser.parse_single_streaming_chunks(delta_text) - - # Update tool call tracking arrays based on incremental parsing results - if result and result.tool_calls: - for tool_call in result.tool_calls: - if tool_call.function: - tool_index = ( - tool_call.index - if tool_call.index is not None - else len(self.prev_tool_call_arr) - 1 - ) - - # Ensure we have enough entries in our tracking arrays - while len(self.prev_tool_call_arr) <= tool_index: - self.prev_tool_call_arr.append({"name": "", "arguments": ""}) - while len(self.streamed_args_for_tool) <= tool_index: - self.streamed_args_for_tool.append("") - - # Update tool name if provided - if tool_call.function.name: - self.prev_tool_call_arr[tool_index]["name"] = ( - tool_call.function.name - ) - - # Update arguments incrementally - if tool_call.function.arguments is not None: - # Concatenate the incremental arguments - # to the existing streamed arguments - self.prev_tool_call_arr[tool_index]["arguments"] += ( - tool_call.function.arguments - ) - self.streamed_args_for_tool[tool_index] += ( - tool_call.function.arguments - ) - return result - - def parser_should_check_for_unstreamed_tool_arg_tokens(self) -> bool: - """Skip the remaining_call calculation in serving_chat.""" - return False From d2125ea681784c17251c47939f5ed6d1c67384f8 Mon Sep 17 00:00:00 2001 From: Aarushi Jain <142941703+aarushjain29@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:30:13 -0500 Subject: [PATCH 220/248] [ROCm][CI] Raise the MI355 DeepSeek-R1 GSM8K startup wait to 1800s (#59666) Signed-off-by: aarushjain29 --- tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml | 2 +- tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml index a482c47bff..0f6e87f8c2 100644 --- a/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-DP_MI355.yaml @@ -2,7 +2,7 @@ model_name: "deepseek-ai/DeepSeek-R1" accuracy_threshold: 0.95 num_questions: 1319 num_fewshot: 5 -startup_max_wait_seconds: 1200 +startup_max_wait_seconds: 1800 env: VLLM_ROCM_USE_AITER: "1" VLLM_ROCM_USE_AITER_MLA: "1" diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml index 74776f46b6..44476c385a 100644 --- a/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-TP_MI355.yaml @@ -2,7 +2,7 @@ model_name: "deepseek-ai/DeepSeek-R1" accuracy_threshold: 0.95 num_questions: 1319 num_fewshot: 5 -startup_max_wait_seconds: 1200 +startup_max_wait_seconds: 1800 env: VLLM_ROCM_USE_AITER: "1" VLLM_ROCM_USE_AITER_MLA: "1" From e35082e7b627f403b36488cc68a21e291b8d6c8a Mon Sep 17 00:00:00 2001 From: hcl Date: Fri, 2 Oct 2026 07:35:58 +0800 Subject: [PATCH 221/248] [Bugfix][CLI] Include inherited field docstrings in get_attr_docs (#49821) Signed-off-by: Chenglun Hu Signed-off-by: hclsys Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tests/config/test_config_utils.py | 41 +++++++++++++++++++++++++++++-- vllm/config/utils.py | 27 ++++++++++++++++---- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index b08c1987c4..e209815380 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -8,7 +8,12 @@ import pytest from vllm.config.cache import CacheConfig from vllm.config.scheduler import SchedulerConfig -from vllm.config.utils import get_hash_factors, hash_factors, normalize_value +from vllm.config.utils import ( + get_attr_docs, + get_hash_factors, + hash_factors, + normalize_value, +) # Helpers @@ -114,7 +119,7 @@ def test_normalize_value_uuid_and_to_json(): [ (lambda x: x), (type("CallableInstance", (), {"__call__": lambda self: 0}))(), - (lambda: (lambda: 0))(), # nested function instance + (lambda: lambda: 0)(), # nested function instance ], ) def test_error_cases(bad): @@ -190,6 +195,31 @@ def test_ir_config_hash_initializes_provider_and_tracks_implementation(monkeypat assert config.compute_hash() != first_hash +def test_get_attr_docs_includes_inherited_fields(): + """get_attr_docs must collect attribute docstrings inherited from base + classes, not just those declared in the class's own body.""" + + @dataclass + class Base: + inherited: object = None + """doc for inherited (base).""" + overridden: object = None + """base version of overridden.""" + + @dataclass + class Derived(Base): + own: object = None + """doc for own (derived).""" + overridden: object = None + """derived override of overridden.""" + + docs = get_attr_docs(Derived) + + assert docs["own"] == "doc for own (derived)." + assert docs["inherited"] == "doc for inherited (base)." + assert docs["overridden"] == "derived override of overridden." + + def test_envs_compile_factors_stable(): """Test that envs.compile_factors() hash is stable across fresh initializations. @@ -325,3 +355,10 @@ print(hash_factors(envs.compile_factors())) "HOME relocation changed the compile-cache env hash - a " "location-only derived var is leaking into the key" ) + + +def test_get_attr_docs_rejects_non_class(): + """The annotation forbids it, but the runtime guard still protects + untyped callers, so keep asserting the documented TypeError.""" + with pytest.raises(TypeError, match="Given object was not a class."): + get_attr_docs(object()) # type: ignore[arg-type] diff --git a/vllm/config/utils.py b/vllm/config/utils.py index c3d7bb1056..3b2ad88c87 100644 --- a/vllm/config/utils.py +++ b/vllm/config/utils.py @@ -158,17 +158,20 @@ def getattr_iter( return default_factory() if default_factory is not None else default -def get_attr_docs(cls: type[Any]) -> dict[str, str]: +def _get_own_attr_docs(cls: type[Any], out: dict[str, str]) -> None: """Get any docstrings placed after attribute assignments in a class body. https://davidism.com/mit-license/ """ - cls_node = ast.parse(textwrap.dedent(inspect.getsource(cls))).body[0] + try: + source = inspect.getsource(cls) + except (OSError, TypeError): + # ``object`` and classes without retrievable source (e.g. builtins). + return + cls_node = ast.parse(textwrap.dedent(source)).body[0] if not isinstance(cls_node, ast.ClassDef): - raise TypeError("Given object was not a class.") - - out = {} + return # Consider each pair of nodes. for a, b in pairwise(cls_node.body): @@ -194,6 +197,20 @@ def get_attr_docs(cls: type[Any]) -> dict[str, str]: out[target.id] = doc + +def get_attr_docs(cls: type[Any]) -> dict[str, str]: + """Get any docstrings placed after attribute assignments in a class body or + its bases.""" + if not isinstance(cls, type): + raise TypeError("Given object was not a class.") + + out: dict[str, str] = {} + + # Walk the MRO from the most-base class to ``cls`` so that a docstring + # redefined on a subclass overrides the one inherited from a base class. + for base in reversed(cls.__mro__): + _get_own_attr_docs(base, out) + return out From e6674262f19a0da389dad1c37b5bbd937db80778 Mon Sep 17 00:00:00 2001 From: Divakar Verma <137818590+divakar-amd@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:37:36 -0400 Subject: [PATCH 222/248] [Bugfix][CI] Widen DBO+DP+EP GSM8K accuracy margin on ROCm (#59700) Signed-off-by: Divakar Verma Co-authored-by: Claude Sonnet 5 --- tests/v1/distributed/test_dbo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/v1/distributed/test_dbo.py b/tests/v1/distributed/test_dbo.py index 37e47e3c69..632bdfff67 100644 --- a/tests/v1/distributed/test_dbo.py +++ b/tests/v1/distributed/test_dbo.py @@ -11,6 +11,7 @@ import pytest import torch from tests.utils import RemoteOpenAIServer +from vllm.platforms import current_platform from vllm.utils.import_utils import has_deep_ep # Detect Blackwell / B200 (compute capability 10.x) @@ -30,7 +31,10 @@ DP_SIZE = 2 # GSM8K eval configuration NUM_QUESTIONS = 256 # Fast eval for CI; but must be large enough to hit dbo thresholds NUM_SHOTS = 5 # Few-shot examples -MIN_ACCURACY = 0.62 # Expected 0.64 with 2% buffer (based on vLLM test data) + +# Expected 0.64 with 2% buffer (based on vLLM test data) +# On ROCm, widened to 0.61 after observing 0.617 amd-ci nightly flake twice. +MIN_ACCURACY = 0.61 if current_platform.is_rocm() else 0.62 # Increase max_num_seqs to trigger DBO for decode batches # With 64 seqs, decode batches should exceed the 32 token threshold From 28c57456db9220fbdde6041a20b97bebd141259b Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Fri, 2 Oct 2026 05:33:36 +0530 Subject: [PATCH 223/248] [Bugfix][Frontend] Accept Anthropic tool_addition and tool_removal content blocks (#57693) Signed-off-by: Anurag Singh Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Antigravity Co-authored-by: sfeng33 <4florafeng@gmail.com> --- .../test_anthropic_messages_conversion.py | 85 +++++++++++++++++++ vllm/entrypoints/anthropic/protocol.py | 62 ++++++++++++++ vllm/entrypoints/anthropic/serving.py | 52 +++++++++++- 3 files changed, 197 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index d14f24eba0..fd0d0dc2a9 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -27,6 +27,7 @@ from pydantic import BaseModel, Field, ValidationError from vllm.entrypoints.anthropic.api_router import attach_router from vllm.entrypoints.anthropic.protocol import ( + AnthropicCountTokensRequest, AnthropicMessagesRequest, ) from vllm.entrypoints.anthropic.serving import ( @@ -2032,3 +2033,87 @@ class TestProbeDisabledThinkingEffort: @pytest.mark.asyncio async def test_renderer_rejects_none(self): assert await self._probe(self._reject_none) == "low" + + +class TestMidConversationToolChanges: + """``tool_addition``/``tool_removal`` blocks decide which tools the chat + template is given, since it has no other way to see them.""" + + @staticmethod + def _convert_tools(*changes: dict) -> dict[str, bool | None]: + request = _make_request( + [ + {"role": "user", "content": "Hello"}, + { + "role": "system", + "content": [{"type": "text", "text": "Tools changed."}, *changes], + }, + ], + tools=[ + {"name": "bash", "input_schema": {}}, + {"name": "search", "input_schema": {}, "defer_loading": True}, + {"name": "fetch", "input_schema": {}, "defer_loading": True}, + ], + ) + result = _convert(request) + assert result.messages[-1] == {"role": "system", "content": "Tools changed."} + assert result.tools is not None + return {t.function.name: t.function.defer_loading for t in result.tools} + + @staticmethod + def _change(block_type: str, name: str) -> dict: + return {"type": block_type, "tool": {"type": "tool_reference", "name": name}} + + def test_addition_loads_deferred_tool(self): + tools = self._convert_tools(self._change("tool_addition", "search")) + assert tools == {"bash": None, "search": None, "fetch": True} + + def test_removal_withdraws_tool_until_added_again(self): + removal = self._change("tool_removal", "bash") + assert "bash" not in self._convert_tools(removal) + readded = self._convert_tools(removal, self._change("tool_addition", "bash")) + assert list(readded) == ["bash", "search", "fetch"] + + def test_addition_defines_tool_by_value(self): + definition = {"name": "db_query", "input_schema": {}} + tools = self._convert_tools( + { + "type": "tool_addition", + "tool": {"type": "tool_definition", "definition": definition}, + } + ) + assert list(tools) == ["bash", "search", "fetch", "db_query"] + + def test_unknown_tool_reference_is_rejected(self): + with pytest.raises(ValueError, match="not declared in tools"): + self._convert_tools(self._change("tool_addition", "missing")) + + def test_removal_by_value_is_rejected(self): + definition = {"name": "bash", "input_schema": {}} + with pytest.raises(ValidationError, match="only accepts a tool_reference"): + self._convert_tools( + { + "type": "tool_removal", + "tool": {"type": "tool_definition", "definition": definition}, + } + ) + + @pytest.mark.parametrize( + "request_cls", [AnthropicMessagesRequest, AnthropicCountTokensRequest] + ) + @pytest.mark.parametrize("in_top_level_system", [False, True]) + def test_tool_change_outside_system_message_is_rejected( + self, request_cls, in_top_level_system + ): + change = self._change("tool_addition", "search") + if in_top_level_system: + fields = { + "messages": [{"role": "user", "content": "Hi"}], + "system": [change], + } + else: + fields = {"messages": [{"role": "user", "content": [change]}]} + with pytest.raises( + ValidationError, match='only allowed in messages with role "system"' + ): + request_cls(model="test-model", max_tokens=128, **fields) diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index 9bd40a9161..5abfaffed3 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -44,6 +44,8 @@ class AnthropicContentBlock(BaseModel): "tool_reference", "thinking", "redacted_thinking", + "tool_addition", + "tool_removal", ] text: str | None = None # For image content @@ -62,6 +64,28 @@ class AnthropicContentBlock(BaseModel): signature: str | None = None # For redacted thinking content (safety-filtered by the API) data: str | None = None + # For tool_addition/tool_removal content + tool: "AnthropicToolChange | None" = None + + @model_validator(mode="after") + def validate_tool_change(self) -> "AnthropicContentBlock": + if self.type not in ("tool_addition", "tool_removal"): + return self + if self.tool is None: + raise ValueError(f"tool is required for {self.type} blocks") + if self.type == "tool_removal" and self.tool.type != "tool_reference": + raise ValueError("tool_removal only accepts a tool_reference") + return self + + +def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None: + if content is None or isinstance(content, str): + return + for block in content: + if block.type in ("tool_addition", "tool_removal"): + raise ValueError( + f'{block.type} blocks are only allowed in messages with role "system"' + ) class AnthropicMessage(BaseModel): @@ -70,6 +94,12 @@ class AnthropicMessage(BaseModel): role: Literal["user", "assistant", "system"] content: str | list[AnthropicContentBlock] + @model_validator(mode="after") + def validate_tool_change_role(self) -> "AnthropicMessage": + if self.role != "system": + _reject_tool_changes(self.content) + return self + class AnthropicTool(BaseModel): """Tool definition.""" @@ -90,6 +120,26 @@ class AnthropicTool(BaseModel): return v +class AnthropicToolChangeReference(BaseModel): + """A tool named by a tool_addition or tool_removal block.""" + + type: Literal["tool_reference"] + name: str + + +class AnthropicToolChangeDefinition(BaseModel): + """A tool defined by value in a tool_addition block.""" + + type: Literal["tool_definition"] + definition: AnthropicTool + + +AnthropicToolChange = Annotated[ + AnthropicToolChangeReference | AnthropicToolChangeDefinition, + Field(discriminator="type"), +] + + class AnthropicToolChoice(BaseModel): """Tool Choice definition.""" @@ -231,6 +281,12 @@ class AnthropicMessagesRequest(BaseModel): raise ValueError("max_tokens must be positive") return v + @field_validator("system") + @classmethod + def validate_system(cls, v): + _reject_tool_changes(v) + return v + @model_validator(mode="after") def validate_thinking_budget(self) -> "AnthropicMessagesRequest": # P/D prefill legs are sent with max_tokens=1 and never decode; the @@ -343,6 +399,12 @@ class AnthropicCountTokensRequest(BaseModel): raise ValueError("Model is required") return v + @field_validator("system") + @classmethod + def validate_system(cls, v): + _reject_tool_changes(v) + return v + class AnthropicCountTokensResponse(BaseModel): """Anthropic messages.count_tokens response.""" diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 91820de2fd..cf5fd9ecc1 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -31,6 +31,8 @@ from vllm.entrypoints.anthropic.protocol import ( AnthropicOutputConfig, AnthropicStreamEvent, AnthropicThinkingConfig, + AnthropicTool, + AnthropicToolChangeDefinition, AnthropicUsage, ) from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption @@ -710,6 +712,51 @@ class AnthropicServingMessages(OpenAIServingChat): } ) + @classmethod + def _resolve_tools( + cls, + anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, + ) -> list[AnthropicTool] | None: + """Apply tool_addition/tool_removal blocks in ``messages`` to ``tools``. + + Chat templates see a single tool list, so this resolves the tools in + effect at the end of the conversation: a tool added by reference is no + longer deferred, a tool added by value is declared, and a removed tool + is dropped. + """ + changes = [ + block + for msg in anthropic_request.messages + if not isinstance(msg.content, str) + for block in msg.content + if block.type in ("tool_addition", "tool_removal") + ] + if not changes: + return anthropic_request.tools + + tools = {tool.name: tool for tool in anthropic_request.tools or []} + removed: set[str] = set() + for block in changes: + change = block.tool + assert change is not None + if isinstance(change, AnthropicToolChangeDefinition): + name = change.definition.name + tools[name] = change.definition + else: + name = change.name + if name not in tools: + raise ValueError( + f"{block.type} references tool {name!r}, " + "which is not declared in tools" + ) + if block.type == "tool_addition": + tools[name] = tools[name].model_copy(update={"defer_loading": None}) + if block.type == "tool_removal": + removed.add(name) + else: + removed.discard(name) + return [tool for name, tool in tools.items() if name not in removed] + @classmethod def _convert_tools( cls, @@ -717,11 +764,12 @@ class AnthropicServingMessages(OpenAIServingChat): req: ChatCompletionRequest, ) -> None: """Convert Anthropic tools to OpenAI format.""" - if anthropic_request.tools is None: + anthropic_tools = cls._resolve_tools(anthropic_request) + if anthropic_tools is None: return tools = [] - for tool in anthropic_request.tools: + for tool in anthropic_tools: tools.append( ChatCompletionToolsParam.model_validate( { From 5b6b657e13bd1005be47e45c1d140bb565941afe Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 1 Oct 2026 17:17:37 -0700 Subject: [PATCH 224/248] [ci] Update mergify to rebase if behind by 100 commits (#59710) Signed-off-by: Kevin H. Luu --- .github/mergify.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/mergify.yml b/.github/mergify.yml index 9151c5c0a1..73ea58aed6 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -571,7 +571,7 @@ pull_request_rules: - base = main - label=ready - "#approved-reviews-by >= 1" - - "#commits-behind >= 50" + - "#commits-behind >= 100" - "#check-failure = 0" - -closed - -draft From eec3c36a478bdacc5ae322af9a0cde6c5604182b Mon Sep 17 00:00:00 2001 From: ukannika Date: Thu, 1 Oct 2026 17:21:18 -0700 Subject: [PATCH 225/248] [ROCm][BugFix] Revert AITER PA gluon decode from ROCM_AITER_FA (#54805) Signed-off-by: ukannika Co-authored-by: Andreas Karatzas Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/v1/attention/backends/rocm_aiter_fa.py | 314 +++++--------------- 1 file changed, 74 insertions(+), 240 deletions(-) diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 3df3b26bba..329aee5359 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -8,11 +8,7 @@ from typing import ClassVar import torch from vllm._aiter_ops import rocm_aiter_ops -from vllm.config import ( - VllmConfig, - get_current_vllm_config_or_none, - get_layers_from_vllm_config, -) +from vllm.config import VllmConfig, get_layers_from_vllm_config from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention @@ -42,43 +38,9 @@ from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.ops.merge_attn_states import merge_attn_states from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout -_PA_GLUON_MAX_QUERY_LEN = 4 -_PA_GLUON_MAX_QUERY_GROUP_SIZE = 64 -# Query group sizes the gluon paged-attention decode kernel is validated for: 8, 16 -_PA_GLUON_QUERY_GROUP_SIZES = (8, 16) - -# The kernel is only validated for this head size and kernel block size. -_PA_GLUON_HEAD_SIZE = 128 -_PA_GLUON_BLOCK_SIZE = 128 - - -def _pa_gluon_supports(num_heads_q: int, num_heads_kv: int, head_size: int) -> bool: - """Whether the head config can use the PA decode gluon kernel. - - Requires the shuffle KV cache layout (the kernel reads K/V in that layout) - plus a head config the kernel is validated for. Both the advertised kernel - block sizes and the decode dispatch go through this, so a config can never - be offered a 128-token page that gluon will then decline to serve. - """ - return ( - rocm_aiter_ops.is_shuffle_kv_cache_enabled() - and num_heads_kv > 0 - and num_heads_q % num_heads_kv == 0 - and num_heads_q // num_heads_kv in _PA_GLUON_QUERY_GROUP_SIZES - and head_size == _PA_GLUON_HEAD_SIZE - ) - - _PARTITION_SIZE_ROCM = 256 _CP_TOKENS_PER_ITER_ROCM = 32 * 1024 if current_platform.is_rocm(): - from aiter.ops.triton.gluon.pa_decode_gluon import ( - get_recommended_splits, - ) - from aiter.ops.triton.gluon.pa_decode_gluon import ( - pa_decode_gluon as _pa_decode_gluon, - ) - from vllm.triton_utils import tl, triton def block_size(x, head_dim): @@ -168,8 +130,10 @@ if current_platform.is_rocm(): # for kv cache layout as # K: [num_blocks, num_head, head_dim // x, page_size, x] # V: [num_blocks, num_head, page_size // x, head_dim, x] - # The sides only sit a dense page apart when they are separate - # planes (H outermost), so take the block stride from the tensor. + # Take the block stride from the tensor, as the NHD branch does: + # the two sides only sit a dense page apart when they are separate + # planes, and models that pin a block-compact layout interleave + # them instead. key_cache_ptr_offset = ( key_cache_ptr + block_id * k_cache_stride0 @@ -363,7 +327,6 @@ logger = init_logger(__name__) @dataclass class AiterFlashAttentionDecodeMetadata: max_query_len: int - uniform_query_len: int | None @dataclass @@ -598,15 +561,8 @@ class AiterFlashAttentionMetadataBuilder( decode_metadata = None if num_decodes > 0: - decode_max_query_len = query_lens_cpu[:num_decodes].max().item() - uniform_query_len = ( - decode_max_query_len - if num_decode_tokens == num_decodes * decode_max_query_len - else None - ) decode_metadata = AiterFlashAttentionDecodeMetadata( - max_query_len=decode_max_query_len, - uniform_query_len=uniform_query_len, + max_query_len=query_lens_cpu[:num_decodes].max().item(), ) prefill_metadata = None @@ -778,27 +734,9 @@ class AiterFlashAttentionMetadataBuilder( """ num_reqs = common_attn_metadata.num_reqs num_tokens = common_attn_metadata.num_actual_tokens - max_query_len = common_attn_metadata.max_query_len - - # Uniform-decode assumption does not hold for the - # drafter's first forward after a target step: it inherits the target's - # per-request query lengths, so rows can be longer than gluon's limit or - # ragged. Those batches need the real split, which costs a sync. - # _PA_GLUON_MAX_QUERY_LEN only binds when gluon is the decode consumer, - # so test that rather than the shuffle layout alone. - if _pa_gluon_supports(self.num_heads_q, self.num_heads_kv, self.headdim) and ( - max_query_len > _PA_GLUON_MAX_QUERY_LEN - or num_tokens != num_reqs * max_query_len - ): - return self.build( - common_prefix_len=0, common_attn_metadata=common_attn_metadata - ) decode_metadata = AiterFlashAttentionDecodeMetadata( - max_query_len=max_query_len, - uniform_query_len=( - max_query_len if num_tokens == num_reqs * max_query_len else None - ), + max_query_len=common_attn_metadata.max_query_len, ) return AiterFlashAttentionMetadata( @@ -853,21 +791,6 @@ class AiterFlashAttentionBackend(AttentionBackend): @staticmethod def get_supported_kernel_block_sizes(kv_cache_spec=None) -> list[int | MultipleOf]: - if not rocm_aiter_ops.is_shuffle_kv_cache_enabled(): - return [16, 32] - # Only gluon serves 128-token pages; the pa_fwd_asm/ll4mi decode - # fallback is limited to 16 and 32. Advertise 128 only when gluon can - # run so selection never picks a page we cannot serve. - vllm_config = get_current_vllm_config_or_none() - if vllm_config is not None and vllm_config.model_config is not None: - mc = vllm_config.model_config - pc = vllm_config.parallel_config - if _pa_gluon_supports( - mc.get_num_attention_heads(pc), - mc.get_num_kv_heads(pc), - mc.get_head_size(), - ): - return [16, 32, 128] return [16, 32] @classmethod @@ -924,6 +847,14 @@ class AiterFlashAttentionBackend(AttentionBackend): @classmethod def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]: + if rocm_aiter_ops.is_shuffle_kv_cache_enabled(): + # pa_fwd_asm strides between pages by a whole dense page, so the two + # head slots customize_spec publishes have to be separate planes + # spanning every block: H outermost. Under LBHNC the slots alternate + # block by block, doubling the stride the kernel assumes. LBHNC + # stays listed for models whose mixed HNC shapes need a + # block-compact layout; those read K/V by stride instead. + return (KVCacheLayout.LHBNC, KVCacheLayout.LBHNC) # K and V come out of the content dim as transposed views rather than # copies, so the head dim may sit on either side of the block dim, but # the layer must stay outermost. @@ -1358,25 +1289,11 @@ class AiterFlashAttentionImpl(AttentionImpl): if num_decodes > 0: assert attn_metadata.decode_metadata is not None decode_max_query_len = attn_metadata.decode_metadata.max_query_len - decode_query_len = attn_metadata.decode_metadata.uniform_query_len - # check if we can use the gluon paged-attention decode kernel - use_gluon = ( - _pa_gluon_supports( - self.num_heads, self.num_kv_heads, self.head_size - ) - and key_cache.shape[1] == _PA_GLUON_BLOCK_SIZE - and decode_query_len is not None - and decode_query_len <= _PA_GLUON_MAX_QUERY_LEN - and decode_query_len * (self.num_heads // self.num_kv_heads) - <= _PA_GLUON_MAX_QUERY_GROUP_SIZE - and (decode_query_len == 1 or attn_metadata.causal) - ) # Use unified_attention for the decodes the paged kernels can't - # take: sliding window, sinks, or a multi-token batch that gluon - # declined (pa_fwd_asm and paged_attention_v1 don't support - # sinks). - if not use_gluon and ( + # take: sliding window, sinks, or a multi-token batch + # (pa_fwd_asm and paged_attention_v1 don't support sinks). + if ( self.sliding_window[0] != -1 or decode_max_query_len > 1 or self.sinks is not None @@ -1505,145 +1422,62 @@ class AiterFlashAttentionImpl(AttentionImpl): num_blocks, num_kv_heads, block_size // x, head_size, x ) - if use_gluon: - is_fp8_kv = is_quantized_kv_cache(self.kv_cache_dtype) - # Per-tensor descale, as a float32 [1] tensor. - k_scale_gluon = ( - layer._k_scale.reshape(1).to(torch.float32) - if is_fp8_kv - else None - ) - v_scale_gluon = ( - layer._v_scale.reshape(1).to(torch.float32) - if is_fp8_kv - else None - ) - compute_type = ( - current_platform.fp8_dtype() if is_fp8_kv else query.dtype - ) - # The kernel folds the query positions into the group - # dim, so the intermediate buffers are sized by the - # combined extent. - query_group_size = decode_query_len * ( - num_heads // num_kv_heads - ) - - sliding_window_int = ( - self.sliding_window[0] + 1 - if self.sliding_window[0] > 0 - else 0 - ) - if sliding_window_int > 0: - max_context_partition_num = 1 - context_partition_size = 128 - else: - max_context_partition_num = get_recommended_splits( - num_decodes, num_kv_heads - ) - context_partition_size = _PARTITION_SIZE_ROCM - - intermediate_shape = ( - num_decodes, - num_kv_heads, - max_context_partition_num, - query_group_size, - ) - exp_sums = torch.empty( - intermediate_shape, - dtype=torch.float32, - device=query.device, - ) - max_logits = torch.empty_like(exp_sums) - temporary_output = torch.empty( - (*intermediate_shape, head_size), - dtype=output.dtype, - device=query.device, - ) - - _pa_decode_gluon( - output=output[:num_decode_tokens], - query=query[:num_decode_tokens], - key_cache=new_key_cache, - value_cache=new_value_cache, - context_lengths=attn_metadata.seq_lens[:num_decodes].to( - torch.int32 - ), - block_tables=attn_metadata.block_table[:num_decodes].to( - torch.int32 - ), - softmax_scale=self.scale, - query_length=decode_query_len, - max_context_partition_num=max_context_partition_num, - context_partition_size=context_partition_size, - compute_type=compute_type, - query_scale=None, - key_scale=k_scale_gluon, - value_scale=v_scale_gluon, - exp_sums=exp_sums, - max_logits=max_logits, - temporary_output=temporary_output, - alibi_slopes=self.alibi_slopes, - sinks=self.sinks, - sliding_window=sliding_window_int, - ps=True, - ) - else: - # Unlike gluon, this kernel derives block addresses - # arithmetically, so it only reads the right bytes when - # each side is a dense plane of pages. - assert new_key_cache.stride(0) == ( - num_kv_heads * head_size * block_size - ), ( - "paged_attention_common needs the K/V sides as " - "dense planes; the resolved KV cache layout " - "interleaves them within a block." - ) - num_seqs = attn_metadata.seq_lens.shape[0] - max_num_partitions = ( - attn_metadata.max_seq_len + _PARTITION_SIZE_ROCM - 1 - ) // _PARTITION_SIZE_ROCM - tmp_out = torch.empty( - (num_seqs, num_heads, max_num_partitions, head_size), - dtype=query.dtype, - device=query.device, - ) - exp_sums = torch.empty( - (num_seqs, num_heads, max_num_partitions), - dtype=torch.float32, - device=query.device, - ) - max_logits = torch.empty_like(exp_sums) - k_qscale = ( - layer._k_scale - if attn_metadata.k_scale is None - else attn_metadata.k_scale - ) - v_qscale = ( - layer._v_scale - if attn_metadata.v_scale is None - else attn_metadata.v_scale - ) - rocm_aiter_ops.paged_attention_common( - Q=query[:num_decode_tokens], - K=new_key_cache, - V=new_value_cache, - tmp_out=tmp_out, - max_logits=max_logits, - exp_sums=exp_sums, - max_seq_len=attn_metadata.max_seq_len, - block_tables=attn_metadata.block_table[:num_decodes], - context_lens=attn_metadata.seq_lens[:num_decodes], - block_tables_stride0=attn_metadata.block_table[ - :num_decodes - ].stride(0), - scale=self.scale, - K_QScale_hip=k_qscale, - V_QScale_hip=v_qscale, - K_QScale_asm=k_qscale, - V_QScale_asm=v_qscale, - out_=output[:num_decode_tokens], - kv_cache_dtype=self.kv_cache_dtype, - ) + # This kernel derives block addresses arithmetically, so it + # only reads the right bytes when each side is a dense + # plane of pages. + assert new_key_cache.stride(0) == ( + num_kv_heads * head_size * block_size + ), ( + "paged_attention_common needs the K/V sides as dense " + "planes; the resolved KV cache layout interleaves them " + "within a block." + ) + num_seqs = attn_metadata.seq_lens.shape[0] + max_num_partitions = ( + attn_metadata.max_seq_len + _PARTITION_SIZE_ROCM - 1 + ) // _PARTITION_SIZE_ROCM + tmp_out = torch.empty( + (num_seqs, num_heads, max_num_partitions, head_size), + dtype=query.dtype, + device=query.device, + ) + exp_sums = torch.empty( + (num_seqs, num_heads, max_num_partitions), + dtype=torch.float32, + device=query.device, + ) + max_logits = torch.empty_like(exp_sums) + k_qscale = ( + layer._k_scale + if attn_metadata.k_scale is None + else attn_metadata.k_scale + ) + v_qscale = ( + layer._v_scale + if attn_metadata.v_scale is None + else attn_metadata.v_scale + ) + rocm_aiter_ops.paged_attention_common( + Q=query[:num_decode_tokens], + K=new_key_cache, + V=new_value_cache, + tmp_out=tmp_out, + max_logits=max_logits, + exp_sums=exp_sums, + max_seq_len=attn_metadata.max_seq_len, + block_tables=attn_metadata.block_table[:num_decodes], + context_lens=attn_metadata.seq_lens[:num_decodes], + block_tables_stride0=attn_metadata.block_table[ + :num_decodes + ].stride(0), + scale=self.scale, + K_QScale_hip=k_qscale, + V_QScale_hip=v_qscale, + K_QScale_asm=k_qscale, + V_QScale_asm=v_qscale, + out_=output[:num_decode_tokens], + kv_cache_dtype=self.kv_cache_dtype, + ) else: _, num_heads, head_size = query.shape nbytes_per_qo_elem = torch.finfo(query.dtype).bits // 8 From d58da79de76d2c3dfd6799f3355dec9eb8af9b19 Mon Sep 17 00:00:00 2001 From: Zijing Liu Date: Thu, 1 Oct 2026 18:24:33 -0700 Subject: [PATCH 226/248] [Metrics][P/D] Add KV-fetch stage gauges for async KV loads (#58874) Signed-off-by: Zijing Liu Signed-off-by: Nick Hill Co-authored-by: Nick Hill Co-authored-by: Claude Opus 5.5 (1M context) --- .../unit/test_remote_prefill_lifecycle.py | 28 +++++++++++++ tests/v1/kv_connector/unit/utils.py | 2 + vllm/v1/core/sched/scheduler.py | 34 ++++++++++++++++ vllm/v1/metrics/loggers.py | 40 +++++++++++++++++++ vllm/v1/metrics/stats.py | 12 ++++++ 5 files changed, 116 insertions(+) diff --git a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py index daddd20b4a..59fcc678bf 100644 --- a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py +++ b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py @@ -5,6 +5,7 @@ from unittest.mock import patch import pytest +from vllm.v1.metrics.stats import KV_FETCH_STAGES from vllm.v1.outputs import ( EMPTY_MODEL_RUNNER_OUTPUT, KVConnectorOutput, @@ -28,6 +29,11 @@ def _num_waiting_requests(scheduler) -> int: return len(scheduler.waiting) + len(scheduler.kv_holding_waiting) +def _kv_fetch_stages(scheduler) -> tuple[int, ...]: + """(waiting_to_start, in_progress, completed_waiting) request counts.""" + return tuple(scheduler._kv_fetch_counts[s] for s in KV_FETCH_STAGES) + + def test_basic_lifecycle(): """Test lifecycle of a remote prefill.""" vllm_config = create_vllm_config() @@ -50,10 +56,12 @@ def test_basic_lifecycle(): scheduler.add_request(request) request_id = request.request_id + assert _kv_fetch_stages(scheduler) == (1, 0, 0) # STEP (1): # (1a): schedule() scheduler_output = scheduler.schedule() + assert _kv_fetch_stages(scheduler) == (0, 1, 0) # Nothing running and empty scheduler output. assert len(scheduler.running) == 0 @@ -105,11 +113,13 @@ def test_basic_lifecycle(): ) assert _num_waiting_requests(scheduler) == 1 assert request_id in scheduler.finished_recving_kv_req_ids + assert _kv_fetch_stages(scheduler) == (0, 0, 1) # STEP (3): # (3a): schedule(): this should actually schedule. scheduler_output = scheduler.schedule() assert len(scheduler.running) == 1 + assert _kv_fetch_stages(scheduler) == (0, 0, 0) # Confirm the block are actually allocated. num_hashed_blocks = 0 @@ -582,6 +592,24 @@ def test_cannot_recv(): assert_scheduler_empty(scheduler) +def test_kv_fetch_stages_cleared_on_abort(): + vllm_config = create_vllm_config() + scheduler = create_scheduler(vllm_config) + BLOCK_SIZE = vllm_config.cache_config.block_size + request = create_request( + request_id=1, + block_size=BLOCK_SIZE, + num_tokens=int(BLOCK_SIZE * 2.5), + do_remote_prefill=True, + ) + scheduler.add_request(request) + scheduler.schedule() + assert _kv_fetch_stages(scheduler) == (0, 1, 0) + + scheduler.finish_requests(request.request_id, RequestStatus.FINISHED_ABORTED) + assert _kv_fetch_stages(scheduler) == (0, 0, 0) + + @patch( "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform" ) diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 2febe3109f..6f27212281 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -60,6 +60,8 @@ def assert_scheduler_empty(scheduler: Scheduler): assert len(scheduler.finished_req_ids) == 0 assert len(scheduler.finished_recving_kv_req_ids) == 0 assert len(scheduler._inflight_prefills) == 0 + assert not scheduler._kv_fetch_stages + assert not any(scheduler._kv_fetch_counts.values()) # EncoderCacheManager. assert len(scheduler.encoder_cache_manager.freed) == 0 diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index fcb1421b10..5585c9c70c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -62,6 +62,10 @@ from vllm.v1.kv_cache_interface import ( ) from vllm.v1.metrics.perf import ModelMetrics, PerfStats from vllm.v1.metrics.stats import ( + KV_FETCH_COMPLETED_WAITING, + KV_FETCH_IN_PROGRESS, + KV_FETCH_STAGES, + KV_FETCH_WAITING_TO_START, PrefixCacheStats, RequestSpecDecodeMetrics, SchedulerStats, @@ -412,6 +416,13 @@ class Scheduler(SchedulerInterface): # async KV loads). Their remaining-block reservation gates async loads. self._inflight_prefills: set[Request] = set() + # Requests with an async KV load, by stage (KV_FETCH_STAGES). + # None when not tracked (stats disabled or no KV connector). + self._kv_fetch_stages: dict[Request, str] | None = ( + {} if self.log_stats and self.connector is not None else None + ) + self._kv_fetch_counts: dict[str, int] = dict.fromkeys(KV_FETCH_STAGES, 0) + def _mamba_block_aligned_split( self, request: Request, @@ -1039,6 +1050,9 @@ class Scheduler(SchedulerInterface): num_local_cached_tokens=num_new_local_computed_tokens, num_external_cached_tokens=num_external_computed_tokens, ) + self._set_kv_fetch_stage( + request, KV_FETCH_WAITING_TO_START if load_kv_async else None + ) else: # KVTransfer: WAITING reqs have num_computed_tokens > 0 # after async KV recvs are completed. A streaming-input @@ -1276,6 +1290,7 @@ class Scheduler(SchedulerInterface): # only the successfully loaded tokens. request.num_computed_tokens = num_computed_tokens self._inflight_prefills.add(request) + self._set_kv_fetch_stage(request, KV_FETCH_IN_PROGRESS) if self.needs_kv_cache_zeroing: # Skip zeroing of the blocks the async load will # overwrite; the zeroing could race the write. @@ -1316,6 +1331,7 @@ class Scheduler(SchedulerInterface): input_budget -= num_new_tokens + draft_slots request.status = RequestStatus.RUNNING request.num_computed_tokens = num_computed_tokens + self._set_kv_fetch_stage(request, None) if pad_spec_decode: assert num_new_tokens == 1 + self.num_spec_tokens scheduled_spec_decode_tokens[request_id] = [ @@ -2588,6 +2604,11 @@ class Scheduler(SchedulerInterface): ) if self.connector is not None: self.connector.on_new_request(request) + kv_transfer_params = request.kv_transfer_params + if kv_transfer_params and kv_transfer_params.get("do_remote_prefill"): + # Count P/D requests on arrival: connector matching is + # skipped while all run slots are taken. + self._set_kv_fetch_stage(request, KV_FETCH_WAITING_TO_START) if self.log_stats: request.record_event(EngineCoreEventType.QUEUED) @@ -2664,6 +2685,7 @@ class Scheduler(SchedulerInterface): if self.aux_output_connector is not None: self.aux_output_connector.request_finished(request) self._inflight_prefills.discard(request) + self._set_kv_fetch_stage(request, None) connector_delay_free_blocks, kv_xfer_params = self._connector_finished(request) # EC Connector: mirror the KV hook. The contract requires firing @@ -2889,6 +2911,7 @@ class Scheduler(SchedulerInterface): num_running_reqs=num_running, num_waiting_reqs=num_waiting - num_deferred, num_skipped_waiting_reqs=num_deferred, + num_kv_fetch_reqs_by_stage=self._kv_fetch_counts.copy(), kv_cache_usage=self.kv_cache_manager.usage, prefix_cache_stats=prefix_cache_stats, connector_prefix_cache_stats=connector_prefix_cache_stats, @@ -3028,6 +3051,16 @@ class Scheduler(SchedulerInterface): 1 + self.num_spec_tokens + self.num_lookahead_tokens, self.block_size ) + def _set_kv_fetch_stage(self, request: Request, stage: str | None) -> None: + kv_fetch_stages = self._kv_fetch_stages + if kv_fetch_stages is not None: + prev_stage = kv_fetch_stages.pop(request, None) + if prev_stage is not None: + self._kv_fetch_counts[prev_stage] -= 1 + if stage is not None: + kv_fetch_stages[request] = stage + self._kv_fetch_counts[stage] += 1 + def _inflight_prefill_reserved_blocks(self) -> int: """Num blocks in-flight prefills still need to finish (their reservation).""" return sum( @@ -3148,6 +3181,7 @@ class Scheduler(SchedulerInterface): req = self.requests[req_id] if req.status == RequestStatus.WAITING_FOR_REMOTE_KVS: self.finished_recving_kv_req_ids.add(req_id) + self._set_kv_fetch_stage(req, KV_FETCH_COMPLETED_WAITING) else: assert RequestStatus.is_finished(req.status) self._free_blocks(self.requests[req_id]) diff --git a/vllm/v1/metrics/loggers.py b/vllm/v1/metrics/loggers.py index ac93b26a8f..681f80ce29 100644 --- a/vllm/v1/metrics/loggers.py +++ b/vllm/v1/metrics/loggers.py @@ -26,6 +26,7 @@ from vllm.v1.metrics.buckets import histogram_buckets from vllm.v1.metrics.perf import PerfMetricsLogging, PerfMetricsProm from vllm.v1.metrics.prometheus import unregister_vllm_metrics from vllm.v1.metrics.stats import ( + KV_FETCH_STAGES, CachingMetrics, IterationStats, MultiModalCacheStats, @@ -292,6 +293,13 @@ class LoggingStatLogger(StatLoggerBase): log_parts.append("Deferred: %d reqs") log_args.append(self.last_scheduler_stats.num_skipped_waiting_reqs) + kv_fetch = self.last_scheduler_stats.num_kv_fetch_reqs_by_stage + if any(kv_fetch.values()): + log_parts.append( + "KV fetch: %d waiting to start, %d in progress, %d completed waiting" + ) + log_args.extend(kv_fetch.get(stage, 0) for stage in KV_FETCH_STAGES) + if self.num_preemptions > 0: log_parts.append("Preemptions: %d") log_args.append(self.num_preemptions) @@ -393,6 +401,10 @@ class AggregatedLoggingStatLogger(LoggingStatLogger, AggregateStatLoggerBase): self.last_scheduler_stats.num_skipped_waiting_reqs += ( last_scheduler_stats.num_skipped_waiting_reqs ) + engine_kv_fetch = last_scheduler_stats.num_kv_fetch_reqs_by_stage + total_kv_fetch = self.last_scheduler_stats.num_kv_fetch_reqs_by_stage + for stage, num_reqs in engine_kv_fetch.items(): + total_kv_fetch[stage] = total_kv_fetch.get(stage, 0) + num_reqs self.last_scheduler_stats.kv_cache_usage += ( last_scheduler_stats.kv_cache_usage ) @@ -547,6 +559,30 @@ class PrometheusStatLogger(AggregateStatLoggerBase): gauge_waiting_by_reason, per_engine_labelvalues_with_reason ) + self.gauge_kv_fetch_by_stage: dict[str, dict[int, Gauge]] = {} + kv_transfer_config = vllm_config.kv_transfer_config + if kv_transfer_config is not None and kv_transfer_config.is_kv_consumer: + gauge_kv_fetch_by_stage = self._gauge_cls( + name="vllm:num_requests_kv_fetch_by_stage", + documentation=( + "Number of waiting requests by async KV load stage. " + "Stage labels: 'waiting_to_start' = needs an async KV load " + "that has not started; 'in_progress' = load started and not " + "yet reported finished; 'completed_waiting' = load finished, " + "request not running yet." + ), + multiprocess_mode="mostrecent", + labelnames=labelnames + ["stage"], + ) + for stage in KV_FETCH_STAGES: + per_engine_labelvalues_with_stage = { + idx: labelvalues + [stage] + for idx, labelvalues in per_engine_labelvalues.items() + } + self.gauge_kv_fetch_by_stage[stage] = create_metric_per_engine( + gauge_kv_fetch_by_stage, per_engine_labelvalues_with_stage + ) + gauge_engine_sleep_state = self._gauge_cls( name="vllm:engine_sleep_state", documentation=( @@ -1062,6 +1098,10 @@ class PrometheusStatLogger(AggregateStatLoggerBase): self.gauge_waiting_by_reason[WAITING_REASON_DEFERRED][engine_idx].set( scheduler_stats.num_skipped_waiting_reqs ) + for stage, gauges in self.gauge_kv_fetch_by_stage.items(): + gauges[engine_idx].set( + scheduler_stats.num_kv_fetch_reqs_by_stage.get(stage, 0) + ) self.gauge_kv_cache_usage[engine_idx].set(scheduler_stats.kv_cache_usage) self.counter_prefix_cache_queries[engine_idx].inc( diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index da88c9379a..4db98884fe 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -182,6 +182,16 @@ class SchedulerIterationDetails: is_dummy: bool = False +KV_FETCH_WAITING_TO_START = "waiting_to_start" +KV_FETCH_IN_PROGRESS = "in_progress" +KV_FETCH_COMPLETED_WAITING = "completed_waiting" +KV_FETCH_STAGES = ( + KV_FETCH_WAITING_TO_START, + KV_FETCH_IN_PROGRESS, + KV_FETCH_COMPLETED_WAITING, +) + + @dataclass class SchedulerStats: """Stats associated with the scheduler.""" @@ -192,6 +202,8 @@ class SchedulerStats: # waiting requests deferred by transient constraints or a blocked status num_skipped_waiting_reqs: int = 0 + num_kv_fetch_reqs_by_stage: dict[str, int] = field(default_factory=dict) + # These are used for internal DP load-balancing. step_counter: int = 0 current_wave: int = 0 From 89772ddb1f0d9a5858e55b1efefdac5912231b05 Mon Sep 17 00:00:00 2001 From: liuzhenwei Date: Fri, 2 Oct 2026 09:28:58 +0800 Subject: [PATCH 227/248] [CI][XPU] Skip CUDA-IPC weight sync metrics test on Intel (#59556) Signed-off-by: zhenwei-intel --- .buildkite/intel_jobs/entrypoints_intel.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/intel_jobs/entrypoints_intel.yaml b/.buildkite/intel_jobs/entrypoints_intel.yaml index e8006bac8f..7e823e0245 100644 --- a/.buildkite/intel_jobs/entrypoints_intel.yaml +++ b/.buildkite/intel_jobs/entrypoints_intel.yaml @@ -77,6 +77,7 @@ steps: 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc + --ignore=entrypoints/serve/dev/rlhf/test_weight_operation_metrics.py --deselect="tests/entrypoints/serve/dev/rlhf/state_transitions/test_pause_resume.py::TestPauseResume::test_clear_cache_preserves_output_and_controls_prefix_cache[MRV1]" && PYTHONPATH=/workspace/vllm pytest -v -s entrypoints/serve/dev/rpc' From 4caa060053eba78722c610806188c5566e803e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mi=C5=82osz=20Grunwald?= Date: Fri, 2 Oct 2026 03:47:47 +0200 Subject: [PATCH 228/248] [TEST][CI] Fix serve rlhf tests subprocess import errors (#59622) Signed-off-by: microslaw Co-authored-by: microslaw Co-authored-by: Kunshang Ji --- tests/entrypoints/serve/dev/rlhf/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/entrypoints/serve/dev/rlhf/__init__.py diff --git a/tests/entrypoints/serve/dev/rlhf/__init__.py b/tests/entrypoints/serve/dev/rlhf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 6660926d920bc6f5bc0c5d018182ffe45fd1a744 Mon Sep 17 00:00:00 2001 From: chowdmia Date: Thu, 1 Oct 2026 19:01:07 -0700 Subject: [PATCH 229/248] =?UTF-8?q?[Bugfix][CPU]=20Fix=20macOS=20build=20o?= =?UTF-8?q?n=20Apple=20Clang=20<=2017:=20structured=20binding=E2=80=A6=20(?= =?UTF-8?q?#58932)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Miajan Chowdhury Co-authored-by: Miajan Chowdhury Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- csrc/cpu/sgl-kernels/fla.cpp | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/csrc/cpu/sgl-kernels/fla.cpp b/csrc/cpu/sgl-kernels/fla.cpp index dd6f3b29ab..0cf34acaac 100644 --- a/csrc/cpu/sgl-kernels/fla.cpp +++ b/csrc/cpu/sgl-kernels/fla.cpp @@ -2306,21 +2306,35 @@ std::tuple chunk_gated_delta_rule_cpu( chunk_gated_delta_rule_fwd_intra(key_, value, g_, beta, cu_seqlens, chunk_indices); // fused `chunk_gated_delta_rule_fwd_h` + `chunk_fwd_o` - auto launch_inter = [&](auto state_tag) { + auto launch_inter = [local_query = query_, + local_key = key_, + local_w = w, + local_u = u, + local_decay_mask = decay_mask, + local_chunk_offsets = chunk_offsets, + g_, + initial_state, + output_final_state, + cu_seqlens, + initial_state_indices](auto state_tag) { using state_t = decltype(state_tag); return chunk_gated_delta_rule_fwd_inter( - query_, - key_, - w, - u, + local_query, + local_key, + local_w, + local_u, g_, - decay_mask, + local_decay_mask, initial_state, output_final_state, cu_seqlens, - chunk_offsets, + local_chunk_offsets, initial_state_indices); }; + + + + auto output_and_state = initial_state.scalar_type() == at::kFloat ? launch_inter(float{}) From e5d33a36e71da9e3e122ac9b139f3ab3b944ef71 Mon Sep 17 00:00:00 2001 From: ai-jz <156989844+ai-jz@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:10:35 -0700 Subject: [PATCH 230/248] [Bugfix][Rust Frontend] Preserve whitespace in GLM string arguments (#59654) Signed-off-by: ai-jz Co-authored-by: ai-jz Co-authored-by: Bugen Zhao --- rust/src/parser/src/tool/glm_xml/mod.rs | 36 +++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/rust/src/parser/src/tool/glm_xml/mod.rs b/rust/src/parser/src/tool/glm_xml/mod.rs index 58ef4a0045..9883acebb6 100644 --- a/rust/src/parser/src/tool/glm_xml/mod.rs +++ b/rust/src/parser/src/tool/glm_xml/mod.rs @@ -261,7 +261,7 @@ fn parse_parameter(input: &mut &str) -> ModalResult<(String, String)> { _: literal(ARG_KEY_END), _: ws0, _: literal(ARG_VALUE_START), - take_until(0.., ARG_VALUE_END).map(str::trim), + take_until(0.., ARG_VALUE_END), _: literal(ARG_VALUE_END), ) .parse_next(input)?; @@ -274,7 +274,7 @@ mod tests { use serde_json::{Value, json}; use thiserror_ext::AsReport; - use super::Glm45MoeToolParser; + use super::{Glm45MoeToolParser, Glm47MoeToolParser}; use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; use crate::tool::{ToolParser, ToolParserTestExt as _}; @@ -320,6 +320,38 @@ mod tests { ); } + #[test] + fn glm_parse_complete_preserves_string_whitespace() { + let tools = test_tools(); + let text = glm45_tool_call("get_weather", &[("city", " Paris \n"), ("days", " 2 ")]); + for mut parser in [ + Glm45MoeToolParser::create(&tools).unwrap(), + Glm47MoeToolParser::create(&tools).unwrap(), + ] { + let output = parser.parse_complete(&text).unwrap(); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({"city": " Paris \n", "days": 2}) + ); + } + } + + #[test] + fn glm_streaming_preserves_string_whitespace() { + let tools = test_tools(); + let text = glm45_tool_call("get_weather", &[("city", " Paris \n"), ("days", " 2 ")]); + for mut parser in [ + Glm45MoeToolParser::create(&tools).unwrap(), + Glm47MoeToolParser::create(&tools).unwrap(), + ] { + let output = collect_stream(&mut *parser, &split_by_chars(&text, 7)); + assert_eq!( + serde_json::from_str::(&output.calls()[0].arguments).unwrap(), + json!({"city": " Paris \n", "days": 2}) + ); + } + } + #[test] fn glm45_streaming_extracts_multiple_tool_calls() { let mut parser = Glm45MoeToolParser::new(&test_tools()); From 866f02db8a6944f87541ee9e77a6da6865c0066a Mon Sep 17 00:00:00 2001 From: aoshen02 Date: Fri, 2 Oct 2026 10:14:37 +0800 Subject: [PATCH 231/248] [Feature] Release WorkspaceManager scratch on sleep (#59156) Signed-off-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- .../memory/cumem/test_cumem.py | 94 +++++++++++++++++++ vllm/device_allocator/__init__.py | 3 + vllm/device_allocator/cumem.py | 33 ++++--- vllm/device_allocator/xpumem.py | 15 ++- vllm/v1/worker/gpu_worker.py | 2 + vllm/v1/worker/workspace.py | 16 +++- 6 files changed, 140 insertions(+), 23 deletions(-) diff --git a/tests/basic_correctness/memory/cumem/test_cumem.py b/tests/basic_correctness/memory/cumem/test_cumem.py index 135a3fc1ed..36a54ae219 100644 --- a/tests/basic_correctness/memory/cumem/test_cumem.py +++ b/tests/basic_correctness/memory/cumem/test_cumem.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import gc + import pytest import torch @@ -233,6 +235,98 @@ def test_discard_tags(): assert torch.allclose(weights, torch.ones_like(weights)) +@pytest.mark.parametrize("tag", ["workspace", None], ids=["workspace", "default"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +def test_selective_wake_restores_internal_tags(tag): + """A selective wake defers only weights/kv_cache; other tags always wake.""" + allocator = get_mem_allocator_instance() + with allocator.use_memory_pool(tag): + internal = torch.empty(16 << 20, dtype=torch.uint8, device=DEVICE_TYPE) + with allocator.use_memory_pool("kv_cache"): + kv = torch.empty(32 << 20, dtype=torch.uint8, device=DEVICE_TYPE) + + allocator.sleep(offload_tags=()) + assert mapped_usage(allocator) == 0 + + allocator.wake_up(tags=["weights"]) + assert mapped_usage(allocator) == internal.nbytes + allocator.wake_up(tags=["kv_cache"]) + assert mapped_usage(allocator) == internal.nbytes + kv.nbytes + + +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") +def test_workspace_scratch_discarded_on_sleep(): + """Workspace scratch lives in the sleep-mode pool: sleep discards it, any + (selective) wake remaps it at the same address so captured graphs replay, + and a buffer outgrown by a resize stays usable until it is released.""" + from vllm.device_allocator.sleep_mode_backend import CuMemBackend + from vllm.v1.worker.workspace import WorkspaceManager + + allocator = get_mem_allocator_instance() + manager = WorkspaceManager( + torch.device(DEVICE_TYPE), + alloc_context=lambda: allocator.use_memory_pool("workspace"), + ) + (small,) = manager.get_simultaneous(((32 << 20,), torch.uint8)) + # Resize while a view of the old buffer is alive: the tag is re-entered. + (scratch,) = manager.get_simultaneous(((64 << 20,), torch.uint8)) + assert len(allocator.allocator_and_pools["workspace"]) == 2 + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + scratch.add_(1) + + backend = CuMemBackend() + backend.suspend(level=1) + assert mapped_usage(allocator) == 0 + # Discarded, not copied. + assert all( + d.tag == "workspace" and d.cpu_backup_tensor is None + for d in allocator.pointer_to_data.values() + ) + + backend.resume(tags=["kv_cache"]) + assert mapped_usage(allocator) == small.nbytes + scratch.nbytes + scratch.zero_() + graph.replay() + assert int(scratch.sum()) == scratch.numel() + small.fill_(1) + + # The next resize releases the outgrown buffer; wake does not remap it. + del small + (big,) = manager.get_simultaneous(((128 << 20,), torch.uint8)) + assert mapped_usage(allocator) == scratch.nbytes + big.nbytes + backend.suspend(level=1) + backend.resume(tags=["weights"]) + assert mapped_usage(allocator) == scratch.nbytes + big.nbytes + + +@pytest.mark.parametrize("free_x_early", [True, False], ids=["x-freed", "x-alive"]) +@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") +@pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") +def test_reentered_tag(free_x_early): + """Enter a tag twice (two workspace ubatches): freeing X returns its memory + by the next entry, without touching Y or breaking sleep/wake.""" + allocator = get_mem_allocator_instance() + nbytes = 64 << 20 + with allocator.use_memory_pool("t"): + xs = [torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE)] + if free_x_early: + xs.clear() + with allocator.use_memory_pool("t"): + y = torch.empty(nbytes, dtype=torch.uint8, device=DEVICE_TYPE) + xs.clear() + gc.collect() + with allocator.use_memory_pool("t"): + pass + assert mapped_usage(allocator) == nbytes + y.fill_(1) + allocator.sleep(offload_tags="t") + allocator.wake_up() + assert int(y.sum()) == y.numel() + allocator.release_pools() + + @create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn") @pytest.mark.skipif(current_platform.is_xpu(), reason="Uses the CuMem allocator") def test_level2_discards_ordinary_tensor_with_weights_tag(): diff --git a/vllm/device_allocator/__init__.py b/vllm/device_allocator/__init__.py index 02d5160316..b70b7430b4 100644 --- a/vllm/device_allocator/__init__.py +++ b/vllm/device_allocator/__init__.py @@ -13,6 +13,9 @@ from vllm.platforms import current_platform # py_handle has type list[int] on ROCm and int otherwise HandleType: TypeAlias = tuple[int, int, int, list[int] | int] +# Selective wake may defer only these; any wake restores every other tag. +DEFERRABLE_TAGS: tuple[str, ...] = ("weights", "kv_cache") + @dataclasses.dataclass class AllocationData: diff --git a/vllm/device_allocator/cumem.py b/vllm/device_allocator/cumem.py index 2e5612e5fa..6bf0ea71f8 100644 --- a/vllm/device_allocator/cumem.py +++ b/vllm/device_allocator/cumem.py @@ -16,7 +16,7 @@ from typing import Any import torch -from vllm.device_allocator import AllocationData, HandleType +from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType from vllm.device_allocator.alloc_conf import ( EXPANDABLE_SEGMENTS, conf_flag_enabled, @@ -139,7 +139,7 @@ class CuMemAllocator: def __init__(self): self.pointer_to_data: dict[int, AllocationData] = {} self.current_tag: str = CuMemAllocator.default_tag - self.allocator_and_pools: dict[str, Any] = {} + self.allocator_and_pools: dict[str, list[Any]] = {} # Creating strong references to the two callbacks here to prevent # these ephemeral bound-method objects being garbage collected. # See discussions in https://github.com/vllm-project/vllm/pull/22724 @@ -167,7 +167,7 @@ class CuMemAllocator: if not self.allocator_and_pools: return - pool_entries = list(self.allocator_and_pools.values()) + pool_entries = [e for es in self.allocator_and_pools.values() for e in es] self.allocator_and_pools.clear() mem_pools = [entry[0] for entry in pool_entries] @@ -334,9 +334,9 @@ class CuMemAllocator: memory, and the rest of the data will have empty memory. Args: - tags: The tags of the memory allocation that will be loaded - back to GPU memory. If None, all memory allocation will be loaded - back to GPU memory. + tags: The deferrable tags (weights, kv_cache) to load back to GPU + memory; every other tag is always loaded back. If None, all + memory allocation will be loaded back to GPU memory. """ gc.collect() @@ -345,7 +345,7 @@ class CuMemAllocator: for ptr, data in self.pointer_to_data.items(): if not data.is_asleep: continue - if tags is None or data.tag in tags: + if tags is None or data.tag in tags or data.tag not in DEFERRABLE_TAGS: handle = data.handle create_and_map(handle) data.is_asleep = False @@ -392,6 +392,10 @@ class CuMemAllocator: old_tag = self.current_tag self.current_tag = tag try: + if tag != old_tag: + # Older pools of this tag are never reused, so trim them. + for pool, _ in self.allocator_and_pools.get(tag, []): + self._trim(pool) with use_memory_pool_with_allocator( self.python_malloc_callback, self.python_free_callback ) as data: @@ -400,7 +404,7 @@ class CuMemAllocator: # and the memory pool. # to avoid the issue, we keep a reference of the data. # see https://github.com/pytorch/pytorch/issues/146431 . - self.allocator_and_pools[tag] = data + self.allocator_and_pools.setdefault(tag, []).append(data) yield # PyTorch's bug, calling torch.cuda.empty_cache() will error # when using pluggable allocator, see @@ -413,16 +417,19 @@ class CuMemAllocator: # TODO: we should expose `empty_cache` method in the memory # pool. # TODO: ask for help from PyTorch team to expose this method. - allocations = data[0].snapshot() - for allocation in allocations: - if allocation["allocated_size"] == 0: - handle = self._python_free_callback(allocation["address"]) - unmap_and_release(handle) + self._trim(data[0]) finally: self.current_tag = old_tag if expandable_was_enabled: set_alloc_conf(prev_conf) + def _trim(self, pool: Any) -> None: + """Release free segments of a pool that is never allocated from again.""" + for allocation in pool.snapshot(): + data = self.pointer_to_data.get(allocation["address"]) + if allocation["allocated_size"] == 0 and data and not data.is_asleep: + unmap_and_release(self._python_free_callback(allocation["address"])) + def get_current_usage(self) -> int: """Get the total number of bytes allocated in the memory pool.""" sum_bytes: int = 0 diff --git a/vllm/device_allocator/xpumem.py b/vllm/device_allocator/xpumem.py index 17fa69ee84..7d4b40a78e 100644 --- a/vllm/device_allocator/xpumem.py +++ b/vllm/device_allocator/xpumem.py @@ -9,7 +9,7 @@ from typing import Any import torch -from vllm.device_allocator import AllocationData, HandleType +from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType from vllm.logger import init_logger from vllm.utils.torch_utils import PIN_MEMORY @@ -145,7 +145,7 @@ class XpuMemAllocator: def __init__(self): self.pointer_to_data: dict[int, AllocationData] = {} self.current_tag: str = XpuMemAllocator.default_tag - self.allocator_and_pools: dict[str, Any] = {} + self.allocator_and_pools: dict[str, list[Any]] = {} self.python_malloc_callback = self._python_malloc_callback self.python_free_callback = self._python_free_callback @@ -267,7 +267,11 @@ class XpuMemAllocator: for ptr, data in self.pointer_to_data.items(): if not data.is_asleep: continue - if tags is not None and data.tag not in tags: + if ( + tags is not None + and data.tag not in tags + and data.tag in DEFERRABLE_TAGS + ): continue create_and_allocate(data.handle) data.is_asleep = False @@ -300,7 +304,7 @@ class XpuMemAllocator: # MemPool teardown may invoke allocator virtual methods (e.g. raw_delete) # when releasing cached blocks. If allocator wrappers are dropped first, # C++ can hit "pure virtual method called" during shutdown. - pool_entries = list(self.allocator_and_pools.values()) + pool_entries = [e for es in self.allocator_and_pools.values() for e in es] self.allocator_and_pools.clear() mem_pools = [entry[0] for entry in pool_entries] @@ -333,7 +337,8 @@ class XpuMemAllocator: self.python_malloc_callback, self.python_free_callback, ) as data: - self.allocator_and_pools[tag] = data + # Keep every entry's pool alive (pytorch/pytorch#146431). + self.allocator_and_pools.setdefault(tag, []).append(data) yield finally: self.current_tag = old_tag diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index e9021bd142..15e2ee0611 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -532,6 +532,8 @@ class Worker(WorkerBase): self.device, num_ubatches, _num_workspace_lanes(self.vllm_config, self.use_v2_model_runner), + # Scratch holds no state across steps: discard it on sleep. + alloc_context=lambda: self._maybe_get_memory_pool_context("workspace"), ) self.model_runner: GPUModelRunner = self._make_model_runner() if self.rank == 0: diff --git a/vllm/v1/worker/workspace.py b/vllm/v1/worker/workspace.py index a86110f2f8..4bae1f52be 100644 --- a/vllm/v1/worker/workspace.py +++ b/vllm/v1/worker/workspace.py @@ -4,7 +4,7 @@ import inspect import os from collections.abc import Callable, Hashable, Iterator -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager, nullcontext from contextvars import ContextVar from itertools import accumulate from math import prod @@ -58,8 +58,10 @@ class WorkspaceManager: device: torch.device, num_ubatches: int | None = None, num_lanes: int = 1, + alloc_context: Callable[[], AbstractContextManager] = nullcontext, ): self._device = device + self._alloc_context = alloc_context # Cache num ubatches at init based on configuration (default to 1) self._num_ubatches = num_ubatches if num_ubatches is not None else 1 if num_lanes < 1: @@ -261,9 +263,11 @@ class WorkspaceManager: # dead segment in reserved memory which can cause higher peak # memory usage. torch.accelerator.empty_cache() - self._current_workspaces[workspace_id] = torch.empty( - (required_bytes,), dtype=torch.uint8, device=self._device - ) + # In sleep mode, entering the pool also trims outgrown buffers. + with self._alloc_context(): + self._current_workspaces[workspace_id] = torch.empty( + (required_bytes,), dtype=torch.uint8, device=self._device + ) current_workspace = self._current_workspaces[workspace_id] if envs.VLLM_DEBUG_WORKSPACE: @@ -308,6 +312,7 @@ def init_workspace_manager( device: torch.device, num_ubatches: int | None = None, num_lanes: int = 1, + alloc_context: Callable[[], AbstractContextManager] = nullcontext, ) -> None: """Initialize the workspace manager with a device. @@ -318,6 +323,7 @@ def init_workspace_manager( device: The device to allocate workspace on. num_ubatches: Number of workspace ubatch slots. Defaults to 1. num_lanes: Number of independent execution lanes per ubatch. Defaults to 1. + alloc_context: Factory for the context scratch is allocated in. """ global _manager @@ -328,7 +334,7 @@ def init_workspace_manager( _manager._device, device, ) - _manager = WorkspaceManager(device, num_ubatches, num_lanes) + _manager = WorkspaceManager(device, num_ubatches, num_lanes, alloc_context) def lock_workspace() -> None: From e3cae8d2ac6be8dfd319f3bb8061a1ba4d19bf52 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:25:18 +0100 Subject: [PATCH 232/248] [Model] Use upstream GLM-5.3 and Qwen4-Exp configs and processor (#57387) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- requirements/common.txt | 2 +- .../models/glm5next/test_sequence_parallel.py | 2 +- .../multimodal/processing/test_glm5next.py | 49 +- tests/models/qwen4_exp/test_config.py | 10 +- tests/models/qwen4_exp/test_ple.py | 2 +- tests/multimodal/test_video.py | 18 +- tests/test_config.py | 44 + .../processors/test_glm5next.py | 389 ------- tests/transformers_utils/test_config.py | 53 - tests/v1/spec_decode/test_mtp.py | 2 +- vllm/config/model.py | 13 +- vllm/config/speculative.py | 2 +- vllm/models/glm5next/common/attention.py | 32 +- vllm/models/glm5next/common/kda.py | 4 +- vllm/models/glm5next/common/model.py | 100 +- vllm/models/glm5next/common/mtp.py | 8 +- vllm/models/glm5next/common/multimodal.py | 122 ++- vllm/models/qwen4_exp/amd/indexer_qsa.py | 4 +- vllm/models/qwen4_exp/amd/model.py | 47 +- vllm/models/qwen4_exp/amd/mtp.py | 6 +- vllm/models/qwen4_exp/amd/ple_layer.py | 4 +- vllm/models/qwen4_exp/amd/qsa.py | 4 +- vllm/models/qwen4_exp/config.py | 263 ----- vllm/models/qwen4_exp/nvidia/indexer_qsa.py | 4 +- vllm/models/qwen4_exp/nvidia/model.py | 47 +- vllm/models/qwen4_exp/nvidia/mtp.py | 6 +- .../qwen4_exp/nvidia/ngram_embedding.py | 4 +- vllm/models/qwen4_exp/nvidia/ple_layer.py | 4 +- vllm/models/qwen4_exp/nvidia/qsa.py | 4 +- vllm/multimodal/video.py | 95 +- vllm/transformers_utils/config.py | 5 - vllm/transformers_utils/configs/__init__.py | 12 - vllm/transformers_utils/configs/glm5_next.py | 420 -------- vllm/transformers_utils/configs/qwen4_exp.py | 15 - .../transformers_utils/processors/__init__.py | 2 - .../transformers_utils/processors/glm5next.py | 953 ------------------ 36 files changed, 366 insertions(+), 2385 deletions(-) delete mode 100644 tests/transformers_utils/processors/test_glm5next.py delete mode 100644 vllm/models/qwen4_exp/config.py delete mode 100644 vllm/transformers_utils/configs/glm5_next.py delete mode 100644 vllm/transformers_utils/configs/qwen4_exp.py delete mode 100644 vllm/transformers_utils/processors/glm5next.py diff --git a/requirements/common.txt b/requirements/common.txt index eec3c050bc..70bfa93253 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -7,7 +7,7 @@ requests >= 2.33.0 tqdm blake3 py-cpuinfo -transformers >= 5.10.4 +transformers >= 5.16.1 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export tokenizers >= 0.21.1 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 diff --git a/tests/models/glm5next/test_sequence_parallel.py b/tests/models/glm5next/test_sequence_parallel.py index 2769f786bb..c0f59d6186 100644 --- a/tests/models/glm5next/test_sequence_parallel.py +++ b/tests/models/glm5next/test_sequence_parallel.py @@ -15,11 +15,11 @@ from types import SimpleNamespace import pytest import torch from torch import nn +from transformers import Glm5NextTextConfig from vllm.model_executor import parameter from vllm.model_executor.layers import linear from vllm.models.glm5next.common import model as glm_model -from vllm.transformers_utils.configs.glm5_next import Glm5NextTextConfig HIDDEN, INTERMEDIATE, TP_SIZE = 8, 16, 2 diff --git a/tests/models/multimodal/processing/test_glm5next.py b/tests/models/multimodal/processing/test_glm5next.py index 0e1e7b9876..719154a2b4 100644 --- a/tests/models/multimodal/processing/test_glm5next.py +++ b/tests/models/multimodal/processing/test_glm5next.py @@ -15,16 +15,15 @@ The checks below are arithmetic: no weights, no GPU. import pytest import torch from PIL import Image +from transformers.models.glm5_next.video_processing_glm5_next import ( + Glm5NextVideoProcessor, + smart_resize, +) +from transformers.video_utils import VideoMetadata from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform -from vllm.transformers_utils.processors.glm5next import ( - Glm5NextVideoProcessor, - _pixel_budget, - glm_sample_frame_indices, - smart_resize, -) from ...utils import build_model_context @@ -48,27 +47,16 @@ def _pixel_path_grid( width: int, ) -> tuple[int, int, int]: """The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds.""" - min_pixels, max_pixels = _pixel_budget( - video_processor.min_image_tokens, - video_processor.max_image_tokens, - video_processor.patch_size, - video_processor.merge_size, - video_processor.temporal_patch_size, - ) - factor = ( - video_processor.patch_size - * video_processor.merge_size - * video_processor.patch_expand_factor - ) resized_height, resized_width = smart_resize( - t=num_frames, - h=height, - w=width, - t_factor=video_processor.temporal_patch_size, - h_factor=factor, - w_factor=factor, - min_pixels=min_pixels, - max_pixels=max_pixels, + num_frames=num_frames, + height=height, + width=width, + temporal_factor=video_processor.temporal_patch_size, + factor=video_processor.patch_size + * video_processor.merge_size + * video_processor.patch_expand_factor, + min_pixels=video_processor.min_image_tokens, + max_pixels=video_processor.max_image_tokens, ) padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size) return ( @@ -104,13 +92,8 @@ def test_video_placeholders_match_encoder_rows( info = processor.info video_processor = info.get_video_processor() - frame_indices = glm_sample_frame_indices( - total_num_frames, - fps, - duration, - target_fps=video_processor.fps_interval, - max_frame_count=video_processor.max_frame_count_dynamic, - temporal_patch_size=video_processor.temporal_patch_size, + frame_indices = video_processor.sample_frames( + VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration) ) grid_t, grid_h, grid_w = _pixel_path_grid( video_processor, len(frame_indices), height, width diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py index 9d30365b81..6fde77eda2 100644 --- a/tests/models/qwen4_exp/test_config.py +++ b/tests/models/qwen4_exp/test_config.py @@ -7,16 +7,13 @@ from unittest.mock import patch import pytest import torch +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig from vllm.config.speculative import SpeculativeConfig from vllm.model_executor.models.config import ( Qwen3_5ForConditionalGenerationConfig, Qwen4ExpForConditionalGenerationConfig, ) -from vllm.models.qwen4_exp.config import ( - Qwen4ExpConfig, - Qwen4ExpTextConfig, -) from vllm.models.qwen4_exp.nvidia.model_state import Qwen4ExpModelState from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState @@ -37,7 +34,10 @@ def _text_config(**kwargs) -> Qwen4ExpTextConfig: "linear_num_value_heads": 2, "linear_key_head_dim": 8, "linear_value_head_dim": 8, - "num_experts": 0, + "num_experts": 4, + "num_experts_per_tok": 2, + # `Qwen4ExpTextConfig` requires an EOS token whenever PLE is enabled. + "eos_token_id": 1, "hc_count": 2, "hc_lowrank": 4, "ple_layer_ids": [1], diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py index d271877dfc..fb191c1b4f 100644 --- a/tests/models/qwen4_exp/test_ple.py +++ b/tests/models/qwen4_exp/test_ple.py @@ -10,6 +10,7 @@ import pytest import torch from torch import nn from torch.nn import functional as F +from transformers import Qwen4ExpTextConfig import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module import vllm.model_executor.parameter as parameter_module @@ -33,7 +34,6 @@ from vllm.models.qwen4_exp.common.ple import ( compute_ple_shard_overlap, copy_ple_embedding_shard_, ) -from vllm.models.qwen4_exp.config import Qwen4ExpTextConfig from vllm.models.qwen4_exp.nvidia.ngram_embedding import ( Qwen4ExpNGramEmbedding, Qwen4ExpPLEDeviceEmbedding, diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 2bb77b2ed7..d593edba29 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -1591,10 +1591,6 @@ def test_glm5next_backend_indices_match_sampler( ): """The loader must select exactly the frames the processor's sampler would, with target.fps mapping onto the raw-fps override.""" - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - source = VideoSourceMetadata( total_frames_num=total_frames, original_fps=original_fps, duration=duration ) @@ -1604,7 +1600,7 @@ def test_glm5next_backend_indices_match_sampler( source, target, max_frames=max_frames ) - assert indices == glm_sample_frame_indices( + assert indices == Glm5NextVideoBackend._sample_frame_indices( total_frames, original_fps, duration, @@ -1692,10 +1688,6 @@ def test_glm5next_backend_codec_parity(tmp_path, backend): if backend == "torchcodec": pytest.importorskip("torchcodec") - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - total_frames, fps = 120, 10 path = _write_gray_video(tmp_path, total_frames, fps) # Dense default sampling (gap 5) and a sparse max_frames cap (gap 20). @@ -1703,7 +1695,7 @@ def test_glm5next_backend_codec_parity(tmp_path, backend): kwargs: dict[str, Any] = ( {} if max_frames is None else {"max_frames": max_frames} ) - expected = glm_sample_frame_indices( + expected = Glm5NextVideoBackend._sample_frame_indices( total_frames, float(fps), 12.0, max_frame_count=max_frames ) @@ -1726,11 +1718,7 @@ def test_glm5next_backend_decodes_only_sampled_frames(tmp_path): total_frames, fps = 60, 10 path = _write_gray_video(tmp_path, total_frames, fps) - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - - expected = glm_sample_frame_indices(total_frames, float(fps), 6.0) + expected = Glm5NextVideoBackend._sample_frame_indices(total_frames, float(fps), 6.0) frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes()) diff --git a/tests/test_config.py b/tests/test_config.py index 48d286b521..a2e75b1072 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4114,3 +4114,47 @@ def test_revision_resolved_for_model(mock_resolve): assert isinstance(config.revision, ResolvedRevision) assert config.revision.resolved == REVISION mock_resolve.assert_any_call(model, None, config.hf_token) + + +@pytest.mark.parametrize( + ("layer_types", "expected_attention"), + [ + # Qwen3-Next / Qwen3.5 spell their attention layers "full_attention". + (["linear_attention", "full_attention"], 1), + # GLM-5.3-Flash and Qwen4-Exp use sparse attention, which still caches + # every token and so must count as attention. + (["linear_attention", "deepseek_sparse_attention"], 1), + (["linear_attention", "qwen_sparse_attention"], 1), + (["linear_attention", "linear_attention"], 0), + ], +) +def test_hybrid_layer_counts_sparse_attention_as_attention( + layer_types, expected_attention +): + """Sparse-attention layer types consume a full attention KV cache.""" + model_config = object.__new__(ModelConfig) + model_config.hf_config = SimpleNamespace() + model_config.hf_text_config = SimpleNamespace(layer_types=layer_types) + model_config.model_arch_config = SimpleNamespace(text_model_type="glm5_next") + + parallel_config = SimpleNamespace() + with ( + patch.object(ModelConfig, "is_hybrid", True), + patch.object(ModelConfig, "has_noops", False), + patch.object(ModelConfig, "is_attention_free", False), + patch.object( + ModelConfig, + "get_layers_start_end_indices", + return_value=(0, len(layer_types)), + ), + ): + assert ( + model_config.get_num_layers_by_block_type(parallel_config, "attention") + == expected_attention + ) + assert ( + model_config.get_num_layers_by_block_type( + parallel_config, "linear_attention" + ) + == len(layer_types) - expected_attention + ) diff --git a/tests/transformers_utils/processors/test_glm5next.py b/tests/transformers_utils/processors/test_glm5next.py deleted file mode 100644 index 917a3f627c..0000000000 --- a/tests/transformers_utils/processors/test_glm5next.py +++ /dev/null @@ -1,389 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor.""" - -import math - -import pytest -import torch -from PIL import Image -from transformers.image_utils import PILImageResampling -from transformers.video_utils import VideoMetadata - -from vllm.transformers_utils.processors.glm5next import ( - Glm5NextImageProcessor, - Glm5NextVideoProcessor, - _get_pad_content_size, - _resize_or_pad, - glm_sample_frame_indices, - smart_resize, -) - -PATCH_SIZE = 14 -MERGE_SIZE = 2 -PATCH_EXPAND_FACTOR = 1 # checkpoint processor_config.json -FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR # 28 -PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2 # 1568 -MIN_TOKENS = 16 -MAX_TOKENS = 800 # serving-cap equivalent budget, keeps test canvases small -MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN -MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN - - -def resize(height: int, width: int, t: int = 2) -> tuple[int, int]: - return smart_resize( - t, - height, - width, - t_factor=2, - h_factor=FACTOR, - w_factor=FACTOR, - min_pixels=MIN_PIXELS, - max_pixels=MAX_PIXELS, - ) - - -@pytest.mark.parametrize( - ("height", "width", "t", "expected"), - [ - (300, 400, 2, (308, 420)), # ceil alignment, no budget pressure - (50, 80, 2, (112, 168)), # below min budget: proportional upscale - (57, 3, 2, (504, 28)), # slim side ceils to one factor - (2160, 3840, 2, (588, 1064)), # shrink: max canvas under the cap - (112, 11200, 2, (84, 7420)), # extreme aspect stays proportional - (300, 400, 16, (252, 308)), # frame count eats into the budget - (112, 11200, 16, (28, 2800)), # slim side stays positive, budget holds - ], -) -def test_smart_resize_reference_values(height, width, t, expected): - got = smart_resize( - t, - height, - width, - t_factor=2, - h_factor=FACTOR, - w_factor=FACTOR, - min_pixels=MIN_PIXELS, - max_pixels=MAX_PIXELS, - ) - assert got == expected - # The aligned canvas never exceeds the budget it was fitted against. - t_bar = max(2, round(t / 2) * 2) - assert t_bar * got[0] * got[1] <= MAX_PIXELS - - -def test_smart_resize_stays_snapped_and_positive(): - heights = (1, 27, 57, 111, 113, 300, 720, 2160) - widths = (1, 27, 57, 111, 113, 400, 1280, 11200) - for h in heights: - for w in widths: - for t in (2, 4, 16, 64): - resized_h, resized_w = resize(h, w, t) - assert resized_h > 0 and resized_w > 0 - assert resized_h % FACTOR == 0 - assert resized_w % FACTOR == 0 - t_bar = max(2, round(t / 2) * 2) - assert t_bar * resized_h * resized_w <= MAX_PIXELS - - -@pytest.mark.parametrize( - ("height", "width", "match"), - [ - (0, 100, "must be positive"), - (100, 0, "must be positive"), - ], -) -def test_smart_resize_rejects_degenerate_inputs(height, width, match): - with pytest.raises(ValueError, match=match): - resize(height, width, 2) - - -def test_smart_resize_rejects_inverted_budget(): - with pytest.raises(ValueError, match="min_pixels must be less than or equal"): - smart_resize( - 2, - 100, - 100, - t_factor=2, - h_factor=FACTOR, - w_factor=FACTOR, - min_pixels=MAX_PIXELS, - max_pixels=MIN_PIXELS, - ) - - -def test_get_pad_content_size(): - # Oversized content shrinks proportionally, never upscales by default. - assert _get_pad_content_size(300, 400, 308, 420) == (300, 400) - assert _get_pad_content_size(600, 800, 308, 420) == (308, 410) - # allow_upscale enlarges small content toward the canvas. - assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112) - - -def test_resize_or_pad_pads_right_and_bottom(): - stacked = torch.rand(1, 3, 300, 400) - - def identity_resize(x, size, resample=None): - assert (size.height, size.width) == (300, 400) - return x - - padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize) - assert padded.shape == (1, 3, 308, 420) - torch.testing.assert_close(padded[..., :300, :400], stacked) - assert padded[..., 300:, :].eq(0).all() - assert padded[..., :, 400:].eq(0).all() - - def force_resize(x, size, resample=None): - return torch.zeros(x.shape[0], x.shape[1], size.height, size.width) - - assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == ( - 1, - 3, - 308, - 420, - ) - with pytest.raises(ValueError, match="resize_mode"): - _resize_or_pad(stacked, 308, 420, "crop", None, identity_resize) - - -@pytest.fixture(scope="module") -def image_processor(): - return Glm5NextImageProcessor( - min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS - ) - - -@pytest.mark.parametrize( - ("height", "width"), - [(50, 80), (300, 400), (2160, 3840), (112, 11200)], -) -def test_image_grid_matches_smart_resize(image_processor, height, width): - out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt") - resized_h, resized_w = resize(height, width) - grid = out["image_grid_thw"][0].tolist() - assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE] - assert out["pixel_values"].shape == ( - grid[1] * grid[2], - 3 * 2 * PATCH_SIZE * PATCH_SIZE, - ) - - -@pytest.mark.parametrize( - ("height", "width"), - [(50, 80), (300, 400), (2160, 3840), (112, 11200)], -) -def test_image_patch_count_matches_preprocess(image_processor, height, width): - out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt") - grid = out["image_grid_thw"][0].tolist() - expected = grid[1] * grid[2] - assert image_processor.get_number_of_image_patches(height, width) == expected - - -@pytest.fixture(scope="module") -def video_processor(): - return Glm5NextVideoProcessor( - min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS - ) - - -def run_video_preprocess(video_processor, frames, **overrides): - kwargs = dict( - resample=PILImageResampling.BICUBIC, - image_mean=(0.48145466, 0.4578275, 0.40821073), - image_std=(0.26862954, 0.26130258, 0.27577711), - patch_size=PATCH_SIZE, - temporal_patch_size=2, - merge_size=MERGE_SIZE, - patch_expand_factor=PATCH_EXPAND_FACTOR, - return_tensors="pt", - ) - kwargs.update(overrides) - return video_processor._preprocess([frames], **kwargs) - - -def test_video_preprocess_pads_odd_frame_count(video_processor): - out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400)) - grid = out["video_grid_thw"][0].tolist() - # 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420). - assert grid == [4, 22, 30] - assert out["pixel_values_videos"].shape == ( - 4 * 22 * 30, - 3 * 2 * PATCH_SIZE * PATCH_SIZE, - ) - - -def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor): - out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200)) - # Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap. - assert out["video_grid_thw"][0].tolist() == [8, 2, 200] # height 28, not 0 - - -def test_video_preprocess_pads_content_not_distort(video_processor): - frames = torch.zeros(4, 3, 300, 400) - frames[..., 50, 60] = 1.0 # marker inside the content area - # Bypass rescale/normalize so zero padding stays exactly zero. - out = run_video_preprocess( - video_processor, frames, do_rescale=False, do_normalize=False - ) - # Pad mode keeps the 300x400 content aspect on the (308, 420) canvas - # with zero padding on the right/bottom. - grid = out["video_grid_thw"][0].tolist() - assert grid == [2, 22, 30] - patches = out["pixel_values_videos"].view( - grid[0], - grid[1] // MERGE_SIZE, - grid[2] // MERGE_SIZE, - MERGE_SIZE, - MERGE_SIZE, - 3 * 2 * PATCH_SIZE * PATCH_SIZE, - ) - frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1) - # Patch columns fully right of the 400px content are pure padding. - assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0 - - -@pytest.mark.parametrize( - ("total_frames", "fps", "duration", "expected_len", "first", "last"), - [ - # Dense source: the tp-scaled greedy overshoots extract_t, so the - # fixup spreads picks uniformly across the whole video (linspace). - (900, 30.0, 30.0, 60, 0, 899), - (3000, 30.0, 100.0, 200, 0, 2999), - # extract_t capped at 2048 frames. - (72000, 30.0, 2400.0, 2048, 0, 71999), - # Low container fps: the greedy picks every frame. - (48, 2.0, 24.0, 48, 0, 47), - # Duration derived from the frame count when metadata lacks it. - (300, 25.0, 0, 26, 0, 299), - ], -) -def test_glm_sample_frame_indices_behaviour( - total_frames, fps, duration, expected_len, first, last -): - indices = glm_sample_frame_indices(total_frames, fps, duration) - assert len(indices) == expected_len - assert indices[0] == first - assert indices[-1] == last - assert len(indices) % 2 == 0 - assert indices == sorted(indices) - assert all(0 <= idx < total_frames for idx in indices) - - -def test_glm_sample_frame_indices_short_clip_floor_spread(): - # extract_t (20) > total (7): evenly spaced timestamps, deduplicated, - # then pair-padded to an even count. - assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6] - - -@pytest.mark.parametrize( - ("kwargs", "expected_len"), - [ - ({"target_fps": 0.5}, 16), - ({"max_frame_count": 16}, 16), - ({"target_fps": 8}, 240), # 30s * 8 = 240, under the 2048 cap - ], -) -def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len): - indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs) - assert len(indices) == expected_len - assert len(indices) % 2 == 0 - - -def test_sample_frames_fps_interval(video_processor): - indices = video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0) - ) - assert len(indices) == 60 - assert indices[0] == 0 - assert indices[-1] == 899 # uniform spread reaches the final frame - - # Request overrides reach the sampler through both kwarg spellings. The - # 15-frame spread is odd, so pair-padding duplicates the last frame. - assert ( - len( - video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - fps=0.5, - ) - ) - == 16 - ) - assert ( - len( - video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - target_fps=0.5, - ) - ) - == 16 - ) - assert ( - len( - video_processor.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - max_frames=16, - ) - ) - == 16 - ) - - -def test_defaults_mirror_checkpoint_config(): - """Bare instantiation matches the checkpoint's ``processor_config.json`` - token-budget style defaults; ``size`` carries no budget.""" - image_processor = Glm5NextImageProcessor() - assert image_processor.patch_expand_factor == 1 - assert image_processor.min_image_tokens == 16 - assert image_processor.max_image_tokens == 8000 - assert image_processor.resize_mode == "pad" - - video_processor = Glm5NextVideoProcessor() - assert video_processor.patch_expand_factor == 1 - assert video_processor.min_image_tokens == 16 - assert video_processor.max_image_tokens == 240000 - assert video_processor.fps_interval == 2.0 - assert video_processor.max_frame_count_dynamic == 2048 - - -def test_token_budgets_drive_geometry(): - """The token bounds fully determine the pixel budget and the grid.""" - token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512) - img = Image.new("RGB", (400, 300)) - grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist() - # Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420) - # without hitting either bound -> 22x30 patches. - assert grid == [1, 22, 30] - assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30 - - -def test_missing_token_budgets_rejected(): - proc = Glm5NextImageProcessor(min_image_tokens=None) - with pytest.raises(ValueError, match="min_image_tokens"): - proc(Image.new("RGB", (400, 300)), return_tensors="pt") - - -def test_video_config_fields_land(): - """fps_interval / max_frame_count_dynamic from the dedicated config - shape sampling without any request overrides.""" - proc = Glm5NextVideoProcessor( - fps_interval=4, - max_frame_count_dynamic=32, - ) - indices = proc.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0) - ) - # 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32. - assert len(indices) == 32 - assert indices[0] == 0 - assert indices[-1] == 899 - - # Request overrides still win over the config values. - assert ( - len( - proc.sample_frames( - VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0), - fps=0.5, - ) - ) - == 16 - ) diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index 19773dfda4..4fd27ba278 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -24,11 +24,6 @@ from vllm.transformers_utils.config import ( try_get_generation_config, uses_mrope, ) -from vllm.transformers_utils.configs.glm5_next import ( - Glm5NextConfig, - Glm5NextTextConfig, - Glm5NextVisionConfig, -) from vllm.transformers_utils.configs.mistral import adapt_config_dict @@ -174,54 +169,6 @@ def test_mistral_yarn_apply_scale_false_disables_yarn_magnitude_scaling(): assert config.rope_parameters["attention_factor"] == 1.0 -def test_glm5_next_accepts_deepseek_sparse_attention_layers(): - layer_types = ["linear_attention", "deepseek_sparse_attention"] - - config = Glm5NextTextConfig( - num_hidden_layers=len(layer_types), layer_types=layer_types - ) - - assert config.layer_types == layer_types - assert config.layers_block_type == ["linear_attention", "attention"] - - -def test_glm5_next_accepts_prebuilt_subconfigs(): - text_config = Glm5NextTextConfig(hidden_size=1024) - vision_config = Glm5NextVisionConfig(hidden_size=768) - - config = Glm5NextConfig( - text_config=text_config, - vision_config=vision_config, - ) - - assert config.text_config is text_config - assert config.vision_config is vision_config - - -@pytest.mark.parametrize( - ("kwargs", "option"), - [ - ( - {"index_topk": 2048, "index_dsa_use_layernorm": False}, - "index_dsa_use_layernorm", - ), - ( - {"index_topk": 2048, "index_kpool_compress": False}, - "index_kpool_compress", - ), - ( - {"index_topk": 2048, "index_kpool_always_select_tail": False}, - "index_kpool_always_select_tail", - ), - ({"hres_vwnstyle": False}, "hres_vwnstyle"), - ({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"), - ], -) -def test_glm5_next_rejects_unimplemented_config_options(kwargs, option): - with pytest.raises(NotImplementedError, match=option): - Glm5NextTextConfig(**kwargs) - - def test_get_llama3_eos_token(): model_name = "meta-llama/Llama-3.2-1B-Instruct" diff --git a/tests/v1/spec_decode/test_mtp.py b/tests/v1/spec_decode/test_mtp.py index 46c7064b0d..112ff77167 100644 --- a/tests/v1/spec_decode/test_mtp.py +++ b/tests/v1/spec_decode/test_mtp.py @@ -43,7 +43,7 @@ def test_glm_mtp_defers_lm_head(default_vllm_config): index_kpool=4, ) vllm_config = mock.MagicMock() - vllm_config.speculative_config.draft_model_config.hf_config = config + vllm_config.speculative_config.draft_model_config.hf_text_config = config vllm_config.scheduler_config.max_num_batched_tokens = 4 with ( diff --git a/vllm/config/model.py b/vllm/config/model.py index 4c9ed9e13f..4de030e8f3 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -108,6 +108,16 @@ HfOverrides = dict[str, Any] | Callable[[PreTrainedConfig], PreTrainedConfig] ModelImpl = Literal["auto", "vllm", "transformers", "terratorch"] LayerBlockType = Literal["attention", "linear_attention", "mamba"] +_ATTENTION_LAYER_TYPES = frozenset( + { + "full_attention", + "deepseek_sparse_attention", + "qwen_sparse_attention", + } +) +"""`layer_types` spellings that consume a full attention KV cache. Sparse +attention still caches every token, so it counts as attention here.""" + _RUNNER_CONVERTS: dict[RunnerType, list[ConvertType]] = { "generate": [], "pooling": ["embed", "classify"], @@ -1718,7 +1728,8 @@ class ModelConfig: if layer_types_value is not None: if block_type == "attention": return sum( - t == "full_attention" for t in layer_types_value[start:end] + t in _ATTENTION_LAYER_TYPES + for t in layer_types_value[start:end] ) elif block_type == "linear_attention": return sum( diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index dbec9fd743..ff4f76153d 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -1059,7 +1059,7 @@ class SpeculativeConfig: ) if hf_config.model_type == "glm5_next": hf_config.model_type = "glm5_next_mtp" - n_predict = hf_config.num_nextn_predict_layers + n_predict = hf_config.get_text_config().num_nextn_predict_layers hf_config.update( {"n_predict": n_predict, "architectures": ["Glm5NextMTPModel"]} ) diff --git a/vllm/models/glm5next/common/attention.py b/vllm/models/glm5next/common/attention.py index 2e17424f1e..d7bf70af07 100644 --- a/vllm/models/glm5next/common/attention.py +++ b/vllm/models/glm5next/common/attention.py @@ -4,6 +4,7 @@ import torch import torch.nn.functional as F from torch import nn +from transformers import Glm5NextTextConfig from vllm.config import ( CacheConfig, @@ -32,7 +33,6 @@ from vllm.model_executor.utils import maybe_disable_graph_partition from vllm.models.glm5next.nvidia.ops.kpool_compress import fwht128_quant_fp8 from vllm.models.glm5next.sparse_indexer import SparseAttnIndexerKpool from vllm.platforms import current_platform -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES from vllm.utils.math_utils import cdiv, next_power_of_2 from vllm.v1.kv_cache_interface import KpoolTailSpec, MLAAttentionSpec @@ -221,7 +221,7 @@ class Indexer(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: Glm5NextConfig, + config: Glm5NextTextConfig, hidden_size: int, q_lora_rank: int, quant_config: QuantizationConfig | None, @@ -236,7 +236,7 @@ class Indexer(nn.Module): # self.indexer_cfg = config.attn_module_list_cfg[0]["attn_index"] # Indexer is only constructed for v32 configs, where these sparse-indexer # fields are guaranteed populated; narrow away the `int | None` declared - # on Glm5NextConfig for the optional-indexer case. + # on Glm5NextTextConfig for the optional-indexer case. assert config.index_topk is not None assert config.index_n_heads is not None assert config.index_head_dim is not None @@ -419,7 +419,7 @@ class Glm5NextMLAAttention(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: Glm5NextConfig, + config: Glm5NextTextConfig, hidden_size: int, num_heads: int, qk_nope_head_dim: int, @@ -506,28 +506,32 @@ class Glm5NextMLAAttention(nn.Module): prefix=f"{prefix}.o_proj", ) + # `Glm5NextTextConfig` is NoPE-only and so declares no `rope_parameters`; + # a checkpoint may still ship one. + rope_parameters = getattr(config, "rope_parameters", None) + if not skip_rope: - assert config.rope_parameters is not None - if config.rope_parameters["rope_type"] != "default": - config.rope_parameters["rope_type"] = ( + assert rope_parameters is not None + if rope_parameters["rope_type"] != "default": + rope_parameters["rope_type"] = ( "deepseek_llama_scaling" - if config.rope_parameters.get("attention_factor") == 1.0 + if rope_parameters.get("attention_factor") == 1.0 else "deepseek_yarn" ) self.rotary_emb: RotaryEmbedding | None = get_rope( qk_rope_head_dim, max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, + rope_parameters=rope_parameters, is_neox_style=False, ) if ( - config.rope_parameters["rope_type"] != "default" - and config.rope_parameters["rope_type"] == "deepseek_yarn" + rope_parameters["rope_type"] != "default" + and rope_parameters["rope_type"] == "deepseek_yarn" ): - mscale_all_dim = config.rope_parameters.get("mscale_all_dim", False) - scaling_factor = config.rope_parameters["factor"] + mscale_all_dim = rope_parameters.get("mscale_all_dim", False) + scaling_factor = rope_parameters["factor"] mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale else: @@ -539,7 +543,7 @@ class Glm5NextMLAAttention(nn.Module): self.indexer_rope_emb: RotaryEmbedding | None = get_rope( qk_rope_head_dim, max_position=max_position_embeddings, - rope_parameters=config.rope_parameters, + rope_parameters=rope_parameters, is_neox_style=not config.indexer_rope_interleave, ) # The sparse indexer projects from the MLA q-lora rank, which is diff --git a/vllm/models/glm5next/common/kda.py b/vllm/models/glm5next/common/kda.py index c67ac5b012..141a8cc7cb 100644 --- a/vllm/models/glm5next/common/kda.py +++ b/vllm/models/glm5next/common/kda.py @@ -6,6 +6,7 @@ from dataclasses import replace import torch from torch import nn +from transformers import Glm5NextTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig, get_current_vllm_config @@ -44,7 +45,6 @@ from vllm.model_executor.utils import ( ) from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata from vllm.v1.kv_cache_interface import MambaSpec from vllm.v1.worker.workspace import current_workspace_manager @@ -201,7 +201,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): def __init__( self, - config: Glm5NextConfig, + config: Glm5NextTextConfig, vllm_config: VllmConfig, prefix: str = "", ) -> None: diff --git a/vllm/models/glm5next/common/model.py b/vllm/models/glm5next/common/model.py index 3ff3476375..e15c00ecb8 100644 --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py @@ -6,6 +6,7 @@ from typing import ClassVar, Literal import torch from torch import nn +from transformers import Glm5NextTextConfig from vllm.config import ParallelConfig, VllmConfig from vllm.distributed import ( @@ -83,7 +84,6 @@ from vllm.models.common.ops.sequence_parallel import ( from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from .attention import Glm5NextMLAAttention from .kda import Glm5NextLinearAttention @@ -95,6 +95,57 @@ from .multimodal import ( logger = init_logger(__name__) +_MHC_TAU = 0.05 +"""mHC routing temperature. A GLM-5.3-Flash trained value that neither the +checkpoint nor `Glm5NextTextConfig` carries.""" + +_MHC_POST_MULT_VALUE = 2.0 +"""mHC post-multiplier. A GLM-5.3-Flash trained value that neither the +checkpoint nor `Glm5NextTextConfig` carries.""" + +_LOGIT_SCALE = 1.0 +"""Output logit scale. A GLM-5.3-Flash trained value that neither the +checkpoint nor `Glm5NextTextConfig` carries.""" + +_VISION_RMS_NORM_EPS = 1e-6 +"""Vision tower RMSNorm epsilon. + +GLM-5.3-Flash checkpoints ship `vision_config.rms_norm_eps = 1e-5`, but the +vision tower was trained with 1e-6. Serving with 1e-5 drifts the RMSNorm and +produces repetitive/degraded image descriptions, so force the trained value +regardless of the checkpoint field. +""" + + +def _is_moe(config: Glm5NextTextConfig) -> bool: + return config.n_routed_experts is not None + + +def _is_kda_layer(config: Glm5NextTextConfig, layer_idx: int) -> bool: + layer_types = config.layer_types + return ( + layer_types is not None + and layer_idx < len(layer_types) + and layer_types[layer_idx] == "linear_attention" + ) + + +def _is_linear_attn(config: Glm5NextTextConfig) -> bool: + layer_types = config.layer_types + return layer_types is not None and "linear_attention" in layer_types + + +def _validate_supported_config(config: Glm5NextTextConfig) -> None: + """Reject checkpoints using config options this implementation lacks. + + The kpool indexer kernels always keep the incomplete trailing pool, so a + checkpoint asking otherwise would be served silently wrong. + """ + if config.index_topk is not None and not config.index_kpool_always_select_tail: + raise NotImplementedError( + "GLM-5.3 sparse indexer requires index_kpool_always_select_tail=True" + ) + class Glm5NextMLP(nn.Module): def __init__( @@ -152,7 +203,7 @@ class Glm5NextMLP(nn.Module): class Glm5NextMoE(nn.Module): def __init__( self, - config: Glm5NextConfig, + config: Glm5NextTextConfig, parallel_config: ParallelConfig, quant_config: QuantizationConfig | None = None, prefix: str = "", @@ -227,10 +278,10 @@ class Glm5NextMoE(nn.Module): shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, - top_k=config.num_experts_per_token, + top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, - renormalize=config.moe_renormalize, + renormalize=config.norm_topk_prob, quant_config=quant_config, use_grouped_topk=True, num_expert_group=config.n_group, @@ -280,7 +331,7 @@ class Glm5NextDecoderLayer(nn.Module): def __init__( self, vllm_config: VllmConfig, - config: Glm5NextConfig, + config: Glm5NextTextConfig, layer_idx: int, prefix: str = "", topk_indices_buffer: torch.Tensor | None = None, @@ -295,13 +346,13 @@ class Glm5NextDecoderLayer(nn.Module): self.hidden_size = config.hidden_size self.layer_idx = layer_idx - self.is_moe = config.is_moe + self.is_moe = _is_moe(config) self.num_hidden_layers = config.num_hidden_layers self.rms_norm_eps = config.rms_norm_eps self.num_experts = config.n_routed_experts self.is_mtp_layer = is_mtp_layer self.mhc = config.mhc - is_kda_layer = not is_mtp_layer and config.is_kda_layer(layer_idx) + is_kda_layer = not is_mtp_layer and _is_kda_layer(config, layer_idx) self.layer_kind = "kda" if is_kda_layer else "mla" self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe @@ -331,7 +382,7 @@ class Glm5NextDecoderLayer(nn.Module): quant_config=None, # MLA projections are BF16 in checkpoint prefix=f"{prefix}.self_attn", topk_indices_buffer=topk_indices_buffer, - skip_rope=config.mla_nope, + skip_rope=config.mla_use_nope, ) # MTP layers sit past the base model's hidden layers (layer_idx >= @@ -375,13 +426,13 @@ class Glm5NextDecoderLayer(nn.Module): if self.mhc and not is_mtp_layer: # mhc config - self.mhc_num_residual_streams = config.mhc_num_residual_streams - self.mhc_tau = config.mhc_tau + self.mhc_num_residual_streams = config.hc_mult + self.mhc_tau = _MHC_TAU self.hc_eps = config.hc_eps - self.mhc_sinkhorn_iterations = config.mhc_sinkhorn_iterations - self.mhc_post_mult_value = config.mhc_post_mult_value + self.mhc_sinkhorn_iterations = config.hc_sinkhorn_iters + self.mhc_post_mult_value = _MHC_POST_MULT_VALUE - n = config.mhc_num_residual_streams + n = config.hc_mult d_model = n * self.hidden_size mix_hc = (2 + n) * n @@ -625,7 +676,8 @@ class Glm5NextModel(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_text_config + _validate_supported_config(config) self.config = config self.vocab_size = config.vocab_size @@ -768,7 +820,7 @@ class Glm5NextModel(nn.Module): (".in_proj_qkvbfg_a", ".f_a_proj", 4), (".in_proj_qkvbfg_a", ".g_a_proj", 5), ] - if self.config.is_moe: + if _is_moe(self.config): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) # EPLB: the mapping enumerates physical experts, so it must cover @@ -798,7 +850,7 @@ class Glm5NextModel(nn.Module): # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from # ``kv_a_proj_with_mqa``; pad them with zeros for the model shape. kv_a_pad_size = 0 - if self.config.mla_nope and self.config.qk_rope_head_dim > 0: + if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0: kv_a_pad_size = self.config.qk_rope_head_dim _pending_wk_fp8: dict = {} @@ -912,7 +964,7 @@ class Glm5NextModel(nn.Module): if ( name.endswith(".bias") and name not in params_dict - and not self.config.is_linear_attn + and not _is_linear_attn(self.config) ): # noqa: E501 continue # Remapping the name of FP8 kv-scale. @@ -939,7 +991,7 @@ class Glm5NextForCausalLM( super().__init__() self.model_config = vllm_config.model_config self.vllm_config = vllm_config - self.config = self.model_config.hf_config + self.config = self.model_config.hf_text_config quant_config = vllm_config.quant_config self.quant_config = quant_config self.model = Glm5NextModel( @@ -955,7 +1007,7 @@ class Glm5NextForCausalLM( else: self.lm_head = PPMissingLayer() self.logits_processor = LogitsProcessor( - self.config.vocab_size, scale=self.config.logit_scale + self.config.vocab_size, scale=_LOGIT_SCALE ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: @@ -988,7 +1040,7 @@ class Glm5NextForCausalLM( cls, vllm_config: "VllmConfig" ) -> tuple[tuple[int, int], tuple[int, int, int]]: parallel_config = vllm_config.parallel_config - hf_config = vllm_config.model_config.hf_config + hf_config = vllm_config.model_config.hf_text_config tp_size = parallel_config.tensor_parallel_size num_spec = ( vllm_config.speculative_config.num_speculative_tokens @@ -1092,11 +1144,7 @@ class Glm5NextForConditionalGeneration( self.visual = Glm5NextVisionTransformer( config.text_config, config.vision_config, - # Read eps from the VISION sub-config, not the top-level - # `config.rms_norm_eps`: Glm5NextConfig.__getattribute__ mirrors - # the latter onto text_config (1e-5), silently ignoring the - # vision tower's own (1e-6) rms_norm_eps. - norm_eps=config.vision_config.rms_norm_eps, + norm_eps=_VISION_RMS_NORM_EPS, # Vision tower ships BF16 weights in this fp8 checkpoint (no # weight_scale_inv for visual.*), so it must NOT inherit the # global fp8 quant_config -- doing so incorrectly quantizes @@ -1166,7 +1214,7 @@ class Glm5NextForConditionalGeneration( def get_spec_layer_idx_from_weight_name( - config: Glm5NextConfig, weight_name: str + config: Glm5NextTextConfig, weight_name: str ) -> int | None: if hasattr(config, "num_nextn_predict_layers") and ( config.num_nextn_predict_layers > 0 diff --git a/vllm/models/glm5next/common/mtp.py b/vllm/models/glm5next/common/mtp.py index 8361a053b8..816cc46fdc 100644 --- a/vllm/models/glm5next/common/mtp.py +++ b/vllm/models/glm5next/common/mtp.py @@ -40,7 +40,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): def __init__(self, vllm_config: VllmConfig, prefix: str) -> None: super().__init__() assert vllm_config.speculative_config is not None - config = vllm_config.speculative_config.draft_model_config.hf_config + config = vllm_config.speculative_config.draft_model_config.hf_text_config self.config = config self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -114,7 +114,7 @@ class Glm5NextMultiTokenPredictorLayer(nn.Module): class Glm5NextMultiTokenPredictor(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_text_config self.mtp_start_layer_idx = config.num_hidden_layers self.num_mtp_layers = config.num_nextn_predict_layers self.layers = torch.nn.ModuleDict( @@ -211,7 +211,7 @@ class Glm5NextMultiTokenPredictor(nn.Module): class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - self.config = vllm_config.model_config.hf_config + self.config = vllm_config.model_config.hf_text_config self.quant_config = vllm_config.quant_config self.model = Glm5NextMultiTokenPredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") @@ -313,7 +313,7 @@ class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): # GLM-5.3-Flash NoPE checkpoints omit the RoPE rows from # ``kv_a_proj_with_mqa``; the FP8-to-BF16 path pads them for the model. kv_a_pad_size = 0 - if self.config.mla_nope and self.config.qk_rope_head_dim > 0: + if self.config.mla_use_nope and self.config.qk_rope_head_dim > 0: kv_a_pad_size = self.config.qk_rope_head_dim for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: diff --git a/vllm/models/glm5next/common/multimodal.py b/vllm/models/glm5next/common/multimodal.py index 60b5b47a60..75921b697f 100644 --- a/vllm/models/glm5next/common/multimodal.py +++ b/vllm/models/glm5next/common/multimodal.py @@ -3,13 +3,14 @@ """GLM-5.3-Flash vision tower and multimodal processor.""" from collections.abc import Mapping -from functools import cached_property, partial +from functools import partial from typing import Any import numpy as np import torch import torch.nn as nn from einops import rearrange +from transformers.video_utils import VideoMetadata from vllm.distributed import ( get_tensor_model_parallel_world_size, @@ -44,7 +45,6 @@ from vllm.model_executor.models.vision import ( ) from vllm.models.common.ops import fused_q_kv_rmsnorm from vllm.multimodal.parse import ImageSize, MultiModalDataItems -from vllm.transformers_utils.processors.glm5next import glm_sample_frame_indices from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -616,37 +616,48 @@ class Glm5NextVisionTransformer(nn.Module): return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) -class Glm5NextProcessingInfo(Glm4vProcessingInfo): - """Wires up the vLLM-native processor for the multimodal checkpoint. +_MAX_VIDEO_TOKENS = 30000 - The checkpoint's ``processor_config.json`` declares a custom ``processor_class`` - and stores its image/video processor configs inline (no standalone - ``preprocessor_config.json``), so ``AutoProcessor`` cannot resolve the - config. We bypass it and build our own ``Glm5NextProcessor`` - (``vllm/transformers_utils/processors/glm5next.py``), a port of the - training-side pipeline that no longer imports transformers' GLM processor - classes. The port applies ``patch_expand_factor`` (checkpoint ships 1) - inside ``smart_resize``'s spatial factor. + +class Glm5NextProcessingInfo(Glm4vProcessingInfo): + """Token-budget geometry for the multimodal checkpoint. + + This checkpoint's ``processor_config.json`` ships the token-budget style + (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, so the + inherited Glm4v ``size.longest_edge`` path does not apply. """ - @cached_property - def _glm5_hf_processor(self): - from vllm.transformers_utils.processors.glm5next import Glm5NextProcessor - - return Glm5NextProcessor.from_pretrained(self.ctx.model_config.model) - def get_hf_processor(self, **kwargs: object): - return self._glm5_hf_processor + processor = super().get_hf_processor(**kwargs) + # Cap video inputs to keep encoder profiling from starving the KV cache. + video_processor = processor.video_processor + video_processor.max_image_tokens = min( + video_processor.max_image_tokens, _MAX_VIDEO_TOKENS + ) + return processor + + @staticmethod + def _alignment_factor(proc) -> int: + """Spatial factor canvases align to, as ``smart_resize`` takes it.""" + return proc.patch_size * proc.merge_size * proc.patch_expand_factor + + @classmethod + def _pixels_per_token(cls, proc) -> int: + """Pixels one vision token covers, matching ``smart_resize``'s own + ``temporal_factor * factor ** 2``.""" + return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2 def _processor_pixel_budget(self, proc) -> tuple[int, int]: - from vllm.transformers_utils.processors.glm5next import _pixel_budget - - return _pixel_budget( - proc.min_image_tokens, - proc.max_image_tokens, - proc.patch_size, - proc.merge_size, - proc.temporal_patch_size, + """(min_pixels, max_pixels) from the processor's token bounds.""" + if proc.min_image_tokens is None or proc.max_image_tokens is None: + raise ValueError( + "min_image_tokens and max_image_tokens must be provided by " + "processor_config.json (or per-call kwargs)." + ) + pixels_per_token = self._pixels_per_token(proc) + return ( + proc.min_image_tokens * pixels_per_token, + proc.max_image_tokens * pixels_per_token, ) def _get_image_max_pixels(self) -> int: @@ -687,19 +698,20 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): else: frame_indices = [ int(idx) - for idx in glm_sample_frame_indices( - int(metadata.get("total_num_frames", total_frames)), - video_fps, - float(metadata.get("duration") or 0), - target_fps=video_processor.fps_interval, - max_frame_count=video_processor.max_frame_count_dynamic, - temporal_patch_size=temporal_patch_size, + for idx in video_processor.sample_frames( + VideoMetadata( + total_num_frames=int( + metadata.get("total_num_frames", total_frames) + ), + fps=video_fps, + duration=metadata.get("duration"), + ) ) ] num_frames = len(frame_indices) if not frame_indices or num_frames <= 0: - # ``duration * fps_interval < 1`` selects no frames at all; the + # ``duration * fps < 1`` selects no frames at all; the # pixel path raises an opaque IndexError on the same input a moment # later, so say what happened while there is still a request. raise ValueError( @@ -731,12 +743,13 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): The inherited Glm4v path resolves the pixel budget from ``size.longest_edge`` and resizes with GLM-4V's ``smart_resize``. This - checkpoint's ``processor_config.json`` ships the token-budget style - (``min_image_tokens`` / ``max_image_tokens``) with no ``size`` key, and - the alignment factor carries ``patch_expand_factor`` — resolve both - from the vLLM-native processor so profiling matches runtime geometry. + checkpoint ships token bounds instead, and the alignment factor carries + ``patch_expand_factor`` — resolve both from the processor so profiling + matches runtime geometry. """ - from vllm.transformers_utils.processors.glm5next import smart_resize + from transformers.models.glm5_next.image_processing_glm5_next import ( + smart_resize, + ) vision_config = self.get_hf_config().vision_config patch_size = vision_config.patch_size @@ -744,22 +757,25 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): temporal_patch_size = vision_config.temporal_patch_size image_processor = self.get_hf_processor().image_processor - factor = patch_size * merge_size * image_processor.patch_expand_factor - # Keep the profiling search viable when the caller's budget is below - # one aligned canvas of the requested duration. - max_image_pixels = max(max_image_pixels, temporal_patch_size * factor * factor) + factor = self._alignment_factor(image_processor) + temporal_factor = image_processor.temporal_patch_size + # `smart_resize` denominates its bounds in vision tokens. Round down, + # but keep the profiling search viable when the caller's budget is + # below one aligned canvas of the requested duration. + max_image_tokens = max( + max_image_pixels // self._pixels_per_token(image_processor), 1 + ) if do_resize: - t = num_frames if num_frames > temporal_patch_size else temporal_patch_size + t = num_frames if num_frames > temporal_factor else temporal_factor resized_height, resized_width = smart_resize( - t=t, - h=image_height, - w=image_width, - t_factor=temporal_patch_size, - h_factor=factor, - w_factor=factor, + num_frames=t, + height=image_height, + width=image_width, + temporal_factor=temporal_factor, + factor=factor, min_pixels=1, - max_pixels=max_image_pixels, + max_pixels=max_image_tokens, ) preprocessed_size = ImageSize(width=resized_width, height=resized_height) else: @@ -777,7 +793,7 @@ class Glm5NextProcessingInfo(Glm4vProcessingInfo): class Glm5NextMultiModalProcessor(Glm4vMultiModalProcessor): - """The vLLM-native ``Glm5NextProcessor`` extracts image/video features + """``Glm5NextProcessor`` extracts image/video features only and passes the prompt text through unchanged, so prompt expansion (image token repeat, video frame/timestamp structure) is owned by vLLM's prompt-update machinery — the inherited ``_get_prompt_updates`` builds diff --git a/vllm/models/qwen4_exp/amd/indexer_qsa.py b/vllm/models/qwen4_exp/amd/indexer_qsa.py index 2b8c7b1444..4345763fd4 100644 --- a/vllm/models/qwen4_exp/amd/indexer_qsa.py +++ b/vllm/models/qwen4_exp/amd/indexer_qsa.py @@ -8,6 +8,7 @@ from typing import cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.forward_context import get_forward_context @@ -15,9 +16,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from ..common.qsa_cache import ( QSACompressedKeyCache, diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index 2cdc3c26fa..16509d609c 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -7,6 +7,7 @@ from itertools import islice import torch from torch import nn +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig @@ -47,7 +48,6 @@ from vllm.model_executor.models.qwen3_5 import ( ) from vllm.model_executor.models.qwen3_next import ( Qwen3NextAttention, - Qwen3NextMLP, Qwen3NextSparseMoeBlock, ) from vllm.model_executor.models.qwen3_vl import ( @@ -71,13 +71,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_interface import MambaSpec -from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .ple_layer import Qwen4ExpPLELayer @@ -179,8 +175,6 @@ class Qwen4ExpDecoderLayer(nn.Module): ) -> None: super().__init__() config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - model_config = vllm_config.model_config - cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config self.config = config @@ -213,16 +207,13 @@ class Qwen4ExpDecoderLayer(nn.Module): prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, ) - elif layer_type in ATTENTION_LAYER_TYPES: - use_qsa = ( - layer_type == QSA_LAYER_TYPE - or getattr(config, "indexer_n_heads", None) is not None - ) + elif layer_type == "qwen_sparse_attention": + use_qsa = getattr(config, "indexer_n_heads", None) is not None if not use_qsa: self.self_attn = Qwen3NextAttention( config, - model_config=model_config, - cache_config=cache_config, + model_config=vllm_config.model_config, + cache_config=vllm_config.cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) @@ -237,24 +228,9 @@ class Qwen4ExpDecoderLayer(nn.Module): else: raise ValueError(f"Invalid layer_type {layer_type}") - mlp_only_layers = getattr(config, "mlp_only_layers", []) - num_experts = getattr(config, "num_experts", 0) or 0 - absolute_layer_id = self.layer_idx + 1 - is_moe_layer = self.layer_idx not in mlp_only_layers and ( - num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0 + self.mlp = Qwen4ExpSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.mlp" ) - if is_moe_layer: - self.mlp = Qwen4ExpSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.mlp" - ) - else: - self.mlp = Qwen3NextMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) hc_config = HyperConnectionConfig( hc_count=config.hc_count, @@ -313,7 +289,7 @@ class Qwen4ExpDecoderLayer(nn.Module): if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type in ATTENTION_LAYER_TYPES: + elif self.layer_type == "qwen_sparse_attention": attn_out = self.self_attn( hidden_states=block_input, positions=positions, @@ -404,11 +380,8 @@ class Qwen4ExpModel(nn.Module): self._qsa_layer_ids = frozenset( layer_idx for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type == QSA_LAYER_TYPE - or ( - layer_type == "full_attention" - and getattr(config, "indexer_n_heads", None) is not None - ) + if layer_type == "qwen_sparse_attention" + and getattr(config, "indexer_n_heads", None) is not None ) self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) diff --git a/vllm/models/qwen4_exp/amd/mtp.py b/vllm/models/qwen4_exp/amd/mtp.py index f4302aa757..d06f6436e7 100644 --- a/vllm/models/qwen4_exp/amd/mtp.py +++ b/vllm/models/qwen4_exp/amd/mtp.py @@ -18,6 +18,7 @@ from collections.abc import Iterable import regex as re import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig, replace, set_current_vllm_config @@ -45,9 +46,6 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm @@ -222,7 +220,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): self.layers = nn.ModuleList( Qwen4ExpDecoderLayer( draft_vllm_config, - layer_type="full_attention", + layer_type="qwen_sparse_attention", prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", ) for idx in range(self.num_mtp_layers) diff --git a/vllm/models/qwen4_exp/amd/ple_layer.py b/vllm/models/qwen4_exp/amd/ple_layer.py index 11291ff385..bd0607b613 100644 --- a/vllm/models/qwen4_exp/amd/ple_layer.py +++ b/vllm/models/qwen4_exp/amd/ple_layer.py @@ -8,6 +8,7 @@ from collections.abc import Iterable, Sequence import torch import torch.nn.functional as F from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.forward_context import get_forward_context @@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, ) from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.utils.torch_utils import direct_register_custom_op from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ( diff --git a/vllm/models/qwen4_exp/amd/qsa.py b/vllm/models/qwen4_exp/amd/qsa.py index 3c8a61bd54..325e0e3d91 100644 --- a/vllm/models/qwen4_exp/amd/qsa.py +++ b/vllm/models/qwen4_exp/amd/qsa.py @@ -8,6 +8,7 @@ from typing import ClassVar, cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.config.cache import CacheDType @@ -23,9 +24,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.models.qwen3_next import Qwen3NextAttention from vllm.platforms import current_platform -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.utils.torch_utils import ( LayerNameType, _encode_layer_name, diff --git a/vllm/models/qwen4_exp/config.py b/vllm/models/qwen4_exp/config.py deleted file mode 100644 index dfac816d5e..0000000000 --- a/vllm/models/qwen4_exp/config.py +++ /dev/null @@ -1,263 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Qwen4Exp model configuration.""" - -from typing import Any, ClassVar, cast - -from transformers import PreTrainedConfig -from transformers.models.qwen3_vl.configuration_qwen3_vl import ( - Qwen3VLVisionConfig, -) - -from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig - -_QSA_CONFIG_FIELDS = ( - "indexer_n_heads", - "indexer_kv_heads", - "indexer_head_dim", - "indexer_budget", - "indexer_compress_ratio", -) - -# Transformers labels Qwen4Exp's sparse-attention layers `qwen_sparse_attention`. -# Older checkpoints label them `full_attention` and mark QSA by setting the -# indexer fields for the whole model. -QSA_LAYER_TYPE = "qwen_sparse_attention" -ATTENTION_LAYER_TYPES = ("full_attention", QSA_LAYER_TYPE) - - -class Qwen4ExpVisionConfig(Qwen3VLVisionConfig): - model_type = "qwen4_exp" - base_config_key = "vision_config" - - -class Qwen4ExpTextConfig(Qwen3NextConfig): - model_type = "qwen4_exp_text" - base_config_key = "text_config" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - hc_count: int = 4, - hc_lowrank: int = 320, - ple_layer_ids: list[int] | None = None, - ple_embed_dim: int | None = None, - ple_conv_kernel_size: int = 4, - ngram_size: int = 3, - heads_per_ngram: int = 8, - ngram_vocab_size_base: int = 20_000_000, - make_ngram_vocab_size_divisible_by: int = 128, - output_gate_type: str = "sigmoid", - rope_parameters: dict[str, Any] | None = None, - layer_types: list[str] | None = None, - **kwargs: Any, - ) -> None: - if hc_count <= 1: - raise ValueError(f"Qwen4Exp requires hc_count > 1, got {hc_count}.") - - if rope_parameters is not None: - if kwargs.get("rope_scaling") is None: - kwargs["rope_scaling"] = rope_parameters - if kwargs.get("rope_theta") is None and "rope_theta" in rope_parameters: - kwargs["rope_theta"] = rope_parameters["rope_theta"] - if ( - kwargs.get("partial_rotary_factor") is None - and "partial_rotary_factor" in rope_parameters - ): - kwargs["partial_rotary_factor"] = rope_parameters[ - "partial_rotary_factor" - ] - - rope_scaling = kwargs.get("rope_scaling") - rope_theta = kwargs.get("rope_theta", 10_000.0) - super().__init__(layer_types=layer_types, **kwargs) - - normalized_rope_parameters = self.rope_parameters - self.rope_scaling = ( - rope_scaling or rope_parameters or normalized_rope_parameters - ) - self.rope_parameters = rope_parameters or normalized_rope_parameters - self.rope_theta = rope_theta - - self.hc_count = hc_count - self.hc_lowrank = hc_lowrank - self.ple_layer_ids = ple_layer_ids or [] - self.ple_embed_dim = ( - self.hidden_size if ple_embed_dim is None else ple_embed_dim - ) - self.ple_conv_kernel_size = ple_conv_kernel_size - self.ngram_size = ngram_size - self.heads_per_ngram = heads_per_ngram - self.ngram_vocab_size_base = ngram_vocab_size_base - self.make_ngram_vocab_size_divisible_by = make_ngram_vocab_size_divisible_by - self.output_gate_type = output_gate_type - - self._validate_ple_config() - self._validate_ple_layer_ids() - self._validate_qsa_config() - - def _validate_ple_config(self) -> None: - if self.hc_lowrank <= 0: - raise ValueError(f"hc_lowrank must be positive, got {self.hc_lowrank}") - if self.ngram_size < 2: - raise ValueError(f"ngram_size must be >= 2, got {self.ngram_size}") - if self.heads_per_ngram <= 0: - raise ValueError( - f"heads_per_ngram must be positive, got {self.heads_per_ngram}" - ) - if self.ple_embed_dim <= 0: - raise ValueError( - f"ple_embed_dim must be positive, got {self.ple_embed_dim}" - ) - ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram - if self.ple_embed_dim % ngram_heads: - raise ValueError( - "ple_embed_dim must be divisible by total ngram heads: " - f"{self.ple_embed_dim} % {ngram_heads} != 0" - ) - if self.ple_conv_kernel_size <= 0: - raise ValueError( - "ple_conv_kernel_size must be positive, got " - f"{self.ple_conv_kernel_size}" - ) - if self.ngram_vocab_size_base <= 0: - raise ValueError("ngram_vocab_size_base must be positive") - if self.make_ngram_vocab_size_divisible_by <= 0: - raise ValueError("make_ngram_vocab_size_divisible_by must be positive") - - def _validate_ple_layer_ids(self) -> None: - invalid = [ - layer_id - for layer_id in self.ple_layer_ids - if not 1 <= int(layer_id) <= self.num_hidden_layers - ] - if invalid: - raise ValueError( - "ple_layer_ids are 1-based and must refer to an existing layer; " - f"got {invalid} for {self.num_hidden_layers} layers" - ) - - def _validate_qsa_config(self) -> None: - configured = {name: getattr(self, name, None) for name in _QSA_CONFIG_FIELDS} - if all(value is None for value in configured.values()): - if QSA_LAYER_TYPE in (self.layer_types or ()): - raise ValueError( - f"{QSA_LAYER_TYPE} layers require the QSA config fields: " - f"{list(_QSA_CONFIG_FIELDS)}" - ) - return - - missing = [name for name, value in configured.items() if value is None] - if missing: - raise ValueError(f"QSA config is missing required fields: {missing}") - - values = {name: int(cast(int, value)) for name, value in configured.items()} - if any(value <= 0 for value in values.values()): - raise ValueError(f"QSA config values must be positive: {values}") - if values["indexer_kv_heads"] != 1: - raise ValueError("the QSA MQA operators require indexer_kv_heads=1") - if values["indexer_budget"] % values["indexer_compress_ratio"] != 0: - raise ValueError( - "indexer_budget must be divisible by indexer_compress_ratio" - ) - block_topk = values["indexer_budget"] // values["indexer_compress_ratio"] - if block_topk not in (512, 2048): - raise ValueError( - "QSA requires indexer_budget / indexer_compress_ratio " - f"to be 512 or 2048, got {block_topk}" - ) - rotary_dim = int(self.head_dim * self.partial_rotary_factor) - if rotary_dim > values["indexer_head_dim"]: - raise ValueError( - "QSA indexer_head_dim must cover the attention rotary " - f"dimension, got {values['indexer_head_dim']} < {rotary_dim}" - ) - - @property - def layers_block_type(self) -> list[str]: - return [ - "attention" if layer_type in ATTENTION_LAYER_TYPES else layer_type - for layer_type in self.layer_types - ] - - @property - def short_conv_layer_ids(self) -> list[int]: - if not self.ple_layer_ids: - return [] - return sorted({int(layer_id) - 1 for layer_id in self.ple_layer_ids}) - - @property - def short_conv_state_shape(self) -> tuple[int, int] | None: - if not self.short_conv_layer_ids: - return None - ple_state_len = (self.ple_conv_kernel_size - 1) * self.ngram_size - ple_channels = self.hidden_size * self.hc_count - return ple_channels, ple_state_len - - @property - def ngram_context_len(self) -> int: - if not self.ple_layer_ids: - return 0 - return max(int(self.ngram_size) - 1, 0) - - -class Qwen4ExpConfig(PreTrainedConfig): - model_type = "qwen4_exp" - sub_configs: ClassVar[dict[str, type[PreTrainedConfig]]] = { - "vision_config": Qwen4ExpVisionConfig, - "text_config": Qwen4ExpTextConfig, - } - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config: Qwen4ExpTextConfig | dict[str, Any] | None = None, - vision_config: Qwen4ExpVisionConfig | dict[str, Any] | None = None, - image_token_id: int = 248056, - video_token_id: int = 248057, - vision_start_token_id: int = 248053, - vision_end_token_id: int = 248054, - tie_word_embeddings: bool = False, - rope_parameters: dict[str, Any] | None = None, - **kwargs: Any, - ) -> None: - if text_config is not None: - kwargs.pop("split_ngram_parts", None) - - text_kwargs = ( - dict(kwargs) - if text_config is None - and "hidden_size" in kwargs - and "num_hidden_layers" in kwargs - else {} - ) - - if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) - elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() - else: - self.vision_config = vision_config - - if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - self.text_config = self.sub_configs["text_config"](**text_kwargs) - else: - self.text_config = text_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.vision_start_token_id = vision_start_token_id - self.vision_end_token_id = vision_end_token_id - self.rope_parameters = rope_parameters or getattr( - self.text_config, "rope_parameters", {} - ) - super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings) - - -__all__ = [ - "Qwen4ExpConfig", - "Qwen4ExpTextConfig", - "Qwen4ExpVisionConfig", -] diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index 14137de695..c8ffa0dbfb 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.forward_context import get_forward_context @@ -13,9 +14,6 @@ from vllm.model_executor.layers.layernorm import GemmaRMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.mrope import triton_mrope -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from ..common.qsa_cache import ( QSACompressedKeyCache, diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index 20521f1800..95e920bc21 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -7,6 +7,7 @@ from itertools import islice import torch from torch import nn +from transformers import Qwen4ExpConfig, Qwen4ExpTextConfig from vllm.config import VllmConfig from vllm.distributed import get_pp_group @@ -46,7 +47,6 @@ from vllm.model_executor.models.qwen3_5 import ( ) from vllm.model_executor.models.qwen3_next import ( Qwen3NextAttention, - Qwen3NextMLP, Qwen3NextSparseMoeBlock, ) from vllm.model_executor.models.qwen3_vl import ( @@ -70,13 +70,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.kv_cache_interface import MambaSpec -from ..config import ATTENTION_LAYER_TYPES, QSA_LAYER_TYPE, Qwen4ExpConfig from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm from .ops.cute_dsl.hc_down_silu import request_hc_down_silu_warmup @@ -181,8 +177,6 @@ class Qwen4ExpDecoderLayer(nn.Module): ) -> None: super().__init__() config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config - model_config = vllm_config.model_config - cache_config = vllm_config.cache_config quant_config = vllm_config.quant_config self.config = config @@ -215,16 +209,13 @@ class Qwen4ExpDecoderLayer(nn.Module): prefix=f"{prefix}.linear_attn", gqa_interleaved_layout=False, ) - elif layer_type in ATTENTION_LAYER_TYPES: - use_qsa = ( - layer_type == QSA_LAYER_TYPE - or getattr(config, "indexer_n_heads", None) is not None - ) + elif layer_type == "qwen_sparse_attention": + use_qsa = getattr(config, "indexer_n_heads", None) is not None if not use_qsa: self.self_attn = Qwen3NextAttention( config, - model_config=model_config, - cache_config=cache_config, + model_config=vllm_config.model_config, + cache_config=vllm_config.cache_config, quant_config=quant_config, prefix=f"{prefix}.self_attn", ) @@ -239,24 +230,9 @@ class Qwen4ExpDecoderLayer(nn.Module): else: raise ValueError(f"Invalid layer_type {layer_type}") - mlp_only_layers = getattr(config, "mlp_only_layers", []) - num_experts = getattr(config, "num_experts", 0) or 0 - absolute_layer_id = self.layer_idx + 1 - is_moe_layer = self.layer_idx not in mlp_only_layers and ( - num_experts > 0 and absolute_layer_id % config.decoder_sparse_step == 0 + self.mlp = Qwen4ExpSparseMoeBlock( + vllm_config=vllm_config, prefix=f"{prefix}.mlp" ) - if is_moe_layer: - self.mlp = Qwen4ExpSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.mlp" - ) - else: - self.mlp = Qwen3NextMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) hc_config = HyperConnectionConfig( hc_count=config.hc_count, @@ -317,7 +293,7 @@ class Qwen4ExpDecoderLayer(nn.Module): if self.layer_type == "linear_attention": attn_out = self.linear_attn(hidden_states=block_input) - elif self.layer_type in ATTENTION_LAYER_TYPES: + elif self.layer_type == "qwen_sparse_attention": attn_out = self.self_attn( hidden_states=block_input, positions=positions, @@ -397,11 +373,8 @@ class Qwen4ExpModel(nn.Module): self._qsa_layer_ids = frozenset( layer_idx for layer_idx, layer_type in enumerate(config.layer_types) - if layer_type == QSA_LAYER_TYPE - or ( - layer_type == "full_attention" - and getattr(config, "indexer_n_heads", None) is not None - ) + if layer_type == "qwen_sparse_attention" + and getattr(config, "indexer_n_heads", None) is not None ) self.embed_tokens = VocabParallelEmbedding(self.vocab_size, config.hidden_size) diff --git a/vllm/models/qwen4_exp/nvidia/mtp.py b/vllm/models/qwen4_exp/nvidia/mtp.py index d9f26fc022..61290617ae 100644 --- a/vllm/models/qwen4_exp/nvidia/mtp.py +++ b/vllm/models/qwen4_exp/nvidia/mtp.py @@ -18,6 +18,7 @@ from collections.abc import Iterable import regex as re import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import VllmConfig, replace, set_current_vllm_config from vllm.distributed import get_pp_group @@ -44,9 +45,6 @@ from vllm.model_executor.models.utils import ( maybe_prefix, ) from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from .hyperconnection import GatedResidual, HyperConnectionConfig from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm @@ -212,7 +210,7 @@ class Qwen4ExpMultiTokenPredictor(nn.Module): self.layers = nn.ModuleList( Qwen4ExpDecoderLayer( draft_vllm_config, - layer_type="full_attention", + layer_type="qwen_sparse_attention", prefix=f"{prefix}.layers.{self.mtp_start_layer_idx + idx}", ) for idx in range(self.num_mtp_layers) diff --git a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py index 0b6fd650b1..3be6ea6013 100644 --- a/vllm/models/qwen4_exp/nvidia/ngram_embedding.py +++ b/vllm/models/qwen4_exp/nvidia/ngram_embedding.py @@ -6,6 +6,7 @@ from collections.abc import Iterable import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.config import get_current_vllm_config from vllm.logger import init_logger @@ -13,9 +14,6 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, ) from vllm.model_executor.models.utils import AutoWeightsLoader -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from ..common.ngram_embedding import ( Qwen4ExpPLEDeviceEmbedding, diff --git a/vllm/models/qwen4_exp/nvidia/ple_layer.py b/vllm/models/qwen4_exp/nvidia/ple_layer.py index 2aac4f0f58..ee8a2b47dd 100644 --- a/vllm/models/qwen4_exp/nvidia/ple_layer.py +++ b/vllm/models/qwen4_exp/nvidia/ple_layer.py @@ -6,6 +6,7 @@ from collections.abc import Sequence import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config @@ -17,9 +18,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, is_conv_state_dim_first, ) -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum from vllm.v1.attention.backends.short_conv_attn import ( PleShortConvAttentionBackend, diff --git a/vllm/models/qwen4_exp/nvidia/qsa.py b/vllm/models/qwen4_exp/nvidia/qsa.py index 62aa6459fb..d00cd186b0 100644 --- a/vllm/models/qwen4_exp/nvidia/qsa.py +++ b/vllm/models/qwen4_exp/nvidia/qsa.py @@ -8,6 +8,7 @@ from typing import ClassVar, cast import torch from torch import nn +from transformers import Qwen4ExpTextConfig from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig @@ -25,9 +26,6 @@ from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding, get_ro from vllm.model_executor.models.qwen3_next import Qwen3NextAttention from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability -from vllm.transformers_utils.configs.qwen4_exp import ( - Qwen4ExpTextConfig, -) from vllm.utils.torch_utils import ( kv_cache_dtype_str_to_dtype, ) diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index ff74183a55..44a8e9452d 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -714,15 +714,92 @@ class GLM46VVideoBackend(VideoBackend): class Glm5NextVideoBackend(VideoBackend): """GLM-5.3-Flash fps-interval video backend. - Selects frames with the same ``glm_sample_frame_indices`` sampler the - processor falls back to, so only the sampled frames are - materialized. ``fps_interval`` semantics (default 2.0) with a - temporal-patch-scaled greedy walk, frame count capped at 2048, temporal - pairs kept even. Request overrides: ``fps`` -> fps interval, + Selects frames with the same sampler the processor falls back to, so only + the sampled frames are materialized. ``fps_interval`` semantics (default + 2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048, + temporal pairs kept even. Request overrides: ``fps`` -> fps interval, ``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2). """ _SEEK_GAP_THRESHOLD: ClassVar[int] = 64 + _DEFAULT_FPS: ClassVar[float] = 2.0 + _DEFAULT_MAX_FRAMES: ClassVar[int] = 2048 + + @classmethod + def _sample_frame_indices( + cls, + total_frames: int, + fps: float, + duration: float, + *, + target_fps: float | None = None, + max_frame_count: int | None = None, + temporal_patch_size: int = 2, + ) -> list[int]: + """GLM video frame sampling (training-reference parity). + + ``target_fps`` is the ``fps_interval`` request knob. The greedy walk + advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on + frame-dense sources it collects more candidates than ``extract_t`` and + the ``> extract_t`` fixup re-spreads the picks uniformly with + ``np.linspace`` -- that fallback is the intended reference behavior, not + an accident. Short clips (fewer frames than ``extract_t``) are spread at + evenly spaced timestamps (``floor`` sampling; the linspace variant + samples frames unevenly and cost 4 points on video grounding evals). + Request overrides: ``target_fps`` -> fps interval, ``max_frame_count`` + -> frame cap. + """ + max_frame_idx = total_frames - 1 + if not duration: + duration = (round(max_frame_idx / fps) + 1) if fps else 0 + if max_frame_count is None: + max_frame_count = cls._DEFAULT_MAX_FRAMES + if target_fps is None: + target_fps = cls._DEFAULT_FPS + + extract_t = int(duration * target_fps) + extract_t = min(extract_t, int(max_frame_count)) + + duration_per_frame = 1 / fps + timestamps = [i * duration_per_frame for i in range(total_frames)] + max_second = int(duration) + + if total_frames < extract_t: + frame_indices = [ + math.floor(_i * total_frames / extract_t) for _i in range(extract_t) + ] + else: + frame_indices = [] + current_second = 0.0 + inv_fps = 1 / (temporal_patch_size * target_fps) + for frame_index in range(total_frames): + if timestamps[frame_index] >= current_second: + current_second += inv_fps + frame_indices.append(frame_index) + if current_second >= max_second: + break + + if len(frame_indices) < extract_t: + if len(frame_indices) == 0: + start, end = 0, max(total_frames - 1, 0) + else: + start, end = frame_indices[0], frame_indices[-1] + frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() + elif len(frame_indices) > extract_t: + frame_indices = np.linspace( + 0, total_frames - 1, extract_t, dtype=int + ).tolist() + + seen, uniq = set(), [] + for idx in frame_indices: + if idx not in seen: + seen.add(idx) + uniq.append(int(idx)) + + if len(uniq) & 1: + uniq.append(uniq[-1]) + + return uniq @classmethod def compute_frames_index_to_sample( @@ -731,13 +808,7 @@ class Glm5NextVideoBackend(VideoBackend): target: VideoTargetMetadata, **kwargs, ) -> list[int]: - # Lazy import: the processor module sits behind the - # transformers_utils package init, which multimodal must not pull in. - from vllm.transformers_utils.processors.glm5next import ( - glm_sample_frame_indices, - ) - - return glm_sample_frame_indices( + return cls._sample_frame_indices( source.total_frames_num, source.original_fps, source.duration or 0, diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 432fb1379f..273c504f9d 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -107,9 +107,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( k3_dspark="K3DSparkConfig", funaudiochat="FunAudioChatConfig", granite4_vision="Granite4VisionConfig", - glm5_next="Glm5NextConfig", - glm5_next_text="Glm5NextTextConfig", - glm5_next_vision="Glm5NextVisionConfig", hyperclovax="HyperCLOVAXConfig", hy_v3="HYV3Config", hy_v4="HYV4Config", @@ -145,8 +142,6 @@ _CONFIG_REGISTRY: dict[str, type[PreTrainedConfig]] = LazyConfigDict( qianfan_ocr="QianfanOCRConfig", qwen3_asr="Qwen3ASRConfig", qwen3_next="Qwen3NextConfig", - qwen4_exp="Qwen4ExpConfig", - qwen4_exp_text="Qwen4ExpTextConfig", qwen3_5="Qwen3_5Config", qwen3_5_text="Qwen3_5TextConfig", qwen3_5_moe="Qwen3_5MoeConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index a302d5d729..5bd47fb948 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -43,9 +43,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "FunAudioChatConfig": "vllm.transformers_utils.configs.funaudiochat", "FunAudioChatAudioEncoderConfig": "vllm.transformers_utils.configs.funaudiochat", "Granite4VisionConfig": "vllm.transformers_utils.configs.granite4_vision", - "Glm5NextConfig": "vllm.transformers_utils.configs.glm5_next", - "Glm5NextTextConfig": "vllm.transformers_utils.configs.glm5_next", - "Glm5NextVisionConfig": "vllm.transformers_utils.configs.glm5_next", "HYV3Config": "vllm.transformers_utils.configs.hy_v3", "HYV4Config": "vllm.transformers_utils.configs.hy_v4", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", @@ -95,9 +92,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr", "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr", "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next", - "Qwen4ExpConfig": "vllm.transformers_utils.configs.qwen4_exp", - "Qwen4ExpTextConfig": "vllm.transformers_utils.configs.qwen4_exp", - "Qwen4ExpVisionConfig": "vllm.transformers_utils.configs.qwen4_exp", "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5", "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe", @@ -140,9 +134,6 @@ __all__ = [ "FunAudioChatConfig", "FunAudioChatAudioEncoderConfig", "Granite4VisionConfig", - "Glm5NextConfig", - "Glm5NextTextConfig", - "Glm5NextVisionConfig", "HYV3Config", "HYV4Config", "HyperCLOVAXConfig", @@ -187,9 +178,6 @@ __all__ = [ "QianfanOCRVisionConfig", "Qwen3ASRConfig", "Qwen3NextConfig", - "Qwen4ExpConfig", - "Qwen4ExpTextConfig", - "Qwen4ExpVisionConfig", "Qwen3_5Config", "Qwen3_5TextConfig", "Qwen3_5MoeConfig", diff --git a/vllm/transformers_utils/configs/glm5_next.py b/vllm/transformers_utils/configs/glm5_next.py deleted file mode 100644 index 4a565884a2..0000000000 --- a/vllm/transformers_utils/configs/glm5_next.py +++ /dev/null @@ -1,420 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from transformers.configuration_utils import PreTrainedConfig - -from vllm.logger import init_logger - -logger = init_logger(__name__) - - -class Glm5NextTextConfig(PreTrainedConfig): - model_type = "glm5_next_text" - base_config_key = "text_config" - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - model_type="glm5_next_text", - vocab_size: int = 154880, - hidden_size: int = 4096, - head_dim: int | None = None, - intermediate_size: int = 12288, - num_hidden_layers: int = 45, - num_attention_heads: int = 64, - num_key_value_heads: int | None = None, - hidden_act: str = "silu", - rms_norm_eps: float = 1e-5, - pad_token_id: int | None = 151329, - bos_token_id: int | None = None, - eos_token_id: int | list[int] | None = None, - rope_parameters: dict | None = None, - max_position_embeddings: int = 1048576, - tie_word_embeddings: bool = False, - moe_intermediate_size: int = 2048, - moe_renormalize: bool = True, - scoring_func: str = "sigmoid", - n_routed_experts: int | None = 288, - num_experts_per_token: int = 7, - n_shared_experts: int = 1, - routed_scaling_factor: float = 2.5, - topk_method: str | None = None, - first_k_dense_replace: int = 0, - moe_layer_freq: int = 1, - use_grouped_topk: bool = True, - n_group: int = 1, - topk_group: int = 1, - mla: bool = True, - q_lora_rank: int | None = 1536, - kv_lora_rank: int | None = 512, - qk_nope_head_dim: int = 256, - qk_rope_head_dim: int = 0, - v_head_dim: int | None = 256, - mla_nope: bool | None = True, - num_nextn_predict_layers: int = 1, - # Per-layer layout: "linear_attention" | "deepseek_sparse_attention" - layer_types: list[str] | None = None, - # Per-layer MLP: "dense" | "sparse" - mlp_layer_types: list[str] | None = None, - # Linear-attention (KDA) head config (flattened from the old - # linear_attn_config dict). - linear_head_dim: int = 128, - linear_num_heads: int = 64, - linear_conv_kernel_dim: int = 4, - linear_lower_bound: float = -5.0, - index_head_dim: int | None = None, - index_topk: int | None = None, - index_n_heads: int | None = None, - index_dsa_use_layernorm: bool = True, - index_kpool_compress: bool = True, - # Every ``index_kpool`` indexer K entries pool into one stored entry - # (compress_ratio). The 300B checkpoint ships 4; topk runs at pool - # granularity (select_k = index_topk // index_kpool). - index_kpool: int | None = 4, - index_kpool_always_select_tail: bool = True, - indexer_rope_interleave: bool = False, - mhc: bool | None = True, - mhc_num_residual_streams: int = 4, - hc_eps: float | None = 1e-06, - mhc_tau: float = 0.05, - hres_vwnstyle: bool | None = True, - mhc_no_norm_weight: bool | None = False, - mhc_sinkhorn_iterations: int | None = 20, - mhc_post_mult_value: float | None = 2.0, - swiglu_limit: float | None = None, - logit_scale: float = 1.0, - **kwargs, - ): - # Preserve checkpoint field names and local aliases because their - # consumers use different spellings. - num_experts_per_token = kwargs.get("num_experts_per_tok", num_experts_per_token) - moe_renormalize = kwargs.get("norm_topk_prob", moe_renormalize) - mhc_num_residual_streams = kwargs.get("hc_mult", mhc_num_residual_streams) - mhc_sinkhorn_iterations = kwargs.get( - "hc_sinkhorn_iters", mhc_sinkhorn_iterations - ) - # Checkpoint ships ``mla_use_nope`` (not ``mla_nope``); without this - # alias self.mla_nope silently stays at the param default. - mla_nope = kwargs.get("mla_use_nope", mla_nope) - # Checkpoints ship the KDA head config as the ``linear_attn_config`` - # dict (head_dim / num_heads / short_conv_kernel_size / - # gate_lower_bound) rather than the flattened top-level fields; fold it - # in so the trained values are read instead of the param defaults - # (which only match this checkpoint by coincidence). - linear_cfg = kwargs.get("linear_attn_config") or {} - if linear_cfg: - linear_head_dim = linear_cfg.get("head_dim", linear_head_dim) - linear_num_heads = linear_cfg.get("num_heads", linear_num_heads) - linear_conv_kernel_dim = linear_cfg.get( - "short_conv_kernel_size", linear_conv_kernel_dim - ) - linear_lower_bound = linear_cfg.get("gate_lower_bound", linear_lower_bound) - - if index_topk is not None: - if index_dsa_use_layernorm is not True: - raise NotImplementedError( - "GLM-5.3 sparse indexer requires index_dsa_use_layernorm=True" - ) - if index_kpool_compress is not True: - raise NotImplementedError( - "GLM-5.3 sparse indexer requires index_kpool_compress=True" - ) - if index_kpool_always_select_tail is not True: - raise NotImplementedError( - "GLM-5.3 sparse indexer requires " - "index_kpool_always_select_tail=True" - ) - - if mhc: - if hres_vwnstyle is not True: - raise NotImplementedError("GLM-5.3 mHC requires hres_vwnstyle=True") - if mhc_no_norm_weight not in (False, None): - raise NotImplementedError( - "GLM-5.3 mHC requires mhc_no_norm_weight=False" - ) - - self.model_type = model_type - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.head_dim = ( - head_dim if head_dim is not None else hidden_size // num_attention_heads - ) - self.intermediate_size = intermediate_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - - # for backward compatibility - if num_key_value_heads is None: - num_key_value_heads = num_attention_heads - - self.num_key_value_heads = num_key_value_heads - self.hidden_act = hidden_act - self.rms_norm_eps = rms_norm_eps - self.max_position_embeddings = max_position_embeddings - self.rope_parameters = rope_parameters - - # mla config - self.mla = mla - self.q_lora_rank = q_lora_rank - self.kv_lora_rank = kv_lora_rank - self.qk_nope_head_dim = qk_nope_head_dim - self.qk_rope_head_dim = qk_rope_head_dim - self.v_head_dim = v_head_dim - self.mla_nope = mla_nope - # moe config - self.n_routed_experts = n_routed_experts - self.num_experts_per_token = num_experts_per_token - self.moe_renormalize = moe_renormalize - self.n_shared_experts = n_shared_experts - self.routed_scaling_factor = routed_scaling_factor - self.topk_method = topk_method - self.scoring_func = scoring_func - assert self.scoring_func in ("softmax", "sigmoid") - self.moe_intermediate_size = moe_intermediate_size - self.first_k_dense_replace = first_k_dense_replace - self.moe_layer_freq = moe_layer_freq - self.use_grouped_topk = use_grouped_topk - self.n_group = n_group - self.topk_group = topk_group - self.num_nextn_predict_layers = num_nextn_predict_layers - - # Per-layer attention / MLP layout. Normalize mlp_layer_types from - # first_k_dense_replace when the new-schema field is absent so layer - # construction sees a consistent layout (mirrors cohere2_moe). - self.layer_types = layer_types - if mlp_layer_types is None: - n = self.num_hidden_layers - if first_k_dense_replace is not None: - mlp_layer_types = ["dense"] * first_k_dense_replace + ["sparse"] * ( - n - first_k_dense_replace - ) - else: - mlp_layer_types = ["sparse"] * n - self.mlp_layer_types = mlp_layer_types - - # Linear-attention (KDA) head config. - self.linear_head_dim = linear_head_dim - self.linear_num_heads = linear_num_heads - self.linear_conv_kernel_dim = linear_conv_kernel_dim - self.linear_lower_bound = linear_lower_bound - - # dsa index config - self.index_head_dim = index_head_dim - self.index_topk = index_topk - self.index_n_heads = index_n_heads - self.index_dsa_use_layernorm = index_dsa_use_layernorm - self.index_kpool_compress = index_kpool_compress - self.index_kpool = index_kpool - self.index_kpool_always_select_tail = index_kpool_always_select_tail - self.indexer_rope_interleave = indexer_rope_interleave - - # mhc config - self.mhc = mhc - self.mhc_num_residual_streams = mhc_num_residual_streams - self.mhc_tau = mhc_tau - self.hres_vwnstyle = hres_vwnstyle - self.hc_eps = hc_eps - self.mhc_no_norm_weight = mhc_no_norm_weight - self.mhc_sinkhorn_iterations = mhc_sinkhorn_iterations - self.mhc_post_mult_value = mhc_post_mult_value - - self.swiglu_limit = swiglu_limit - self.logit_scale = logit_scale - - super().__init__( - pad_token_id=pad_token_id, - bos_token_id=bos_token_id, - eos_token_id=eos_token_id, - tie_word_embeddings=tie_word_embeddings, - **kwargs, - ) - - @property - def is_mla(self): - return ( - self.q_lora_rank is not None - or self.kv_lora_rank is not None - or self.qk_nope_head_dim is not None - or self.qk_rope_head_dim is not None - or self.v_head_dim is not None - or self.mla_nope is True - ) - - @property - def is_moe(self): - return self.n_routed_experts is not None - - @property - def is_linear_attn(self) -> bool: - return self.layer_types is not None and any( - t == "linear_attention" for t in self.layer_types - ) - - def is_kda_layer(self, layer_idx: int): - return ( - self.layer_types is not None - and layer_idx < len(self.layer_types) - and self.layer_types[layer_idx] == "linear_attention" - ) - - @property - def layers_block_type(self): - # Map the schema's per-layer types onto the block strings vLLM's hybrid - # accounting (get_num_layers_by_block_type) recognizes: linear-attention - # layers stay "linear_attention"; every other attention variant collapses - # to "attention". - if self.layer_types is None: - return ["attention"] * self.num_hidden_layers - return [ - "linear_attention" if t == "linear_attention" else "attention" - for t in self.layer_types - ] - - -class Glm5NextVisionConfig(PreTrainedConfig): - model_type = "glm5_next_vision" - base_config_key = "vision_config" - - def __init__( - self, - depth: int = 24, - hidden_size: int = 1024, - hidden_act: str = "silu", - image_size: int = 448, - intermediate_size: int = 4096, - num_heads: int = 16, - out_hidden_size: int = 4096, - projection_intermediate_size: int = 10240, - in_channels: int = 3, - initializer_range: float = 0.02, - patch_size: int = 14, - rms_norm_eps: float = 1e-5, - spatial_merge_size: int = 2, - temporal_patch_size: int = 2, - attention_dropout: float = 0.0, - attention_bias: bool = True, - swiglu_limit: float | None = None, - **kwargs, - ): - super().__init__(**kwargs) - - self.depth = depth - self.hidden_size = hidden_size - self.hidden_act = hidden_act - self.image_size = image_size - self.intermediate_size = intermediate_size - self.num_heads = num_heads - self.out_hidden_size = out_hidden_size - # GLM-5.3-Flash merger bottleneck width (absent from the generic - # GLM-OCR vision config); the tower uses it as the PatchMerger - # context_dim instead of text_config.intermediate_size. - self.projection_intermediate_size = projection_intermediate_size - self.in_channels = in_channels - self.initializer_range = initializer_range - self.patch_size = patch_size - # GLM-5.3-Flash checkpoints ship vision_config.rms_norm_eps = 1e-5, - # but the vision tower was trained with 1e-6. Serving with 1e-5 drifts - # the RMSNorm and produces repetitive/degraded image descriptions, so - # force the trained value regardless of the checkpoint field. - self.rms_norm_eps = 1e-6 - self.spatial_merge_size = spatial_merge_size - self.temporal_patch_size = temporal_patch_size - self.attention_dropout = attention_dropout - self.attention_bias = attention_bias - self.swiglu_limit = swiglu_limit - - -class Glm5NextConfig(PreTrainedConfig): - model_type = "glm5_next" - sub_configs = { - "vision_config": Glm5NextVisionConfig, - "text_config": Glm5NextTextConfig, - } - keys_to_ignore_at_inference = ["past_key_values"] - - def __init__( - self, - text_config=None, - vision_config=None, - image_token_id: int = 154854, - video_token_id: int = 154855, - image_start_token_id: int = 154830, - image_end_token_id: int = 154831, - video_start_token_id: int = 154832, - video_end_token_id: int = 154833, - **kwargs, - ): - # Init super() first so base-class defaults don't clobber text-config - # values set below (PreTrainedConfig has many text-related defaults - # that differ from Glm5NextTextConfig). - super().__init__(**kwargs) - - if isinstance(vision_config, dict): - self.vision_config = self.sub_configs["vision_config"](**vision_config) - elif vision_config is None: - self.vision_config = self.sub_configs["vision_config"]() - else: - self.vision_config = vision_config - - if isinstance(text_config, dict): - self.text_config = self.sub_configs["text_config"](**text_config) - elif text_config is None: - # Backward compatibility: a flat top-level checkpoint (no nested - # text_config) folds its text fields into Glm5NextTextConfig. - self.text_config = self.sub_configs["text_config"](**kwargs) - else: - self.text_config = text_config - - self.image_token_id = image_token_id - self.video_token_id = video_token_id - self.image_start_token_id = image_start_token_id - self.image_end_token_id = image_end_token_id - self.video_start_token_id = video_start_token_id - self.video_end_token_id = video_end_token_id - - # Mirror attention implementation recursively onto sub-configs. - self._attn_implementation = kwargs.pop("attn_implementation", None) - - # Config-metadata fields that belong to the top-level (multimodal) config - # and must NOT be mirrored onto text_config: ``architectures`` / - # ``torch_dtype`` differ between the top-level config and the text - # sub-config, and mirroring them makes the top-level ``architectures`` - # silently read back as None (PreTrainedConfig initializes both to None), - # which then fails model-class resolution ("No model architectures are - # specified"). - _UNMIRRORED_KEYS = [ - "_name_or_path", - "model_type", - "dtype", - "torch_dtype", - "architectures", - "_attn_implementation_internal", - ] - - def __setattr__(self, key, value): - unmirrored = type(self)._UNMIRRORED_KEYS - if ( - (text_config := super().__getattribute__("__dict__").get("text_config")) - is not None - and key not in unmirrored - and key in text_config.__dict__ - ): - setattr(text_config, key, value) - else: - super().__setattr__(key, value) - - def __getattribute__(self, key): - unmirrored = type(self)._UNMIRRORED_KEYS - if ( - "text_config" in super().__getattribute__("__dict__") - and key not in unmirrored - ): - text_config = super().__getattribute__("text_config") - # Forward both instance attributes AND class-defined properties/ - # methods of the text config, so a flat text-only checkpoint - # (model_type "glm5_next", no nested text_config) sees is_moe / - # is_kda_layer / layers_block_type like a Glm5NextTextConfig. - if key in text_config.__dict__ or key in type(text_config).__dict__: - return getattr(text_config, key) - - return super().__getattribute__(key) diff --git a/vllm/transformers_utils/configs/qwen4_exp.py b/vllm/transformers_utils/configs/qwen4_exp.py deleted file mode 100644 index ed4c6cca8e..0000000000 --- a/vllm/transformers_utils/configs/qwen4_exp.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Exports for the package-owned Qwen4Exp configs.""" - -from vllm.models.qwen4_exp.config import ( - Qwen4ExpConfig, - Qwen4ExpTextConfig, - Qwen4ExpVisionConfig, -) - -__all__ = [ - "Qwen4ExpConfig", - "Qwen4ExpTextConfig", - "Qwen4ExpVisionConfig", -] diff --git a/vllm/transformers_utils/processors/__init__.py b/vllm/transformers_utils/processors/__init__.py index b46183f4e1..b913545377 100644 --- a/vllm/transformers_utils/processors/__init__.py +++ b/vllm/transformers_utils/processors/__init__.py @@ -17,7 +17,6 @@ __all__ = [ "FireRedASR2Processor", "FunASRProcessor", "GLM4VProcessor", - "Glm5NextProcessor", "Granite4VisionProcessor", "H2OVLProcessor", "Moondream3Processor", @@ -56,7 +55,6 @@ _CLASS_TO_MODULE: dict[str, str] = { "FireRedASR2Processor": "vllm.transformers_utils.processors.fireredasr2", "FunASRProcessor": "vllm.transformers_utils.processors.funasr", "GLM4VProcessor": "vllm.transformers_utils.processors.glm4v", - "Glm5NextProcessor": "vllm.transformers_utils.processors.glm5next", "Granite4VisionProcessor": "vllm.transformers_utils.processors.granite4_vision", "H2OVLProcessor": "vllm.transformers_utils.processors.h2ovl", "InternVLProcessor": "vllm.transformers_utils.processors.internvl", diff --git a/vllm/transformers_utils/processors/glm5next.py b/vllm/transformers_utils/processors/glm5next.py deleted file mode 100644 index a6cba9d77f..0000000000 --- a/vllm/transformers_utils/processors/glm5next.py +++ /dev/null @@ -1,953 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""vLLM-native multimodal processor for GLM-5.3-Flash.""" - -import math - -import numpy as np -import torch -from torchvision.transforms.v2 import functional as tvF -from transformers.image_processing_backends import ( - TorchvisionBackend, - group_images_by_shape, - reorder_images, -) -from transformers.image_processing_utils import BatchFeature -from transformers.image_utils import ( - OPENAI_CLIP_MEAN, - OPENAI_CLIP_STD, - ChannelDimension, - ImageInput, - PILImageResampling, - SizeDict, - get_image_size, -) -from transformers.models.auto.image_processing_auto import get_image_processor_config -from transformers.processing_utils import ( - ImagesKwargs, - MultiModalData, - ProcessingKwargs, - ProcessorMixin, - Unpack, - VideosKwargs, -) -from transformers.tokenization_utils_base import PreTokenizedInput, TextInput -from transformers.utils import TensorType, logging -from transformers.video_processing_utils import BaseVideoProcessor -from transformers.video_utils import ( - VideoInput, - VideoMetadata, - group_videos_by_shape, - reorder_videos, -) - -from vllm.transformers_utils.repo_utils import get_hf_file_to_dict - -logger = logging.get_logger(__name__) - -# Cap video inputs at 30,000 vision tokens to keep encoder profiling from -# starving the KV cache. Image inputs retain their checkpoint-defined budget. -_MAX_VIDEO_TOKENS = 30000 - -# Frame-sampler fallbacks (mirror the checkpoint's fps_interval=2 / -# max_frame_count_dynamic=2048); used when neither the request nor the -# processor config overrides them. -GLM_VIDEO_DEFAULT_FPS = 2.0 -GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048 - - -def glm_sample_frame_indices( - total_frames: int, - fps: float, - duration: float, - *, - target_fps: float | None = None, - max_frame_count: int | None = None, - temporal_patch_size: int = 2, -) -> list[int]: - """GLM video frame sampling (training-reference parity). - - ``target_fps`` is the ``fps_interval`` request knob. The greedy walk - advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on - frame-dense sources it collects more candidates than ``extract_t`` and - the ``> extract_t`` fixup re-spreads the picks uniformly with - ``np.linspace`` -- that fallback is the intended reference behavior, not - an accident. Short clips (fewer frames than ``extract_t``) are spread at - evenly spaced timestamps (``floor`` sampling; the linspace variant - samples frames unevenly and cost 4 points on video grounding evals). - Request overrides: ``target_fps`` -> fps interval, ``max_frame_count`` - -> frame cap. - """ - max_frame_idx = total_frames - 1 - if not duration: - duration = (round(max_frame_idx / fps) + 1) if fps else 0 - if max_frame_count is None: - max_frame_count = GLM_VIDEO_DEFAULT_MAX_FRAMES - if target_fps is None: - target_fps = GLM_VIDEO_DEFAULT_FPS - - extract_t = int(duration * target_fps) - extract_t = min(extract_t, int(max_frame_count)) - - duration_per_frame = 1 / fps - timestamps = [i * duration_per_frame for i in range(total_frames)] - max_second = int(duration) - - if total_frames < extract_t: - frame_indices = [ - math.floor(_i * total_frames / extract_t) for _i in range(extract_t) - ] - else: - frame_indices = [] - current_second = 0.0 - inv_fps = 1 / (temporal_patch_size * target_fps) - for frame_index in range(total_frames): - if timestamps[frame_index] >= current_second: - current_second += inv_fps - frame_indices.append(frame_index) - if current_second >= max_second: - break - - if len(frame_indices) < extract_t: - if len(frame_indices) == 0: - start, end = 0, max(total_frames - 1, 0) - else: - start, end = frame_indices[0], frame_indices[-1] - frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() - elif len(frame_indices) > extract_t: - frame_indices = np.linspace(0, total_frames - 1, extract_t, dtype=int).tolist() - - seen, uniq = set(), [] - for idx in frame_indices: - if idx not in seen: - seen.add(idx) - uniq.append(int(idx)) - - if len(uniq) & 1: - uniq.append(uniq[-1]) - - return uniq - - -def _ceil_to_factor(value: int, factor: int) -> int: - """Round a positive integer upward to the nearest multiple of factor.""" - return math.ceil(value / factor) * factor - - -def _fit_aligned_size_within_budget( - t: int, - h: int, - w: int, - h_factor: int, - w_factor: int, - max_pixels: int, -) -> tuple[int, int]: - """Largest proportional size whose upward-aligned canvas fits the budget. - - Binary search on the unaligned content height; each candidate is rounded - upward to h_factor/w_factor, so the returned canvas always satisfies - ``t * aligned_h * aligned_w <= max_pixels``. - """ - minimum_pixels = t * h_factor * w_factor - if max_pixels < minimum_pixels: - raise ValueError( - f"max_pixels={max_pixels} is too small. At least " - f"{minimum_pixels} pixels are required for one aligned patch." - ) - - low, high = 1, h - best_h, best_w = h_factor, w_factor - while low <= high: - content_h = (low + high) // 2 - content_w = max(1, math.floor(w * content_h / h)) - aligned_h = _ceil_to_factor(content_h, h_factor) - aligned_w = _ceil_to_factor(content_w, w_factor) - if t * aligned_h * aligned_w <= max_pixels: - best_h, best_w = aligned_h, aligned_w - low = content_h + 1 - else: - high = content_h - 1 - return best_h, best_w - - -def smart_resize( - t: int, - h: int, - w: int, - t_factor: int = 1, - h_factor: int = 28, - w_factor: int = 28, - min_pixels: int = 56 * 56, - max_pixels: int = 14 * 14 * 4 * 1280, -) -> tuple[int, int]: - """GLM-5.3-Flash ``smart_resize``: upward-aligned canvas under a - ``t_bar * h_bar * w_bar`` pixel budget. - - Height/width always round UP to their factors (content is then padded, - never cropped or distorted); an over-budget canvas is refit by binary - search instead of one-shot square-root scaling. ``h_factor`` / - ``w_factor`` carry ``patch_expand_factor`` on top of - ``patch_size * merge_size``; ``t_factor`` is ``temporal_patch_size``. For - a still image ``t = t_factor = temporal_patch_size`` so ``t_bar = - temporal_patch_size``. - """ - if min(t, h, w, t_factor, h_factor, w_factor) <= 0: - raise ValueError("Image dimensions and alignment factors must be positive.") - if min_pixels <= 0 or max_pixels <= 0: - raise ValueError("min_pixels and max_pixels must be positive.") - if min_pixels > max_pixels: - raise ValueError("min_pixels must be less than or equal to max_pixels.") - - t_bar = max(t_factor, round(t / t_factor) * t_factor) - h_bar = _ceil_to_factor(h, h_factor) - w_bar = _ceil_to_factor(w, w_factor) - - if t_bar * h_bar * w_bar > max_pixels: - h_bar, w_bar = _fit_aligned_size_within_budget( - t=t_bar, - h=h, - w=w, - h_factor=h_factor, - w_factor=w_factor, - max_pixels=max_pixels, - ) - elif t_bar * h_bar * w_bar < min_pixels: - beta = math.sqrt(min_pixels / (t * h * w)) - h_bar = _ceil_to_factor(max(1, math.ceil(h * beta)), h_factor) - w_bar = _ceil_to_factor(max(1, math.ceil(w * beta)), w_factor) - - # Alignment can push a candidate slightly over a tight max_pixels - # budget. Refit it when that happens. - if t_bar * h_bar * w_bar > max_pixels: - h_bar, w_bar = _fit_aligned_size_within_budget( - t=t_bar, - h=h, - w=w, - h_factor=h_factor, - w_factor=w_factor, - max_pixels=max_pixels, - ) - - return h_bar, w_bar - - -def _get_pad_content_size( - image_height: int, - image_width: int, - canvas_height: int, - canvas_width: int, - allow_upscale: bool = False, -) -> tuple[int, int]: - """Aspect-ratio-preserving content size that fits the canvas. - - Oversized images are shrunk proportionally. Small images are enlarged - only when ``allow_upscale``. Padding is applied after the resize. - """ - scale = min(canvas_height / image_height, canvas_width / image_width) - if not allow_upscale: - scale = min(1.0, scale) - content_height = max(1, min(canvas_height, math.floor(image_height * scale))) - content_width = max(1, min(canvas_width, math.floor(image_width * scale))) - return content_height, content_width - - -def _resize_or_pad( - stacked_images: torch.Tensor, - target_height: int, - target_width: int, - resize_mode: str, - resample: "PILImageResampling | tvF.InterpolationMode | int | None", - resize, - allow_upscale: bool = False, -) -> torch.Tensor: - """Resize onto the aligned canvas, or keep the aspect ratio and - zero-pad the right/bottom sides (``resize_mode="pad"``).""" - height, width = stacked_images.shape[-2:] - - if resize_mode == "resize": - return resize( - stacked_images, - size=SizeDict(height=target_height, width=target_width), - resample=resample, - ) - - if resize_mode != "pad": - raise ValueError("resize_mode must be either 'resize' or 'pad'.") - - content_height, content_width = _get_pad_content_size( - image_height=height, - image_width=width, - canvas_height=target_height, - canvas_width=target_width, - allow_upscale=allow_upscale, - ) - - if (content_height, content_width) != (height, width): - stacked_images = resize( - stacked_images, - size=SizeDict(height=content_height, width=content_width), - resample=resample, - ) - - # torchvision padding order: [left, top, right, bottom] -> pad only the - # right and bottom sides. - return tvF.pad( - stacked_images, - padding=[0, 0, target_width - content_width, target_height - content_height], - fill=0, - ) - - -def _pixel_budget( - min_image_tokens: int | None, - max_image_tokens: int | None, - patch_size: int, - merge_size: int, - temporal_patch_size: int, -) -> tuple[int, int]: - """(min_pixels, max_pixels) from the token bounds of - ``processor_config.json``; one vision token covers - ``temporal_patch_size * (patch_size * merge_size) ** 2`` pixels.""" - if min_image_tokens is None or max_image_tokens is None: - raise ValueError( - "min_image_tokens and max_image_tokens must be provided by " - "processor_config.json (or per-call kwargs)." - ) - factor = temporal_patch_size * (patch_size * merge_size) ** 2 - return min_image_tokens * factor, max_image_tokens * factor - - -class Glm5NextImageProcessorKwargs(ImagesKwargs, total=False): # type: ignore[call-arg] - patch_size: int | None - temporal_patch_size: int | None - merge_size: int | None - patch_expand_factor: int | None - resize_mode: str | None - min_image_tokens: int | None - max_image_tokens: int | None - - -class Glm5NextImageProcessor(TorchvisionBackend): - """Fast torchvision image processor for GLM-5.3-Flash. - - ``patch_expand_factor`` multiplies into the ``smart_resize`` spatial - factor ``patch_size * merge_size``. ``resize_mode`` picks the geometry: - ``"pad"`` (default) preserves the aspect ratio and zero-pads the - right/bottom of the upward-aligned canvas, ``"resize"`` stretches onto - it. Defaults mirror the checkpoint's ``image_processor`` config. - """ - - do_resize = True - resample = PILImageResampling.BICUBIC - size = {"longest_edge": 1} # unused: budgets come from the token bounds - do_rescale = True - do_normalize = True - image_mean = OPENAI_CLIP_MEAN - image_std = OPENAI_CLIP_STD - do_convert_rgb = True - patch_size = 14 - temporal_patch_size = 2 - merge_size = 2 - patch_expand_factor = 1 - resize_mode = "pad" - min_image_tokens = 16 - max_image_tokens = 8000 - valid_kwargs = Glm5NextImageProcessorKwargs - model_input_names = ["pixel_values", "image_grid_thw"] - - def _preprocess( - self, - images: list[torch.Tensor], - do_resize: bool, - size: SizeDict, - resample: "PILImageResampling | tvF.InterpolationMode | int | None", - do_rescale: bool, - rescale_factor: float, - do_normalize: bool, - image_mean: float | list[float] | None, - image_std: float | list[float] | None, - patch_size: int, - temporal_patch_size: int, - merge_size: int, - patch_expand_factor: int, - resize_mode: str | None, - min_image_tokens: int | None, - max_image_tokens: int | None, - disable_grouping: bool | None, - return_tensors: str | TensorType | None, - **kwargs, - ) -> BatchFeature: - resize_mode = resize_mode if resize_mode is not None else self.resize_mode - min_pixels, max_pixels = _pixel_budget( - min_image_tokens if min_image_tokens is not None else self.min_image_tokens, - max_image_tokens if max_image_tokens is not None else self.max_image_tokens, - patch_size, - merge_size, - temporal_patch_size, - ) - grouped_images, grouped_images_index = group_images_by_shape( - images, disable_grouping=disable_grouping - ) - resized_images_grouped = {} - for shape, stacked_images in grouped_images.items(): - height, width = stacked_images.shape[-2:] - if do_resize: - resized_height, resized_width = smart_resize( - t=temporal_patch_size, - h=height, - w=width, - t_factor=temporal_patch_size, - h_factor=patch_size * merge_size * patch_expand_factor, - w_factor=patch_size * merge_size * patch_expand_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - stacked_images = _resize_or_pad( - stacked_images, - target_height=resized_height, - target_width=resized_width, - resize_mode=resize_mode, - resample=resample, - resize=self.resize, - allow_upscale=(temporal_patch_size * height * width < min_pixels), - ) - resized_images_grouped[shape] = stacked_images - - resized_images = reorder_images(resized_images_grouped, grouped_images_index) - - grouped_images, grouped_images_index = group_images_by_shape( - resized_images, disable_grouping=disable_grouping - ) - processed_images_grouped = {} - processed_grids = {} - - for shape, stacked_images in grouped_images.items(): - resized_height, resized_width = stacked_images.shape[-2:] - - patches = self.rescale_and_normalize( - stacked_images, - do_rescale, - rescale_factor, - do_normalize, - image_mean, - image_std, - ) - if patches.ndim == 4: # (B, C, H, W) - patches = patches.unsqueeze(1) # (B, T=1, C, H, W) - - if patches.shape[1] % temporal_patch_size != 0: - repeats = patches[:, -1:].repeat( - 1, - temporal_patch_size - (patches.shape[1] % temporal_patch_size), - 1, - 1, - 1, - ) - patches = torch.cat([patches, repeats], dim=1) - - batch_size, t_len, channel = patches.shape[:3] - grid_t = t_len // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - - patches = patches.view( - batch_size, - grid_t, - temporal_patch_size, - channel, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - # (B, grid_t, gh, gw, mh, mw, C, tp, ph, pw) - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) - - flatten_patches = patches.reshape( - batch_size, - grid_t * grid_h * grid_w, - channel * temporal_patch_size * patch_size * patch_size, - ) - - processed_images_grouped[shape] = flatten_patches - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size - - processed_images = reorder_images( - processed_images_grouped, grouped_images_index - ) - processed_grids = reorder_images(processed_grids, grouped_images_index) - - pixel_values = torch.cat(processed_images, dim=0) - image_grid_thw = torch.tensor(processed_grids) - - return BatchFeature( - data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, - tensor_type=return_tensors, - ) - - def preprocess( - self, images: ImageInput, **kwargs: Unpack[Glm5NextImageProcessorKwargs] - ) -> BatchFeature: - return super().preprocess(images, **kwargs) - - def get_number_of_image_patches( - self, height: int, width: int, images_kwargs: dict | None = None - ) -> int: - """Number of image patches (pre-merge) for a given (height, width).""" - images_kwargs = images_kwargs or {} - patch_size = images_kwargs.get("patch_size", self.patch_size) - merge_size = images_kwargs.get("merge_size", self.merge_size) - patch_expand_factor = images_kwargs.get( - "patch_expand_factor", self.patch_expand_factor - ) - min_pixels, max_pixels = _pixel_budget( - images_kwargs.get("min_image_tokens", self.min_image_tokens), - images_kwargs.get("max_image_tokens", self.max_image_tokens), - patch_size, - merge_size, - self.temporal_patch_size, - ) - resized_height, resized_width = smart_resize( - t=self.temporal_patch_size, - h=height, - w=width, - t_factor=self.temporal_patch_size, - h_factor=patch_size * merge_size * patch_expand_factor, - w_factor=patch_size * merge_size * patch_expand_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - return grid_h * grid_w - - -class Glm5NextVideoProcessorKwargs(VideosKwargs, total=False): # type: ignore[call-arg] - fps: list[float] | float - patch_size: int - temporal_patch_size: int - merge_size: int - patch_expand_factor: int - resize_mode: str | None - target_fps: float | None - max_frames: int | None - fps_interval: int | None - max_frame_count_dynamic: int | None - min_image_tokens: int | None - max_image_tokens: int | None - - -class Glm5NextVideoProcessor(BaseVideoProcessor): - """Fast video processor for GLM-5.3-Flash. - - Shares ``smart_resize`` / the pad-mode geometry / the patchify with the - image processor, and adds GLM-5.3-Flash frame sampling - (``glm_sample_frame_indices``: ``fps_interval`` semantics with a - temporal-patch-scaled greedy walk). Defaults mirror the checkpoint's - ``video_processor`` config. - """ - - resample = PILImageResampling.BICUBIC - size = {"longest_edge": 1} # unused: budgets come from the token bounds - image_mean = OPENAI_CLIP_MEAN - image_std = OPENAI_CLIP_STD - do_resize = True - do_rescale = True - do_normalize = True - do_convert_rgb = True - do_sample_frames = True - patch_size = 14 - temporal_patch_size = 2 - patch_expand_factor = 1 - merge_size = 2 - valid_kwargs = Glm5NextVideoProcessorKwargs - num_frames = 16 - fps = 2 - fps_interval = 2.0 - max_frame_count_dynamic = 2048 - resize_mode = "pad" - min_image_tokens = 16 - max_image_tokens = 240000 - model_input_names = ["pixel_values_videos", "video_grid_thw"] - - def sample_frames( - self, - metadata: VideoMetadata, - fps: int | float | None = None, - **kwargs, - ) -> np.ndarray: - """Sample frame indices with GLM's fps-interval policy. - - ``fps`` / ``target_fps``, ``max_frames`` and ``fps_interval`` / - ``max_frame_count_dynamic`` are the overrides described in - :func:`glm_sample_frame_indices`. - """ - if metadata is None or getattr(metadata, "fps", None) is None: - raise ValueError( - "Asked to sample frames per second but no video metadata was " - "provided which is required when sampling in GLM-5.3-Flash. Please " - "pass in `VideoMetadata` object or set `do_sample_frames=False`." - ) - - target_fps = fps if fps is not None else kwargs.get("target_fps") - if target_fps is None: - target_fps = self.fps_interval - indices = glm_sample_frame_indices( - metadata.total_num_frames, - metadata.fps, - metadata.duration or 0, - target_fps=target_fps, - max_frame_count=kwargs.get("max_frames") or self.max_frame_count_dynamic, - temporal_patch_size=self.temporal_patch_size, - ) - return np.array(indices) - - def _preprocess( - self, - videos: list[torch.Tensor], - do_convert_rgb: bool = True, - do_resize: bool = True, - size: SizeDict | None = None, - resample: "PILImageResampling | int | None" = PILImageResampling.BICUBIC, - do_rescale: bool = True, - rescale_factor: float = 1 / 255.0, - do_normalize: bool = True, - image_mean: float | list[float] | None = None, - image_std: float | list[float] | None = None, - patch_size: int | None = None, - temporal_patch_size: int | None = None, - patch_expand_factor: int | None = None, - merge_size: int | None = None, - resize_mode: str | None = None, - min_image_tokens: int | None = None, - max_image_tokens: int | None = None, - return_tensors: str | TensorType | None = None, - **kwargs, - ) -> BatchFeature: - patch_expand_factor = self.patch_expand_factor - patch_size = patch_size if patch_size is not None else self.patch_size - temporal_patch_size = ( - temporal_patch_size - if temporal_patch_size is not None - else self.temporal_patch_size - ) - merge_size = merge_size if merge_size is not None else self.merge_size - resize_mode = resize_mode if resize_mode is not None else self.resize_mode - min_pixels, max_pixels = _pixel_budget( - min_image_tokens if min_image_tokens is not None else self.min_image_tokens, - max_image_tokens if max_image_tokens is not None else self.max_image_tokens, - patch_size, - merge_size, - temporal_patch_size, - ) - grouped_videos, grouped_videos_index = group_videos_by_shape(videos) - resized_videos_grouped = {} - for shape, stacked_videos in grouped_videos.items(): - if do_convert_rgb: - stacked_videos = self.convert_to_rgb(stacked_videos) - b, t_len, c, h, w = stacked_videos.shape - num_frames, height, width = t_len, h, w - if do_resize: - resized_height, resized_width = smart_resize( - t=num_frames, - h=height, - w=width, - t_factor=temporal_patch_size, - h_factor=patch_size * merge_size * patch_expand_factor, - w_factor=patch_size * merge_size * patch_expand_factor, - min_pixels=min_pixels, - max_pixels=max_pixels, - ) - stacked_videos = stacked_videos.view(b * t_len, c, h, w) - stacked_videos = _resize_or_pad( - stacked_videos, - target_height=resized_height, - target_width=resized_width, - resize_mode=resize_mode, - resample=resample, - resize=self.resize, - allow_upscale=(num_frames * height * width < min_pixels), - ) - stacked_videos = stacked_videos.view( - b, t_len, c, resized_height, resized_width - ) - resized_videos_grouped[shape] = stacked_videos - resized_videos = reorder_videos(resized_videos_grouped, grouped_videos_index) - - grouped_videos, grouped_videos_index = group_videos_by_shape(resized_videos) - processed_videos_grouped = {} - processed_grids = {} - for shape, stacked_videos in grouped_videos.items(): - resized_height, resized_width = get_image_size( - stacked_videos[0], channel_dim=ChannelDimension.FIRST - ) - stacked_videos = self.rescale_and_normalize( - stacked_videos, - do_rescale, - rescale_factor, - do_normalize, - image_mean, - image_std, - ) - patches = stacked_videos - - if pad := -patches.shape[1] % temporal_patch_size: - repeats = patches[:, -1:].expand(-1, pad, -1, -1, -1) - patches = torch.cat((patches, repeats), dim=1) - batch_size, grid_t, channel = patches.shape[:3] - grid_t = grid_t // temporal_patch_size - grid_h, grid_w = resized_height // patch_size, resized_width // patch_size - - patches = patches.view( - batch_size, - grid_t, - temporal_patch_size, - channel, - grid_h // merge_size, - merge_size, - patch_size, - grid_w // merge_size, - merge_size, - patch_size, - ) - patches = patches.permute(0, 1, 4, 7, 5, 8, 3, 2, 6, 9) - flatten_patches = patches.reshape( - batch_size, - grid_t * grid_h * grid_w, - channel * temporal_patch_size * patch_size * patch_size, - ) - - processed_videos_grouped[shape] = flatten_patches - processed_grids[shape] = [[grid_t, grid_h, grid_w]] * batch_size - - processed_videos = reorder_videos( - processed_videos_grouped, grouped_videos_index - ) - processed_grids = reorder_videos(processed_grids, grouped_videos_index) - pixel_values_videos = torch.cat(processed_videos, dim=0) - video_grid_thw = torch.tensor(processed_grids) - return BatchFeature( - data={ - "pixel_values_videos": pixel_values_videos, - "video_grid_thw": video_grid_thw, - }, - tensor_type=return_tensors, - ) - - -class Glm5NextProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] - images_kwargs: Glm5NextImageProcessorKwargs - videos_kwargs: Glm5NextVideoProcessorKwargs - _defaults = { - "text_kwargs": { - "padding": False, - "return_token_type_ids": False, - "return_mm_token_type_ids": False, - }, - "videos_kwargs": {"return_metadata": True}, - } - - -class Glm5NextProcessor(ProcessorMixin): - """Wraps a GLM-5.3-Flash image processor, video processor and tokenizer. - - Token expansion per image = ``prod(image_grid_thw) // merge_size**2``; video - frames are expanded with ``<|begin_of_image|>...<|end_of_image|>{ts} seconds`` - structure (mrope timestamps). - """ - - attributes = ["image_processor", "tokenizer", "video_processor"] - image_processor_class = "AutoImageProcessor" - video_processor_class = "AutoVideoProcessor" - tokenizer_class = ("PythonBackend", "TokenizersBackend") - - def __init__( - self, - image_processor=None, - tokenizer=None, - video_processor=None, - chat_template=None, - **kwargs, - ) -> None: - super().__init__( - image_processor, tokenizer, video_processor, chat_template=chat_template - ) - self.image_token = ( - "<|image|>" - if not hasattr(tokenizer, "image_token") - else tokenizer.image_token - ) - self.video_token = ( - "<|video|>" - if not hasattr(tokenizer, "video_token") - else tokenizer.video_token - ) - self.image_token_id = ( - tokenizer.image_token_id - if getattr(tokenizer, "image_token_id", None) - else tokenizer.convert_tokens_to_ids(self.image_token) - ) - self.video_token_id = ( - tokenizer.video_token_id - if getattr(tokenizer, "video_token_id", None) - else tokenizer.convert_tokens_to_ids(self.video_token) - ) - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - """Build the processor directly from the checkpoint config. - - GLM-5.3-Flash stores nested image/video configs in - ``processor_config.json`` and declares a custom processor class. This - method reads those configs directly and caps only the video token budget. - """ - from transformers import AutoTokenizer - - model_path = pretrained_model_name_or_path - tokenizer = AutoTokenizer.from_pretrained(model_path, **kwargs) - - def _cap_cfg(cfg: dict, *, is_video: bool) -> dict: - # Video keeps a serving token cap (the checkpoint's 240k-token - # budget would starve the KV cache at startup profiling); images - # follow the checkpoint budget verbatim so preprocessing matches - # the HF reference exactly. - if is_video and cfg.get("max_image_tokens") is not None: - cfg["max_image_tokens"] = min( - cfg["max_image_tokens"], _MAX_VIDEO_TOKENS - ) - return cfg - - ip_cfg = _cap_cfg( - dict(get_image_processor_config(model_path, **kwargs)), is_video=False - ) - image_processor = Glm5NextImageProcessor( - **{k: v for k, v in ip_cfg.items() if k != "image_processor_type"} - ) - - processor_config = get_hf_file_to_dict( - "processor_config.json", - model_path, - revision=kwargs.get("revision", "main"), - ) - if processor_config is None: - raise ValueError(f"Missing processor_config.json for {model_path}") - vp_cfg = _cap_cfg(dict(processor_config["video_processor"]), is_video=True) - video_processor = Glm5NextVideoProcessor( - **{k: v for k, v in vp_cfg.items() if k != "video_processor_type"} - ) - - return cls( - image_processor=image_processor, - tokenizer=tokenizer, - video_processor=video_processor, - ) - - def __call__( - self, - images: ImageInput | None = None, - text: TextInput - | PreTokenizedInput - | list[TextInput] - | list[PreTokenizedInput] = None, - videos: VideoInput | None = None, - **kwargs: Unpack[Glm5NextProcessorKwargs], - ) -> BatchFeature: - output_kwargs = self._merge_kwargs( - Glm5NextProcessorKwargs, - tokenizer_init_kwargs=self.tokenizer.init_kwargs, - **kwargs, - ) - if images is not None: - image_inputs = self.image_processor( - images=images, **output_kwargs["images_kwargs"] - ) - else: - image_inputs = {} - - if videos is not None: - videos_inputs = self.video_processor( - videos=videos, **output_kwargs["videos_kwargs"] - ) - if "return_metadata" not in kwargs: - videos_inputs.pop("video_metadata") - else: - videos_inputs = {} - - if not isinstance(text, list): - text = [text] - - # Prompt updates expand the unchanged image/video markers after this call. - return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) - return_mm_token_type_ids = output_kwargs["text_kwargs"].pop( - "return_mm_token_type_ids", False - ) - text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) - - if return_mm_token_type_ids: - array_ids = np.array(text_inputs["input_ids"]) - mm_token_type_ids = np.zeros_like(text_inputs["input_ids"]) - mm_token_type_ids[array_ids == self.image_token_id] = 1 - text_inputs["mm_token_type_ids"] = mm_token_type_ids.tolist() - return BatchFeature( - data={**text_inputs, **image_inputs, **videos_inputs}, - tensor_type=return_tensors, - ) - - def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs): - vision_data = {} - if image_sizes is not None: - images_kwargs = Glm5NextProcessorKwargs._defaults.get("images_kwargs", {}) - images_kwargs.update(kwargs) - merge_size = ( - images_kwargs.get("merge_size", None) or self.image_processor.merge_size - ) - - num_image_patches = [ - self.image_processor.get_number_of_image_patches( - *image_size, images_kwargs - ) - for image_size in image_sizes - ] - num_image_tokens = [(n // merge_size**2) for n in num_image_patches] - vision_data.update( - { - "num_image_tokens": num_image_tokens, - "num_image_patches": num_image_patches, - } - ) - - if video_sizes is not None: - videos_kwargs = Glm5NextProcessorKwargs._defaults.get("videos_kwargs", {}) - videos_kwargs.update(kwargs) - num_video_patches = [ - self.video_processor.get_number_of_video_patches( - *video_size, videos_kwargs - ) - for video_size in video_sizes - ] - num_video_tokens = [(n // merge_size**2) for n in num_video_patches] - vision_data["num_video_tokens"] = num_video_tokens - - return MultiModalData(**vision_data) - - def post_process_image_text_to_text( - self, - generated_outputs, - skip_special_tokens=True, - clean_up_tokenization_spaces=False, - **kwargs, - ): - return self.tokenizer.batch_decode( - generated_outputs, - skip_special_tokens=skip_special_tokens, - clean_up_tokenization_spaces=clean_up_tokenization_spaces, - **kwargs, - ) - - -__all__ = [ - "Glm5NextImageProcessor", - "Glm5NextVideoProcessor", - "Glm5NextProcessor", - "smart_resize", -] From 785b990116ab2b7c77f5d946b2f52c2a8f9b9515 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Thu, 1 Oct 2026 22:25:48 -0400 Subject: [PATCH 233/248] [Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623) Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> --- docs/features/batch_invariance.md | 8 +- tests/distributed/test_custom_all_reduce.py | 90 +++++++++++++++++++ tests/v1/determinism/test_batch_invariance.py | 20 ----- vllm/config/parallel.py | 4 - .../device_communicators/cuda_communicator.py | 14 ++- .../device_communicators/custom_all_reduce.py | 23 ++++- .../determinism/batch_invariant.py | 2 + 7 files changed, 130 insertions(+), 31 deletions(-) diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index ff704348b5..182559141c 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -146,11 +146,15 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes -3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) -4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), +3. Disables certain optimizations that may introduce non-determinism (such as sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) +4. Under tensor parallelism, keeps custom all-reduce on with a fixed reduction order (the 1-stage kernel is pinned, and large inputs are reduced in fixed-size chunks), and disables FlashInfer, AITER and QuickReduce all-reduce +5. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). +!!! warning + Batch invariance under tensor parallelism is not yet supported for all-reduces whose size isn't a multiple of 16 bytes, for example a hidden size that isn't a multiple of 8 in fp16/bf16. Such tensors can switch between NCCL and custom all-reduce depending on batch size. All validated models meet this requirement. + !!! note Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility. diff --git a/tests/distributed/test_custom_all_reduce.py b/tests/distributed/test_custom_all_reduce.py index 23d66ee973..84ff524499 100644 --- a/tests/distributed/test_custom_all_reduce.py +++ b/tests/distributed/test_custom_all_reduce.py @@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype( assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected +@pytest.mark.parametrize("batch_invariant", [False, True]) +def test_custom_allreduce_size_gate_ignored_under_batch_invariance( + batch_invariant: bool, +) -> None: + """Batch invariance must not switch backends based on the input size.""" + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = False + communicator.world_size = 2 + communicator.max_size = 1024 + communicator.batch_invariant = batch_invariant + communicator._ptr = 0 + + oversized = torch.empty(1024, dtype=torch.float16) + assert communicator.should_custom_ar(oversized) is batch_invariant + + +@pytest.mark.parametrize("batch_invariant", [False, True]) +def test_custom_reduce_scatter_disabled_under_batch_invariance( + monkeypatch, batch_invariant: bool +) -> None: + """Reduce-scatter stays on one backend under batch invariance.""" + monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True) + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = False + communicator.world_size = 2 + communicator.fully_connected = True + communicator.mnnvl_only = False + communicator.mnnvl_multicast_ptr = 0 + communicator.max_reduce_scatter_size = 1024 + communicator.max_mnnvl_reduce_scatter_size = 1024 + communicator.batch_invariant = batch_invariant + + in_range = torch.empty(64, dtype=torch.float16) + assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant + + @pytest.mark.parametrize( ("major", "local_multicast", "expected"), [ @@ -622,6 +658,60 @@ def eager_allreduce( torch.testing.assert_close(out, inp * (tp_size**num_communication)) +@ray.remote(num_gpus=1, max_calls=1) +def chunked_allreduce( + monkeypatch: pytest.MonkeyPatch, + tp_size, + pp_size, + rank, + distributed_init_port, +): + """Inputs above max_size are reduced in chunks and match the fixed-order + reference bitwise, both eagerly and inside a CUDA graph.""" + with monkeypatch.context() as m: + m.delenv("CUDA_VISIBLE_DEVICES", raising=False) + m.delenv("HIP_VISIBLE_DEVICES", raising=False) + m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage") + device = torch.device(f"cuda:{rank}") + torch.accelerator.set_device_index(device) + init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port) + ensure_model_parallel_initialized(tp_size, pp_size) + group = get_tp_group().device_group + fa = get_tp_group().device_communicator.ca_comm + + # Two full chunks plus a partial one. + dtype = torch.bfloat16 + chunk_numel = fa.max_size // dtype.itemsize + inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype) + gathered = [torch.empty_like(inp) for _ in range(tp_size)] + dist.all_gather(gathered, inp, group=group) + ref = gathered[0].float() + for peer in gathered[1:]: + ref = ref + peer.float() + ref = ref.to(torch.bfloat16) + + out = fa.all_reduce(inp, registered=False) + assert torch.equal(out, ref) + + # Weak-contiguous but not C-contiguous: a transposed matrix. + inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t() + assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t) + + with fa.capture(): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = fa.all_reduce(inp, registered=True) + graph.replay() + assert torch.equal(out, ref) + + +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size): + if tp_size > torch.accelerator.device_count(): + pytest.skip("Not enough GPUs to run the test.") + multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce) + + @pytest.mark.parametrize("tp_size", [2]) @pytest.mark.parametrize("pipeline_parallel_size", [1, 2]) @pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce]) diff --git a/tests/v1/determinism/test_batch_invariance.py b/tests/v1/determinism/test_batch_invariance.py index b167078e06..f49d171f6f 100644 --- a/tests/v1/determinism/test_batch_invariance.py +++ b/tests/v1/determinism/test_batch_invariance.py @@ -176,17 +176,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN( random.seed(seed) tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1")) - # For batch invariance, disable custom all-reduce to ensure deterministic - # all-reduce operations (custom all-reduce may not be deterministic) - import vllm.envs as envs - - disable_custom_ar = envs.VLLM_BATCH_INVARIANT - - if disable_custom_ar: - print(f"\n{'=' * 80}") - print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})") - print(f"{'=' * 80}\n") - llm = LLM( model=TEST_MODEL, tensor_parallel_size=tp_size, @@ -744,15 +733,6 @@ def test_decode_logprobs_match_prefill_logprobs( random.seed(seed) tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1")) - import vllm.envs as envs - - disable_custom_ar = envs.VLLM_BATCH_INVARIANT - - if disable_custom_ar: - print(f"\n{'=' * 80}") - print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})") - print(f"{'=' * 80}\n") - llm = LLM( model=TEST_MODEL, tensor_parallel_size=tp_size, diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index b9136cfbb2..f4b825eb33 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -1087,10 +1087,6 @@ class ParallelConfig: # Lazy import to avoid circular import from vllm.v1.executor import Executor - # Enable batch invariance settings if requested - if envs.VLLM_BATCH_INVARIANT: - self.disable_custom_all_reduce = True - if ( self.distributed_executor_backend is not None and not isinstance(self.distributed_executor_backend, str) diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index fb0c281b4a..ef790f7732 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -68,8 +68,11 @@ class CudaCommunicator(DeviceCommunicatorBase): envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC and not envs.VLLM_BATCH_INVARIANT ) - use_aiter_allreduce = use_custom_allreduce and bool( - rocm_aiter_ops.is_custom_all_reduce_enabled() + # Neither AITER nor QuickReduce all-reduce has a fixed reduction order. + use_aiter_allreduce = ( + use_custom_allreduce + and not envs.VLLM_BATCH_INVARIANT + and bool(rocm_aiter_ops.is_custom_all_reduce_enabled()) ) self.use_custom_allreduce = use_custom_allreduce @@ -166,7 +169,12 @@ class CudaCommunicator(DeviceCommunicatorBase): else: self.use_aiter_ag_rs = True - if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm(): + if ( + use_custom_allreduce + and self.world_size > 1 + and current_platform.is_rocm() + and not envs.VLLM_BATCH_INVARIANT + ): # Initialize a custom quick all-reduce implementation for AMD. # Quick reduce is designed as a complement to custom allreduce # (vLLM's or AITER's), so it is initialized for either backend. diff --git a/vllm/distributed/device_communicators/custom_all_reduce.py b/vllm/distributed/device_communicators/custom_all_reduce.py index b6fb540c7d..9e8477e494 100644 --- a/vllm/distributed/device_communicators/custom_all_reduce.py +++ b/vllm/distributed/device_communicators/custom_all_reduce.py @@ -119,6 +119,7 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402 class CustomAllreduce: _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8, 16] + batch_invariant = False _DEFAULT_ALL_GATHER_MAX_SIZE = 2 * 1024 * 1024 _DEFAULT_MNNVL_ALL_GATHER_MAX_SIZES = { 2: 8 * 1024 * 1024, @@ -179,6 +180,7 @@ class CustomAllreduce: self.mnnvl_multimem_rs_local_ptr = 0 self.mnnvl_multimem_rs_multicast_ptr = 0 self.mnnvl_only = False + self.batch_invariant = envs.VLLM_BATCH_INVARIANT if not custom_ar: # disable because of missing custom allreduce library @@ -504,7 +506,7 @@ class CustomAllreduce: # for 4 or more non NVLink-capable GPUs, custom allreduce provides # little performance improvement over NCCL. if self.world_size == 2 or self.fully_connected: - return inp_size < self.max_size + return self.batch_invariant or inp_size < self.max_size return False def all_reduce( @@ -518,13 +520,27 @@ class CustomAllreduce: """ if out is None: out = torch.empty_like(inp) + chunk_numel = self.max_size // inp.element_size() + if inp.numel() <= chunk_numel: + self._all_reduce_chunk(inp, out, registered) + return out + # Storage-order views: the gate only admits one contiguous block. + flat_inp = inp.as_strided((inp.numel(),), (1,), inp.storage_offset()) + flat_out = out.as_strided((out.numel(),), (1,), out.storage_offset()) + for start in range(0, flat_inp.numel(), chunk_numel): + end = start + chunk_numel + self._all_reduce_chunk(flat_inp[start:end], flat_out[start:end], registered) + return out + + def _all_reduce_chunk( + self, inp: torch.Tensor, out: torch.Tensor, registered: bool + ) -> None: if registered: ops.all_reduce(self._ptr, inp, out, 0, 0) else: ops.all_reduce( self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size ) - return out def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None: """The main allreduce API that provides support for cuda graph.""" @@ -603,6 +619,9 @@ class CustomAllreduce: ) -> _ReduceScatterBackend | None: if self.disabled or not current_platform.is_cuda(): return None + # The size gates below would switch backends per batch. + if self.batch_invariant: + return None if self.world_size == 16 and not self.mnnvl_only: return None inp_size = inp.nbytes diff --git a/vllm/model_executor/determinism/batch_invariant.py b/vllm/model_executor/determinism/batch_invariant.py index 308e8c4f42..688e14d709 100644 --- a/vllm/model_executor/determinism/batch_invariant.py +++ b/vllm/model_executor/determinism/batch_invariant.py @@ -1139,6 +1139,8 @@ def enable_batch_invariant_mode(): def override_envs_for_invariance(): os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0" + # Only the 1-stage kernel has a size- and rank-independent reduction order. + os.environ["VLLM_CUSTOM_ALLREDUCE_ALGO"] = "1stage" os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" From 5a661487e29433b009508c8b3045847eb11041fa Mon Sep 17 00:00:00 2001 From: Hyunjae Woo <107147848+nv-hwoo@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:29:35 -0700 Subject: [PATCH 234/248] [Feature] Add native ModelExpress weight transfer backend (#58399) Signed-off-by: Hyunjae Woo Co-authored-by: aoshen02 --- docs/training/weight_transfer/README.md | 1 + docs/training/weight_transfer/modelexpress.md | 156 ++++++++ tests/distributed/test_weight_transfer.py | 353 ++++++++++++++++++ vllm/distributed/weight_transfer/factory.py | 7 + .../weight_transfer/modelexpress_engine.py | 41 ++ 5 files changed, 558 insertions(+) create mode 100644 docs/training/weight_transfer/modelexpress.md create mode 100644 vllm/distributed/weight_transfer/modelexpress_engine.py diff --git a/docs/training/weight_transfer/README.md b/docs/training/weight_transfer/README.md index 77d2728345..c8382e2a06 100644 --- a/docs/training/weight_transfer/README.md +++ b/docs/training/weight_transfer/README.md @@ -36,6 +36,7 @@ engine drives on your behalf: | [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Checkpoint-coordinate sparse weight patches | | [sharded_rdt](sharded_rdt.md) | NIXL / Ray Direct Transport (pull-based) | Very large models where each worker needs only its own slice (MoE with expert parallelism) | | [nccl_m2n](m2n.md) | NCCL M2N reshard | Trainer and inference use different sharding layouts | +| [ModelExpress](modelexpress.md) | ModelExpress version-based transfer | Install immutable published weight versions, including object-storage checkpoints and deltas | ## Quickstart diff --git a/docs/training/weight_transfer/modelexpress.md b/docs/training/weight_transfer/modelexpress.md new file mode 100644 index 0000000000..fd47e09d22 --- /dev/null +++ b/docs/training/weight_transfer/modelexpress.md @@ -0,0 +1,156 @@ +# ModelExpress Engine + +The `modelexpress` weight transfer engine installs immutable weight versions +published through [ModelExpress](https://github.com/ai-dynamo/modelexpress). +Each update identifies the exact version to install, while ModelExpress handles +source selection, transfer, and installation. + +## When to Use ModelExpress + +- Your workflow manages immutable weight versions through the ModelExpress + Refit API. +- Inference workers need to pull a specific published version through + ModelExpress, using GPU peer transfer or object storage. +- You want to load full checkpoints or replay checkpoint deltas from S3. + +## How It Works + +1. A publisher creates a weight-version resource through the ModelExpress Refit + API, publishes its payload, and marks the version READY. +2. The orchestrator pauses generation and calls `start_weight_update` on the + inference engine. +3. The orchestrator calls `update_weights` with the target `version_id`. + Each worker's `ModelExpressGeneratorClient` resolves and stages that version, + then applies it to the live vLLM model. +4. After a successful update, `finish_weight_update` releases the staged handle. + The orchestrator can then resume inference. + +Initialize the backend before starting an update session, and send exactly one +version per session. Invalid transitions must raise; an orchestrator must keep +inference paused after an update error and retry with a new session only after +cleanup. An error does not roll back weights that were already applied. + +`ModelExpressTrainerClient` is an optional publisher. Custom integrations can +manage weight-version resources directly through the ModelExpress server's +Refit API. For S3, create a STAGING version with the correct model, payload +format, lineage, and index URI; upload the index and referenced shards in the +format expected by `ModelExpressGeneratorClient`; then mark the version READY. +See the [S3 Delta Weight Refit guide](https://github.com/ai-dynamo/modelexpress/blob/main/docs/S3_DELTA_WEIGHT_REFIT.md) +for the publication sequence and generator artifact requirements. + +vLLM's `modelexpress_engine.py` re-exports the engine and its init/update +dataclasses from +`modelexpress_rl.inference.engines.vllm.weight_transfer_engine`. +The implementation and configuration schema live in ModelExpress. Upgrading +the installed ModelExpress package and restarting workers picks up compatible +changes without a vLLM code change. The package must continue to implement +vLLM's `WeightTransferEngine` interface. + +## Inference Side + +!!! warning "Development version" + + This backend currently uses unreleased ModelExpress features from `main`, + which may introduce breaking changes. These features will be included in + the official ModelExpress v0.8.0 release. + +Install ModelExpress from the `main` branch on every worker: + +```bash +uv pip install "git+https://github.com/ai-dynamo/modelexpress@main#subdirectory=modelexpress_client/python" +``` + +The backend is registered natively and loaded only when selected; +`VLLM_PLUGINS=modelexpress` is not required for weight transfer. If the plugin +is enabled, it preserves the native registration and uses the same engine +implementation. + +```python +from vllm import LLM +from vllm.config import WeightTransferConfig + +llm = LLM(model="my-model", weight_transfer_config=WeightTransferConfig(backend="modelexpress")) +``` + +```bash +vllm serve /models/launch \ + --load-format safetensors \ + --weight-transfer-config '{"backend":"modelexpress"}' +``` + +### Initialize the engine + +Initialize once by passing `init_info` to `init_weight_transfer_engine`. +For HTTP administration, set `VLLM_SERVER_DEV_MODE=1` before starting vLLM and +restrict these endpoints to trusted orchestrators. + +For S3 updates, send the following body to `POST /init_weight_transfer_engine`: + +```json +{ + "init_info": { + "model_name": "policy", + "server_url": "modelexpress:8001", + "object_storage_type": "S3", + "initial_base_version_id": "policy-v0", + "seed_checkpoint_path": "/models/launch", + "refit_checkpoint_dir": "/mxdelta/receiver", + "refit_checkpoint_max_size_gb": 200, + "object_storage_region_name": "us-west-2" + } +} +``` + +### `ModelExpressWeightTransferInitInfo` + +These fields belong inside `init_info`, rather than +`--weight-transfer-config`. The installed ModelExpress package defines the +schema and defaults. + +| Field | Default | Description | +| ----- | ------- | ----------- | +| `model_name` | vLLM model name | Logical model identity used for published ModelExpress versions. Set this when it differs from the vLLM model path or name. | +| `server_url` | ModelExpress configuration | ModelExpress server address, such as `modelexpress:8001`. If omitted, uses ModelExpress's environment settings, then `localhost:8001`. | +| `initial_serving_version_id` | `None` | Version already loaded by the inference worker. If omitted, ModelExpress uses the initial version supplied by its runtime, when available. | +| `registration_ttl_seconds` | Three MX heartbeat intervals | Worker registration lifetime in seconds, renewed by ModelExpress. | +| `lease_ttl_seconds` | Registration lifetime | Weight-version lease lifetime in seconds, renewed while held. | +| `max_transfer_attempts` | `3` | Maximum source-discovery and transfer attempts for one staged update. | +| `max_replay_chain_length` | `64` | Maximum number of payload revisions replayed to reach the requested version. | +| `rpc_timeout_seconds` | `30.0` | Deadline in seconds for each control-plane or manifest RPC. | + +### Object storage settings + +To enable object storage, provide `object_storage_type`, +`initial_base_version_id`, `seed_checkpoint_path`, and `refit_checkpoint_dir` +inside `init_info`. Endpoint and region settings also require these four fields. + +| Field | Default | Description | +| ----- | ------- | ----------- | +| `object_storage_type` | `None` | Object storage provider. Use `"S3"`; the generator currently supports S3. | +| `initial_base_version_id` | `None` | Registered base version matching the seed checkpoint. Required for object storage. | +| `seed_checkpoint_path` | `None` | Local seed checkpoint path accessible to each worker. Required for object storage. | +| `refit_checkpoint_dir` | `None` | Local directory for cached checkpoints and delta replay. Required for object storage. | +| `refit_checkpoint_max_size_gb` | ModelExpress default | Checkpoint cache quota in decimal GB. The example sets an explicit 200 GB quota; it does not reserve disk space. | +| `object_storage_endpoint_url` | `None` | Optional endpoint override for an S3-compatible service. | +| `object_storage_region_name` | `None` | Optional S3 region, such as `us-west-2`. | + +The seed checkpoint must match `initial_base_version_id`. Credentials use +ModelExpress's normal credential chain. Omit the object storage settings when +using only GPU peer transfer. + +### Apply a weight update + +After the target version is READY, pause generation and call +`POST /start_weight_update`. Then send the exact immutable version ID to +`POST /update_weights`: + +```json +{"update_info": {"version_id": "policy-v1"}} +``` + +Call `POST /finish_weight_update` only after the update succeeds, then resume +inference. The staged handle is retained until finish or shutdown. Staging and +installation failures propagate; do not resume a worker with uncertain weights. + +See the ModelExpress documentation for publication, source selection, object +storage, and deployment requirements. diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index 1353def738..3af7b7aeb9 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -6,10 +6,14 @@ Unit tests for engine classes (parsing, validation, registry). Integration tests for NCCL and IPC weight transfer between processes using Ray. """ +import builtins import pickle +import runpy import threading import time from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace from unittest.mock import MagicMock import pybase64 as base64 @@ -297,6 +301,36 @@ class TestEngineRegistry: ) assert isinstance(engine, SparseNCCLWeightTransferEngine) + def test_modelexpress_registration_is_native_and_lazy(self, monkeypatch): + from vllm.distributed.weight_transfer import factory + + imported = [] + module = MagicMock() + module.ModelExpressWeightTransferEngine.__name__ = ( + "ModelExpressWeightTransferEngine" + ) + + def import_module(name): + imported.append(name) + return module + + monkeypatch.setattr(factory.importlib, "import_module", import_module) + config = WeightTransferConfig(backend="modelexpress") + vllm_config = create_mock_vllm_config() + device = torch.device("cpu") + model = MagicMock(spec=torch.nn.Module) + + assert imported == [] + engine = WeightTransferEngineFactory.create_engine( + config, vllm_config, device, model + ) + + assert imported == ["vllm.distributed.weight_transfer.modelexpress_engine"] + module.ModelExpressWeightTransferEngine.assert_called_once_with( + config, vllm_config, device, model + ) + assert engine is module.ModelExpressWeightTransferEngine.return_value + def test_create_engine_invalid_backend(self): config = WeightTransferConfig(backend="invalid") with pytest.raises(ValueError, match="Invalid weight transfer backend"): @@ -1976,3 +2010,322 @@ def test_sparse_nccl_trainer_non_sender_skips_client(): engine.send_weights([_sparse_patch()]) engine.send_weight_chunk([_sparse_patch()]) assert client.order == [] + + +# --- Optional ModelExpress Client Lifecycle --- + + +def _import_modelexpress_shim(): + path = ( + Path(__file__).parents[2] + / "vllm/distributed/weight_transfer/modelexpress_engine.py" + ) + runpy.run_path(str(path), run_name="_test_modelexpress_shim") + + +@pytest.mark.parametrize( + "missing_module", + [ + "modelexpress", + "modelexpress_rl", + "modelexpress_rl.inference", + "modelexpress_rl.inference.engines", + "modelexpress_rl.inference.engines.vllm", + "modelexpress_rl.inference.engines.vllm.weight_transfer_engine", + ], +) +def test_modelexpress_missing_backend_explains_source_install( + monkeypatch, missing_module +): + original_import = builtins.__import__ + error = ModuleNotFoundError(name=missing_module) + + def import_without_backend(name, *args, **kwargs): + if name == "modelexpress": + raise error + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_backend) + with pytest.raises(ImportError, match="modelexpress_client/python") as exc_info: + _import_modelexpress_shim() + assert exc_info.value.__cause__ is error + assert "uv pip install" in str(exc_info.value) + + +@pytest.mark.parametrize("missing_module", ["grpc", "modelexpress.internal_dependency"]) +def test_modelexpress_preserves_transitive_import_errors(monkeypatch, missing_module): + original_import = builtins.__import__ + error = ModuleNotFoundError(name=missing_module) + + def import_without_dependency(name, *args, **kwargs): + if name == "modelexpress": + raise error + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_dependency) + with pytest.raises(ModuleNotFoundError) as exc_info: + _import_modelexpress_shim() + assert exc_info.value is error + + +@pytest.fixture +def mx_client(monkeypatch): + client_module = pytest.importorskip("modelexpress_rl.inference.client") + client = MagicMock() + monkeypatch.setattr( + client_module.ModelExpressGeneratorClient, + "initialize", + MagicMock(return_value=client), + ) + monkeypatch.setattr(torch.accelerator, "synchronize", MagicMock()) + return client + + +def _make_modelexpress_engine(): + return WeightTransferEngineFactory.create_engine( + WeightTransferConfig(backend="modelexpress"), + SimpleNamespace( + parallel_config=SimpleNamespace(), + model_config=SimpleNamespace(model="test/model"), + ), + torch.device("cpu"), + torch.nn.Linear(2, 2), + ) + + +@pytest.fixture +def mx_worker(mx_client, monkeypatch): + from vllm.v1.worker import gpu_worker + + monkeypatch.setattr(gpu_worker, "set_current_vllm_config", lambda _: nullcontext()) + worker = object.__new__(gpu_worker.Worker) + worker.weight_transfer_engine = _make_modelexpress_engine() + worker.vllm_config = worker.weight_transfer_engine.vllm_config + worker._weight_update_active = False + worker._weight_update_is_draft = False + worker.model_runner = MagicMock() + return worker + + +def test_modelexpress_worker_rejects_updates_before_initialization( + mx_worker, mx_client +): + with pytest.raises(RuntimeError, match="not initialized"): + mx_worker.start_weight_update() + with pytest.raises(RuntimeError, match="start_weight_update must be called"): + mx_worker.update_weights({"version_id": "version-a"}) + with pytest.raises(RuntimeError, match="without a matching"): + mx_worker.finish_weight_update() + mx_client.stage_weight.assert_not_called() + + +@pytest.mark.parametrize( + "second_update, error_type", + [ + ({"version_id": "version-a"}, RuntimeError), + ({"version_id": "version-b"}, RuntimeError), + ({"version_id": ""}, ValueError), + ({"version_id": None}, ValueError), + ({}, ValueError), + ], +) +def test_modelexpress_worker_recovers_after_rejected_update( + mx_worker, mx_client, second_update, error_type +): + """A rejected payload must release A so a fresh session can actually install B.""" + mx_worker.weight_transfer_engine.init_transfer_engine( + mx_worker.weight_transfer_engine.parse_init_info({}) + ) + staged_a = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock()) + staged_b = SimpleNamespace(version_id="version-b", metrics={}, release=MagicMock()) + mx_client.stage_weight.side_effect = [staged_a, staged_b] + mx_client.apply_weight.return_value = {} + mx_worker.start_weight_update() + mx_worker.update_weights({"version_id": "version-a"}) + + with pytest.raises(error_type): + mx_worker.update_weights(second_update) + staged_a.release.assert_called_once_with() + mx_client.apply_weight.assert_called_once_with(staged_a) + + mx_worker.start_weight_update() + mx_worker.update_weights({"version_id": "version-b"}) + mx_worker.finish_weight_update() + + assert [ + call.kwargs["version"].version_id + for call in mx_client.stage_weight.call_args_list + ] == ["version-a", "version-b"] + mx_client.apply_weight.assert_called_with(staged_b) + staged_b.release.assert_called_once_with() + + +def test_modelexpress_worker_can_retry_failed_finish(mx_worker, mx_client): + mx_worker.weight_transfer_engine.init_transfer_engine( + mx_worker.weight_transfer_engine.parse_init_info({}) + ) + staged = mx_client.stage_weight.return_value + staged.release.side_effect = [RuntimeError("release failed"), None] + mx_worker.start_weight_update() + mx_worker.update_weights({"version_id": "version-a"}) + + with pytest.raises(RuntimeError, match="release failed"): + mx_worker.finish_weight_update() + mx_worker.finish_weight_update() + + assert staged.release.call_count == 2 + assert not mx_worker._weight_update_active + mx_worker.start_weight_update() + + +def test_native_engine_applies_and_releases_exact_versions(mx_client): + from modelexpress_rl.inference.engines.vllm import ( + weight_transfer_engine as mx_engine, + ) + + from vllm.distributed.weight_transfer import modelexpress_engine + + engine = _make_modelexpress_engine() + assert type(engine) is mx_engine.ModelExpressWeightTransferEngine + assert ( + modelexpress_engine.ModelExpressWeightTransferInitInfo + is mx_engine.ModelExpressWeightTransferInitInfo + ) + assert ( + modelexpress_engine.ModelExpressWeightTransferUpdateInfo + is mx_engine.ModelExpressWeightTransferUpdateInfo + ) + assert not engine.supports_draft_weight_update + engine.init_transfer_engine(engine.parse_init_info({})) + + for version_id in ("version-a", "version-b"): + staged = SimpleNamespace(version_id=version_id, metrics={}, release=MagicMock()) + mx_client.stage_weight.return_value = staged + mx_client.apply_weight.return_value = {} + engine.start_weight_update() + engine.update_weights({"version_id": version_id}) + + assert ( + mx_client.stage_weight.call_args.kwargs["version"].version_id == version_id + ) + mx_client.apply_weight.assert_called_with(staged) + torch.accelerator.synchronize.assert_called() + staged.release.assert_not_called() + engine.finish_weight_update() + staged.release.assert_called_once_with() + + engine.shutdown() + engine.shutdown() + mx_client.close.assert_called_once_with() + + +def test_vime_init_config_reaches_mx_client(mx_client): + from modelexpress_rl.inference.client import ModelExpressGeneratorClient + + engine = _make_modelexpress_engine() + ModelExpressGeneratorClient.initialize.assert_not_called() + engine.init_transfer_engine( + engine.parse_init_info( + { + "model_name": "policy", + "server_url": "mx:8001", + "initial_serving_version_id": "serving-a", + "object_storage_type": "S3", + "initial_base_version_id": "base-a", + "seed_checkpoint_path": "/models/launch", + "refit_checkpoint_dir": "/mxdelta/receiver", + "refit_checkpoint_max_size_gb": 200, + "object_storage_endpoint_url": "http://minio:9000", + "object_storage_region_name": "us-west-2", + "registration_ttl_seconds": 90, + "lease_ttl_seconds": 60, + "max_transfer_attempts": 4, + "max_replay_chain_length": 17, + "rpc_timeout_seconds": 12.5, + } + ) + ) + config = ModelExpressGeneratorClient.initialize.call_args.args[0] + assert config.engine_context.model is engine.model + assert config.engine_context.vllm_config is engine.vllm_config + assert config.model_name == "policy" + assert config.server_url == "mx:8001" + assert config.initial_serving_version_id == "serving-a" + assert config.registration_ttl_seconds == 90 + assert config.lease_ttl_seconds == 60 + assert config.max_transfer_attempts == 4 + assert config.max_replay_chain_length == 17 + assert config.rpc_timeout_seconds == 12.5 + storage = config.object_storage + assert storage.storage_type.value == "S3" + assert storage.initial_base_version_id == "base-a" + assert storage.seed_checkpoint_path == "/models/launch" + assert storage.refit_checkpoint_dir == "/mxdelta/receiver" + assert storage.refit_checkpoint_max_size_gb == 200 + assert storage.endpoint_url == "http://minio:9000" + assert storage.region_name == "us-west-2" + + +@pytest.mark.parametrize( + "key", + [ + "object_storage_type", + "initial_base_version_id", + "seed_checkpoint_path", + "refit_checkpoint_dir", + "object_storage_endpoint_url", + "object_storage_region_name", + ], +) +def test_incomplete_storage_config_rejected_before_init(mx_client, key): + from modelexpress_rl.inference.client import ModelExpressGeneratorClient + + engine = _make_modelexpress_engine() + with pytest.raises(ValueError, match="object storage requires"): + engine.init_transfer_engine(engine.parse_init_info({key: "value"})) + ModelExpressGeneratorClient.initialize.assert_not_called() + + +@pytest.mark.parametrize("version_id", ["", " ", None, 1]) +def test_invalid_version_rejected_before_staging(mx_client, version_id): + engine = _make_modelexpress_engine() + engine.init_transfer_engine(engine.parse_init_info({})) + engine.start_weight_update() + with pytest.raises(ValueError, match="version_id is required"): + engine.update_weights({"version_id": version_id}) + mx_client.stage_weight.assert_not_called() + + +@pytest.mark.parametrize("failure", ["stage", "apply", "release"]) +def test_update_failure_preserves_error_and_releases_handle(mx_client, failure): + engine = _make_modelexpress_engine() + engine.init_transfer_engine(engine.parse_init_info({})) + staged = SimpleNamespace(version_id="version-a", metrics={}, release=MagicMock()) + mx_client.stage_weight.return_value = staged + if failure == "stage": + mx_client.stage_weight.side_effect = RuntimeError("stage failed") + else: + mx_client.apply_weight.side_effect = RuntimeError("apply failed") + if failure == "release": + staged.release.side_effect = RuntimeError("release failed") + engine.start_weight_update() + with pytest.raises(RuntimeError, match="stage failed|apply failed"): + engine.update_weights({"version_id": "version-a"}) + assert staged.release.call_count == (0 if failure == "stage" else 1) + torch.accelerator.synchronize.assert_not_called() + engine.shutdown() + mx_client.close.assert_called_once_with() + + +def test_plugin_does_not_replace_native_backend(mx_client): + from modelexpress.engines.vllm.registration import ( + register_plugin_weight_transfer_engine, + ) + from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import ( + ModelExpressWeightTransferEngine, + ) + + native_loader = WeightTransferEngineFactory._registry["modelexpress"] + register_plugin_weight_transfer_engine() + assert WeightTransferEngineFactory._registry["modelexpress"] is native_loader + assert type(_make_modelexpress_engine()) is ModelExpressWeightTransferEngine diff --git a/vllm/distributed/weight_transfer/factory.py b/vllm/distributed/weight_transfer/factory.py index 82228bef34..c0e88e03bf 100644 --- a/vllm/distributed/weight_transfer/factory.py +++ b/vllm/distributed/weight_transfer/factory.py @@ -250,6 +250,13 @@ WeightTransferEngineFactory.register_engine( ) +WeightTransferEngineFactory.register_engine( + "modelexpress", + "vllm.distributed.weight_transfer.modelexpress_engine", + "ModelExpressWeightTransferEngine", +) + + # Trainer-side engines, parallel to the worker registry above. WeightTransferTrainerFactory.register_engine( "nccl", diff --git a/vllm/distributed/weight_transfer/modelexpress_engine.py b/vllm/distributed/weight_transfer/modelexpress_engine.py new file mode 100644 index 0000000000..1172542e85 --- /dev/null +++ b/vllm/distributed/weight_transfer/modelexpress_engine.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Re-export the weight-transfer backend maintained by ModelExpress. + +Install ModelExpress from the main branch's modelexpress_client/python +subdirectory. +""" + +try: + from modelexpress import configure_vllm_logging + from modelexpress_rl.inference.engines.vllm.weight_transfer_engine import ( + ModelExpressWeightTransferEngine, + ModelExpressWeightTransferInitInfo, + ModelExpressWeightTransferUpdateInfo, + ) +except ModuleNotFoundError as exc: + if exc.name not in { + "modelexpress", + "modelexpress_rl", + "modelexpress_rl.inference", + "modelexpress_rl.inference.engines", + "modelexpress_rl.inference.engines.vllm", + "modelexpress_rl.inference.engines.vllm.weight_transfer_engine", + }: + raise + raise ImportError( + "The 'modelexpress' weight transfer backend requires ModelExpress. " + "Install it with `uv pip install " + "'git+https://github.com/ai-dynamo/modelexpress@main" + "#subdirectory=modelexpress_client/python'`." + ) from exc + +configure_vllm_logging() + +__all__ = [ + "ModelExpressWeightTransferEngine", + "ModelExpressWeightTransferInitInfo", + "ModelExpressWeightTransferUpdateInfo", +] From 9bd993564f5f3d172f9da398fd56b64db1074d9e Mon Sep 17 00:00:00 2001 From: Yoni Gozlan <74535834+yonigozlan@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:33:00 -0400 Subject: [PATCH 235/248] [Frontend] Parse tool calls and reasoning from checkpoint response templates (#58604) Co-authored-by: Cursor Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: Yoni Gozlan Signed-off-by: Bugen Zhao --- docs/features/tool_calling.md | 23 + .../test_batched_chat_completions.py | 60 +- .../chat_completion/test_serving_chat.py | 73 + .../derender/test_derender_stream.py | 39 + tests/parser/test_response_template.py | 1349 +++++++++++++++++ vllm/entrypoints/launchers/cli_args.py | 11 + .../openai/chat_completion/batch_serving.py | 30 +- .../openai/chat_completion/serving.py | 3 + vllm/entrypoints/openai/responses/serving.py | 17 +- vllm/parser/abstract_parser.py | 5 + vllm/parser/parser_manager.py | 40 + vllm/parser/response_template.py | 642 ++++++++ vllm/reasoning/__init__.py | 4 + .../response_template_reasoning_parser.py | 6 + vllm/renderers/online_derenderer.py | 3 + vllm/renderers/online_renderer.py | 4 +- vllm/tool_parsers/__init__.py | 4 + .../response_template_tool_parser.py | 6 + 18 files changed, 2298 insertions(+), 21 deletions(-) create mode 100644 tests/parser/test_response_template.py create mode 100644 vllm/parser/response_template.py create mode 100644 vllm/reasoning/response_template_reasoning_parser.py create mode 100644 vllm/tool_parsers/response_template_tool_parser.py diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index bbbc4d6bcd..8dccd3a8b8 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -171,6 +171,29 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template! +### Checkpoint Response Templates (`hf`) + +Checkpoints can define a `response_template` in `tokenizer_config.json` to describe their reasoning, content, and tool-call wire format. Select the +`hf` parser to parse output from this metadata instead of a model-specific parser. Like `--tokenizer-mode hf`, it relies on the checkpoint's +Hugging Face metadata rather than model-specific code: + +```bash +vllm serve \ + --enable-auto-tool-choice \ + --tool-call-parser hf \ + --reasoning-parser hf +``` + +Either parser can be selected on its own. Startup fails if the tokenizer has no `response_template`, or if it lacks the `thinking` or `tool_calls` +field the selected parser needs. The `hf` parser cannot be combined with other reasoning or tool call parsers. + +When streaming, each tool call is emitted whole once its region parses. A call cut off before its closer, for example by a closer that is also a +stop token, is parsed from the text generated so far; a call that does not parse is dropped. + +Response templates currently provide parsing, not format-specific constrained decoding. The parser therefore rejects requests for strict tools, +required or named tool choice, or `parallel_tool_calls=false`. A custom server or request chat template is allowed, but the parser still expects the +checkpoint's output format. + !!! note With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. diff --git a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py index a6d78c826c..946615f971 100644 --- a/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py +++ b/tests/entrypoints/openai/chat_completion/test_batched_chat_completions.py @@ -246,11 +246,15 @@ async def test_batched_chat_completions_logprob_token_ids( } -def _make_request_output(prompt_idx: int, text: str) -> RequestOutput: +def _make_request_output( + prompt_idx: int, + text: str, + prompt_token_ids: list[int] | None = None, +) -> RequestOutput: return RequestOutput( request_id=f"req-{prompt_idx}", prompt=None, - prompt_token_ids=[1, 2, 3], + prompt_token_ids=prompt_token_ids or [1, 2, 3], prompt_logprobs=None, outputs=[ CompletionOutput( @@ -266,8 +270,12 @@ def _make_request_output(prompt_idx: int, text: str) -> RequestOutput: ) -async def _generator(prompt_idx: int, text: str) -> AsyncGenerator[RequestOutput, None]: - yield _make_request_output(prompt_idx, text) +async def _generator( + prompt_idx: int, + text: str, + prompt_token_ids: list[int] | None = None, +) -> AsyncGenerator[RequestOutput, None]: + yield _make_request_output(prompt_idx, text, prompt_token_ids) @pytest.mark.asyncio @@ -402,3 +410,47 @@ def test_batch_rejects_kv_transfer_prompt_token_ids(): ], kv_transfer_params={"prompt_token_ids": [10, 20, 30]}, ) + + +@pytest.mark.asyncio +@pytest.mark.skip_global_cleanup +async def test_batched_parser_receives_each_prompt_prefix() -> None: + prefixes = [] + + class RecordingParser: + def __init__(self, *args, **kwargs): + self.prompt_token_ids = None + + def set_prompt_token_ids(self, prompt_token_ids): + self.prompt_token_ids = prompt_token_ids + + def parse(self, model_output, **kwargs): + prefixes.append(self.prompt_token_ids) + return None, model_output, None + + serving = OpenAIServingChatBatch.__new__(OpenAIServingChatBatch) + serving.response_role = "assistant" + serving.system_fingerprint = None + serving.chat_template = None + serving.chat_template_content_format = "auto" + serving.default_chat_template_kwargs = {} + + request = BatchChatCompletionRequest( + model="test-model", + messages=[ + [{"role": "user", "content": "first"}], + [{"role": "user", "content": "second"}], + ], + ) + await serving.chat_completion_full_generator_batch( + request=request, + generators=[_generator(0, "one", [1]), _generator(1, "two", [2])], + request_id="req-prefix", + model_name="test-model", + all_conversations=request.messages, + tokenizer=object(), + request_metadata=RequestResponseMetadata(request_id="req-prefix"), + parser_cls=RecordingParser, + ) + + assert prefixes == [[1], [2]] diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 9c80f504f7..bfaa2d4ca0 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -790,6 +790,64 @@ async def test_chat_per_request_metrics_suppressed_for_n_greater_than_one(): assert response.metrics is None +@pytest.mark.asyncio +async def test_chat_full_generator_gives_each_choice_parser_the_prompt(): + prompts: list[list[int] | None] = [] + + class RecordingParser: + tool_parser_cls = None + + def __init__(self, *args, **kwargs): + self.prompt_token_ids = None + + def set_prompt_token_ids(self, prompt_token_ids): + self.prompt_token_ids = list(prompt_token_ids) + + def parse(self, model_output, request, **kwargs): + prompts.append(self.prompt_token_ids) + return None, model_output, None + + def count_reasoning_tokens(self, token_ids): + return 0 + + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False) + serving.parser_cls = RecordingParser + serving.model_config = None + serving.chat_template = None + serving.chat_template_content_format = "auto" + serving.default_chat_template_kwargs = {} + request_output = _make_metrics_request_output() + request_output.prompt_token_ids = [7, 8] + request_output.outputs.append( + CompletionOutput( + index=1, + text="Hi", + token_ids=[102], + cumulative_logprob=None, + logprobs=None, + finish_reason="stop", + ) + ) + + await serving.chat_completion_full_generator( + ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + n=2, + ), + _single_request_output(request_output), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + parser=RecordingParser(), + ) + + assert prompts == [[7, 8], [7, 8]] + + @pytest.mark.asyncio async def test_chat_streaming_metrics_ride_on_usage_chunk(): serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=True) @@ -1608,6 +1666,21 @@ async def test_serving_chat_truncation_side_controls_prompt_truncation(): assert left_token_ids == full_token_ids[-4:] +def test_serving_chat_starts_without_tokenizer(): + """Regression test: building the serving objects must not require a + tokenizer, which is absent under `skip_tokenizer_init`.""" + mock_engine = _build_mock_engine() + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True) + mock_engine.renderer = HfRenderer( + MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()), + None, + ) + + serving_chat = _build_serving_chat(mock_engine) + + assert serving_chat.parser_cls is None + + @pytest.mark.asyncio async def test_serving_chat_mistral_token_ids_prompt_is_validated(): """Regression test: when the Mistral tokenizer path returns token IDs diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index 63ee9bcad8..b1db1193a2 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -296,6 +296,45 @@ def parsed_derenderer(tokenizer): return dr +def test_non_streaming_derender_initializes_parser_prefix( + derenderer, tokenizer, monkeypatch +): + from unittest.mock import MagicMock + + from vllm.entrypoints.generate.base.protocol import FunctionCall + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + + parser = MagicMock() + parser.parse.return_value = ( + None, + None, + [FunctionCall(name="tool", arguments="{}")], + ) + monkeypatch.setattr(derenderer, "parser", MagicMock(return_value=parser)) + generated_ids = tokenizer.encode("answer", add_special_tokens=False) + response = GenerateResponse( + request_id="test", + prompt_token_ids=[11, 12], + choices=[ + GenerateResponseChoice( + index=0, + token_ids=generated_ids, + finish_reason="length", + ) + ], + ) + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[{"role": "user", "content": "question"}], + ) + + choices = derenderer._derender_chat(response, request) + + parser.set_prompt_token_ids.assert_called_once_with([11, 12]) + assert choices[0].message.tool_calls[0].function.name == "tool" + assert choices[0].finish_reason == "length" + + class TestDetokenizeDelta: """_detokenize_delta: chunked decode must equal one shot decode.""" diff --git a/tests/parser/test_response_template.py b/tests/parser/test_response_template.py new file mode 100644 index 0000000000..84b62c5512 --- /dev/null +++ b/tests/parser/test_response_template.py @@ -0,0 +1,1349 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +from copy import deepcopy +from types import SimpleNamespace +from typing import Any + +import pytest +from openai.types.responses import NamespaceTool + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, + ChatCompletionToolsParam, +) +from vllm.exceptions import VLLMValidationError +from vllm.parser import response_template as response_template_module +from vllm.parser.gemma4 import Gemma4Parser +from vllm.parser.parser_manager import ParserManager +from vllm.parser.response_template import ( + ResponseTemplateParser, + ResponseTemplateReasoningParser, + ResponseTemplateToolParser, + resolve_response_template, +) +from vllm.reasoning import ReasoningParserManager +from vllm.tool_parsers import ToolParserManager + +# Public metadata shared by the Gemma 4 instruction checkpoints. +GEMMA4_RESPONSE_TEMPLATE: dict[str, Any] = { + "defaults": {"role": "assistant"}, + "fields": { + "content": { + "close": ["", "<|tool_response>", ""], + "content": "text", + }, + "thinking": { + "close": "", + "content": "text", + "open": "<|channel>thought\n", + }, + "tool_calls": { + "close": "", + "content": "json", + "content_args": { + "string_delims": [['<|"|>', '<|"|>']], + "unquoted_keys": True, + }, + "open_pattern": r"<\|tool_call>call:(?P\w+)", + "repeats": True, + "transform": { + "function": { + "arguments": "{content}", + "name": "{name}", + }, + "type": "function", + }, + }, + }, + "start_anchor": ["<|turn>model\n", ""], +} + +PREFIX = "<|turn>model\n" +THINKING = "<|channel>thought\nplan" +TOOL_GENERATION = ( + '<|tool_call>call:set_alarm{hour:7,label:<|"|>morning<|"|>}' +) + +TOOLS = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "set_alarm", + "parameters": { + "type": "object", + "properties": { + "hour": { + "oneOf": [ + {"type": "integer"}, + {"type": "null"}, + ] + }, + "label": {"type": "string"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": {"type": "string"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "set_status", + "parameters": { + "type": "object", + "properties": { + "is_active": {"type": "boolean"}, + "count": {"type": "integer"}, + "score": {"type": "number"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "complex_function", + "parameters": { + "type": "object", + "properties": { + "nested": { + "type": "object", + "properties": {"inner": {"type": "string"}}, + }, + "list": { + "type": "array", + "items": {"type": "string"}, + }, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "configure", + "parameters": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "ratio": {"type": "number"}, + "label": {"type": "string"}, + "value": {"type": ["string", "null"]}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_status", + "parameters": {"type": "object", "properties": {}}, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "get-weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "search", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "filters": { + "type": "object", + "properties": { + "language": {"type": "string"}, + "min_stars": {"type": "integer"}, + }, + }, + }, + }, + }, + ), +] + +VOCAB = { + "<|tool_call>": 48, + "": 49, + "<|channel>": 50, + "": 51, + '<|"|>': 52, + "<|turn>": 53, + "<|tool_response>": 54, + "": 55, +} +PROMPT_TOKEN_IDS = [1] +OPEN_REASONING_PROMPT_TOKEN_IDS = [53, 100, 50, 101] +PROMPT_TOKEN_TEXT = { + 100: "model\n", + 101: "thought\n", + 102: "plan", + 103: "answer", +} + + +class FakeTokenizer: + def __init__(self, response_template=None, *, prefix=PREFIX): + self.response_template = response_template + self.init_kwargs = {} + self.name_or_path = "" + self.prefix = prefix + self.all_special_tokens = list(VOCAB) + self.all_special_ids = list(VOCAB.values()) + + def decode(self, token_ids, skip_special_tokens=False, **_kwargs): + if token_ids == [2]: + return PREFIX + "answer" + if token_ids == OPEN_REASONING_PROMPT_TOKEN_IDS: + return PREFIX + "<|channel>thought\n" + if token_ids != PROMPT_TOKEN_IDS: + inverse_vocab = {token_id: token for token, token_id in VOCAB.items()} + return "".join( + PROMPT_TOKEN_TEXT.get(token_id, inverse_vocab.get(token_id, "")) + for token_id in token_ids + ) + return self.prefix if token_ids else "" + + def get_vocab(self): + return VOCAB + + def encode(self, text, add_special_tokens=False): + del add_special_tokens + if text == "plan": + return [102] + return list(text.encode()) + + +def request(tool_choice="auto", *, include_reasoning=True, **overrides): + values = dict( + tool_choice=tool_choice, + include_reasoning=include_reasoning, + tools=TOOLS, + skip_special_tokens=True, + spaces_between_special_tokens=True, + include_stop_str_in_output=False, + parallel_tool_calls=True, + ) + values.update(overrides) + return SimpleNamespace(**values) + + +def normalize_calls(calls): + return [(call.name, json.loads(call.arguments)) for call in calls or []] + + +def feed_chunks( + parser, + chunks, + req, + *, + prompt_token_ids=PROMPT_TOKEN_IDS, +): + deltas = [] + for index, chunk in enumerate(chunks): + delta = parser.parse_delta( + chunk, + [], + req, + prompt_token_ids=prompt_token_ids if index == 0 else None, + finished=index == len(chunks) - 1, + ) + if delta is not None: + deltas.append(delta) + return deltas + + +def collect_stream( + parser, + text, + req, + chunk_size, + *, + prompt_token_ids=PROMPT_TOKEN_IDS, +): + reasoning = [] + content = [] + calls: dict[int, dict[str, str]] = {} + for start in range(0, len(text), chunk_size): + end = min(start + chunk_size, len(text)) + delta = parser.parse_delta( + text[start:end], + [], + req, + prompt_token_ids=prompt_token_ids if start == 0 else None, + finished=end == len(text), + ) + if delta is None: + continue + if delta.reasoning: + reasoning.append(delta.reasoning) + if delta.content: + content.append(delta.content) + for call in delta.tool_calls: + function = call.function + assert function is not None + slot = calls.setdefault( + call.index, + {"name": "", "arguments": ""}, + ) + slot["name"] += function.name or "" + slot["arguments"] += function.arguments or "" + normalized_calls = [ + (call["name"], json.loads(call["arguments"])) + for _, call in sorted(calls.items()) + ] + return "".join(reasoning), "".join(content), normalized_calls + + +def test_resolve_response_template_from_tokenizer_configuration(): + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE) + assert resolve_response_template(tokenizer) == GEMMA4_RESPONSE_TEMPLATE + + tokenizer = FakeTokenizer() + del tokenizer.response_template + tokenizer.init_kwargs["response_template"] = GEMMA4_RESPONSE_TEMPLATE + assert resolve_response_template(tokenizer) == GEMMA4_RESPONSE_TEMPLATE + + explicit = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + explicit["defaults"]["role"] = "custom" + assert resolve_response_template(tokenizer, explicit) is explicit + + +def test_response_template_parsers_are_registered(): + assert ToolParserManager.get_tool_parser("hf") is ResponseTemplateToolParser + assert ( + ReasoningParserManager.get_reasoning_parser("hf") + is ResponseTemplateReasoningParser + ) + + +@pytest.mark.parametrize( + ("reasoning", "tools"), + [(True, True), (True, False), (False, True)], + ids=["both", "reasoning", "tools"], +) +def test_parser_manager_selects_response_template_by_name(reasoning, tools): + parser_cls = ParserManager.get_parser( + tool_parser_name="hf" if tools else None, + reasoning_parser_name="hf" if reasoning else None, + enable_auto_tools=tools, + tokenizer=FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + ) + + assert parser_cls is not None + assert issubclass(parser_cls, ResponseTemplateParser) + assert (parser_cls.reasoning_parser_cls is not None) is reasoning + assert (parser_cls.tool_parser_cls is not None) is tools + + +def test_parser_manager_validates_checkpoint_metadata_at_startup(): + with pytest.raises(TypeError, match="requires `response_template` metadata"): + ParserManager.get_parser( + reasoning_parser_name="hf", + tokenizer=FakeTokenizer(), + ) + with pytest.raises(TypeError, match="Invalid response_template"): + ParserManager.get_parser( + reasoning_parser_name="hf", + tokenizer=FakeTokenizer({"fields": {}}), + ) + + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + del template["fields"]["tool_calls"] + with pytest.raises(TypeError, match="tool_calls field"): + ParserManager.get_parser( + tool_parser_name="hf", + enable_auto_tools=True, + tokenizer=FakeTokenizer(template), + ) + assert ( + ParserManager.get_parser( + reasoning_parser_name="hf", + tokenizer=FakeTokenizer(template), + ) + is not None + ) + + +def test_response_template_parser_rejects_other_parsers(): + with pytest.raises(TypeError, match="cannot be combined"): + ParserManager.get_parser( + tool_parser_name="hermes", + reasoning_parser_name="hf", + enable_auto_tools=True, + tokenizer=FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + ) + + +def test_request_chat_template_is_honored_with_warning(monkeypatch): + warnings: list[str] = [] + monkeypatch.setattr( + response_template_module.logger, "warning_once", warnings.append + ) + parser = ResponseTemplateParser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + + parser.adjust_request(request(chat_template="{{ messages }}")) + + assert len(warnings) == 1 + assert parser.response_template.fields + + +def test_reasoning_gate_waits_for_content_region(): + parser = ResponseTemplateParser( + FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|channel>thought\n", + ), + TOOLS, + ) + + assert parser.is_reasoning_end([1]) is False + assert parser.is_reasoning_end([2]) is True + + +def test_reasoning_gate_ends_at_content_opener_held_by_prompt(): + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"]["content"]["open_pattern"] = r"<\|channel>final\n" + parser = ResponseTemplateParser( + FakeTokenizer(template, prefix=PREFIX + "<|channel>final\n"), + TOOLS, + ) + + assert parser.is_reasoning_end([1]) is True + + +def streamed_reasoning_end(reasoner, output, prompt=PROMPT_TOKEN_IDS): + """Index of the output token where the per-step gate reports the end.""" + for index, token in enumerate(output): + if reasoner.is_reasoning_end_streaming(prompt + output[: index + 1], [token]): + return index + return None + + +def test_streaming_reasoning_gate_matches_thinking_closer_without_decoding(): + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE) + reasoner = ResponseTemplateReasoningParser(tokenizer) + tokenizer.decode = lambda *_args, **_kwargs: pytest.fail("decoded") # type: ignore[method-assign] + output = [50, 101] + [102] * 100 + [51, 103] + + assert streamed_reasoning_end(reasoner, output) == len(output) - 2 + assert reasoner.is_reasoning_end_streaming(PROMPT_TOKEN_IDS + output, output[-3:]) + + +def test_reasoning_gate_honors_disabled_and_repeated_thinking(): + disabled = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + chat_template_kwargs={"enable_thinking": False}, + ) + assert disabled.is_reasoning_end([1]) is True + + repeated_template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + repeated_template["fields"]["thinking"]["repeats"] = True + repeated = ResponseTemplateParser( + FakeTokenizer(repeated_template), + TOOLS, + response_template=repeated_template, + ) + assert repeated.is_reasoning_end(OPEN_REASONING_PROMPT_TOKEN_IDS) is False + assert repeated.is_reasoning_end([2]) is False + + +@pytest.mark.parametrize( + "text", + [ + "Hello", + THINKING + "The answer is 42.", + THINKING + TOOL_GENERATION, + "Let me check." + TOOL_GENERATION, + ( + '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>,' + 'unit:<|"|>celsius<|"|>}' + TOOL_GENERATION + ), + ( + "<|tool_call>call:complex_function{" + 'nested:{inner:<|"|>value<|"|>},' + 'list:[<|"|>a<|"|>,<|"|>b<|"|>]}' + "" + ), + ], + ids=[ + "content", + "reasoning", + "tool", + "content_then_tool", + "multiple_tools", + "nested_arguments", + ], +) +def test_gemma4_response_template_matches_registered_parser(text): + tokenizer = FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE) + req = request() + registered = Gemma4Parser(tokenizer, TOOLS) + metadata = ResponseTemplateParser( + tokenizer, + TOOLS, + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + metadata.set_prompt_token_ids([1]) + + expected_reasoning, expected_content, expected_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + reasoning, content, calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert reasoning == expected_reasoning + assert content == expected_content + assert normalize_calls(calls) == normalize_calls(expected_calls) + + +def test_metadata_parser_keeps_already_decoded_nullable_values(): + text = ( + "<|tool_call>call:configure{" + "enabled:false,ratio:5.0,label:null,value:null}" + "" + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + response_template=GEMMA4_RESPONSE_TEMPLATE, + ) + parser.set_prompt_token_ids([1]) + + _, content, calls = parser.parse( + text, + request(), + enable_auto_tools=True, + ) + + assert content is None + assert normalize_calls(calls) == [ + ( + "configure", + { + "enabled": False, + "ratio": 5.0, + "label": None, + "value": None, + }, + ) + ] + + +@pytest.mark.parametrize( + "text", + [ + THINKING + TOOL_GENERATION, + THINKING + "The answer is 42.", + "Let me check. " + TOOL_GENERATION, + ( + '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}' + + TOOL_GENERATION + ), + ], + ids=[ + "reasoning_then_tool", + "reasoning_then_content", + "content_then_tool", + "multiple_tools", + ], +) +@pytest.mark.parametrize("chunk_size", [1, 7, 10_000]) +def test_gemma4_response_template_streaming_matches_registered_parser( + text, + chunk_size, +): + req = request() + registered = Gemma4Parser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + assert collect_stream(metadata, text, req, chunk_size) == collect_stream( + registered, + text, + req, + chunk_size, + ) + + +@pytest.mark.parametrize( + ("prompt_token_ids", "text"), + [ + ( + OPEN_REASONING_PROMPT_TOKEN_IDS, + "continued reasoningFinal answer", + ), + ( + PROMPT_TOKEN_IDS, + "Direct answer without reasoning.", + ), + ], + ids=["prefilled_reasoning", "direct_answer"], +) +def test_gemma4_response_template_prompt_state_matches_registered_parser( + prompt_token_ids, + text, +): + req = request() + registered = Gemma4Parser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + chat_template_kwargs={"enable_thinking": True}, + ) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + assert collect_stream( + metadata, + text, + req, + 1, + prompt_token_ids=prompt_token_ids, + ) == collect_stream( + registered, + text, + req, + 1, + prompt_token_ids=prompt_token_ids, + ) + + +def test_gemma4_response_template_reasoning_suppression_matches_registered_parser(): + req = request(include_reasoning=False) + text = THINKING + "Final answer" + + assert collect_stream( + ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ), + text, + req, + 1, + ) == collect_stream( + Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS), + text, + req, + 1, + ) + + +def test_streamed_tool_call_is_emitted_when_its_region_parses(): + chunks = [ + "<|tool_call>call:set_alarm{", + "hour:7,", + 'label:<|"|>morning<|"|>}', + "", + ] + deltas = feed_chunks( + ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ), + chunks, + request(), + ) + + assert all(not delta.tool_calls for delta in deltas[:-1]) + calls = deltas[-1].tool_calls + assert len(calls) == 1 + assert calls[0].id is not None + assert calls[0].index == 0 + assert calls[0].function is not None + assert calls[0].function.name == "set_alarm" + assert json.loads(calls[0].function.arguments) == { + "hour": 7, + "label": "morning", + } + + +def test_response_template_emits_whole_call_from_one_chunk(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + delta = parser.parse_delta( + TOOL_GENERATION, + [], + request(), + prompt_token_ids=PROMPT_TOKEN_IDS, + finished=True, + ) + + assert delta is not None + assert len(delta.tool_calls) == 1 + call = delta.tool_calls[0] + assert call.id is not None + assert call.function is not None + assert call.function.name == "set_alarm" + assert json.loads(call.function.arguments) == {"hour": 7, "label": "morning"} + + +def test_response_template_takes_name_from_region_content(): + template = { + "defaults": {"role": "assistant"}, + "start_anchor": PREFIX, + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform": { + "type": "function", + "function": { + "name": "{content.name}", + "arguments": "{content.arguments}", + }, + }, + } + }, + } + chunks = [ + "", + '{"name":"set_', + 'alarm","arguments":{"hour":7}}', + "", + ] + parser = ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + enable_auto_tools=True, + ) + + deltas = feed_chunks(parser, chunks, request()) + + assert all(not delta.tool_calls for delta in deltas[:-1]) + assert len(deltas[-1].tool_calls) == 1 + call = deltas[-1].tool_calls[0] + assert call.function is not None + assert call.function.name == "set_alarm" + assert json.loads(call.function.arguments) == {"hour": 7} + + +def test_response_template_applies_literal_transformed_name(): + template = { + "defaults": {"role": "assistant"}, + "start_anchor": PREFIX, + "fields": { + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "transform": { + "type": "function", + "function": { + "name": "set_alarm", + "arguments": "{content}", + }, + }, + } + }, + } + parser = ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + enable_auto_tools=True, + ) + + deltas = feed_chunks( + parser, + ["", '{"hour":7}', ""], + request(), + ) + + assert len(deltas) == 1 + assert deltas[0].tool_calls[0].function.name == "set_alarm" + assert json.loads(deltas[0].tool_calls[0].function.arguments) == {"hour": 7} + + +def test_gemma4_response_template_drops_incomplete_call(): + text = '<|tool_call>call:get_weather{city:<|"|>London' + req = request() + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS) + + _, _, registered_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + _, metadata_content, metadata_calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert normalize_calls(registered_calls) == [("get_weather", {"city": "London"})] + assert metadata_content is None + assert metadata_calls is None + + +def test_gemma4_response_template_preserves_content_after_tool_call(): + text = TOOL_GENERATION + "Done." + req = request() + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS) + + _, registered_content, registered_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + _, metadata_content, metadata_calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert registered_content is None + assert metadata_content == "Done." + assert normalize_calls(metadata_calls) == normalize_calls(registered_calls) + + +def test_gemma4_checkpoint_grammar_excludes_hyphenated_name(): + text = '<|tool_call>call:get-weather{city:<|"|>London<|"|>}' + req = request() + registered = Gemma4Parser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + metadata = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + metadata.set_prompt_token_ids(PROMPT_TOKEN_IDS) + + _, _, registered_calls = registered.parse( + text, + req, + enable_auto_tools=True, + ) + _, metadata_content, metadata_calls = metadata.parse( + text, + req, + enable_auto_tools=True, + ) + + assert normalize_calls(registered_calls) == [("get-weather", {"city": "London"})] + assert metadata_content is None + assert metadata_calls is None + + +@pytest.mark.parametrize( + ( + "prefix_suffix", + "generated", + "enable_auto_tools", + "expected_reasoning", + "expected_content", + ), + [ + ( + "<|channel>thought\nprefill", + " generated", + False, + " generated", + None, + ), + ("prefill ", "generated", False, None, "generated"), + ("prefill", "", False, None, None), + (TOOL_GENERATION.split("{", 1)[0], "", True, None, None), + ("<|chan", "", False, None, None), + ("<|chan", "nel", False, None, "nel"), + ( + "<|chan", + "nel>thought\nreason", + False, + "reason", + None, + ), + ], + ids=[ + "reasoning", + "content", + "empty_content", + "empty_tool_open", + "empty_partial_delimiter", + "unresolved_partial_delimiter", + "completed_partial_delimiter", + ], +) +def test_streaming_prefill_is_state_only( + prefix_suffix, + generated, + enable_auto_tools, + expected_reasoning, + expected_content, +): + tokenizer = FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + prefix_suffix, + ) + parser = ResponseTemplateParser( + tokenizer, + TOOLS, + enable_auto_tools=enable_auto_tools, + ) + + delta = parser.parse_delta( + generated, + [], + request(), + prompt_token_ids=[1], + finished=True, + ) + + if expected_reasoning is None and expected_content is None: + assert delta is None + else: + assert delta is not None + assert delta.reasoning == expected_reasoning + assert delta.content == expected_content + + +def test_malformed_call_opened_across_prompt_boundary_is_dropped(): + parser = ResponseTemplateParser( + FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + "<|tool_call>call:set_", + ), + TOOLS, + enable_auto_tools=True, + ) + + opened = parser.parse_delta( + "alarm{", + [], + request(), + prompt_token_ids=[1], + finished=False, + ) + delta = parser.parse_delta( + "hour:7}unexpected", + [], + request(), + finished=True, + ) + + assert opened is None + assert delta is None + + +def test_responses_namespace_tools_use_vllm_flattened_names(): + namespace = NamespaceTool( + type="namespace", + name="calendar", + description="Calendar tools", + tools=[ + { + "type": "function", + "name": "set_alarm", + "description": "Set an alarm", + "parameters": TOOLS[0].function.parameters, + } + ], + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + [namespace], + ) + parser.set_prompt_token_ids([1]) + output = TOOL_GENERATION.replace( + "call:set_alarm", + "call:calendar__set_alarm", + ) + + _, content, calls = parser.parse( + output, + request(), + enable_auto_tools=True, + ) + + assert content is None + assert normalize_calls(calls) == [ + ("calendar__set_alarm", {"hour": 7, "label": "morning"}) + ] + + +def test_streaming_unknown_call_is_forwarded(): + text = TOOL_GENERATION.replace("call:set_alarm", "call:unknown") + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + _, content, calls = collect_stream(parser, text, request(), 7) + + assert content == "" + assert calls == [("unknown", {"hour": 7, "label": "morning"})] + + +@pytest.mark.parametrize( + "text", + [ + TOOL_GENERATION.replace( + "", + "unexpected", + ), + TOOL_GENERATION.removesuffix("") + "", "unexpected") + + _, content, calls = collect_stream( + parser, malformed + TOOL_GENERATION, request(), 7 + ) + + assert content == "" + assert calls == [("set_alarm", {"hour": 7, "label": "morning"})] + + +def test_non_streaming_malformed_call_is_dropped(): + text = TOOL_GENERATION.replace( + "", + "unexpected", + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + parser.set_prompt_token_ids([1]) + + reasoning, content, calls = parser.parse( + text, + request(), + enable_auto_tools=True, + ) + + assert reasoning is None + assert content is None + assert calls is None + + +def test_streaming_malformed_call_is_dropped_before_emission(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + first = parser.parse_delta( + "hello", + [], + request(), + prompt_token_ids=[1], + finished=False, + ) + malformed = TOOL_GENERATION.replace( + "", + "unexpected", + ) + failed = parser.parse_delta( + malformed, + [], + request(), + finished=False, + ) + + assert first is not None + assert first.content == "hello" + assert failed is None + + +def test_prefilled_malformed_call_is_dropped(): + opening, body = TOOL_GENERATION.split("{", 1) + malformed_body = "{" + body.replace( + "", + "unexpected", + ) + tokenizer = FakeTokenizer( + GEMMA4_RESPONSE_TEMPLATE, + prefix=PREFIX + opening, + ) + parser = ResponseTemplateParser( + tokenizer, + TOOLS, + enable_auto_tools=True, + ) + + delta = parser.parse_delta( + malformed_body, + [], + request(), + prompt_token_ids=[1], + finished=True, + ) + + assert delta is None + + +def test_tool_choice_none_preserves_raw_call_as_content(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + _, content, calls = collect_stream( + parser, + TOOL_GENERATION, + request("none"), + 5, + ) + + assert content == TOOL_GENERATION + assert calls == [] + + +def test_adjust_request_preserves_parser_delimiters_without_forcing_stop_text(): + req = request() + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + ) + + assert parser.adjust_request(req) is req + assert req.skip_special_tokens is False + assert req.spaces_between_special_tokens is False + assert req.include_stop_str_in_output is False + + +def test_call_cut_off_before_its_closer_is_parsed(): + """A stop-token closer, such as a tool closer that ends generation, is + absent from the text; the open region is parsed at the end of the stream.""" + trimmed = TOOL_GENERATION.removesuffix("") + parser = ResponseTemplateParser(FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), TOOLS) + streaming = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + + _, content, calls = parser.parse(trimmed, request(), enable_auto_tools=True) + streamed = streaming.parse_delta( + trimmed, + [], + request(), + prompt_token_ids=[1], + finished=True, + ) + + assert content is None + assert normalize_calls(calls) == [("set_alarm", {"hour": 7, "label": "morning"})] + assert streamed is not None + assert [call.function.name for call in streamed.tool_calls] == ["set_alarm"] + + +@pytest.mark.parametrize( + ("req", "match"), + [ + (request("required"), "required tool choice"), + ( + request( + ChatCompletionNamedToolChoiceParam( + type="function", + function={"name": "set_alarm"}, + ) + ), + "named tool choice", + ), + (request("auto", parallel_tool_calls=False), "parallel_tool_calls=False"), + ( + request( + "auto", + tools=[ + { + "type": "function", + "name": "set_alarm", + "parameters": {"type": "object"}, + "strict": True, + } + ], + ), + "strict tools", + ), + ], +) +def test_unconstrained_tool_guarantees_are_rejected(req, match): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + req.tools, + enable_auto_tools=True, + ) + + with pytest.raises(VLLMValidationError, match=match): + parser.adjust_request(req) + + +def test_tool_choice_none_does_not_require_structural_guarantees(): + req = request( + "none", + tools=[ + { + "type": "function", + "name": "set_alarm", + "parameters": {"type": "object"}, + "strict": True, + } + ], + parallel_tool_calls=False, + ) + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + req.tools, + enable_auto_tools=True, + ) + + assert parser.adjust_request(req) is req + + +def test_unsupported_semantic_field_is_rejected(): + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"]["citations"] = { + "open": "", + "close": "", + } + + with pytest.raises(ValueError, match="citations"): + ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + ) + + +@pytest.mark.parametrize( + ("field_name", "field_update"), + [ + ("content", {"content": "json"}), + ("thinking", {"transform": "{content}"}), + ], +) +def test_unsupported_content_field_semantics_are_rejected( + field_name, + field_update, +): + template = deepcopy(GEMMA4_RESPONSE_TEMPLATE) + template["fields"][field_name].update(field_update) + + with pytest.raises(ValueError, match="cannot be streamed"): + ResponseTemplateParser( + FakeTokenizer(template), + TOOLS, + response_template=template, + ) + + +def test_delimited_template_keeps_plain_content_and_tolerates_required_fields(): + template = { + "start_anchor": "", + "fields": { + "thinking": {"open": "", "close": ""}, + "tool_calls": { + "open": "", + "close": "", + "content": "json", + "optional": False, + }, + }, + } + parser = ResponseTemplateParser( + FakeTokenizer(template, prefix=""), + TOOLS, + response_template=template, + ) + + assert parser.parse("plananswer", request()) == ( + "plan", + "answer", + None, + ) + + +def test_separate_parse_calls_use_distinct_tool_call_ids(): + parser = ResponseTemplateParser( + FakeTokenizer(GEMMA4_RESPONSE_TEMPLATE), + TOOLS, + enable_auto_tools=True, + ) + parsed_calls = [] + for _ in range(2): + _, _, calls = parser.parse( + TOOL_GENERATION, + request(), + enable_auto_tools=True, + ) + assert calls is not None + parsed_calls.append(calls[0]) + + assert parsed_calls[0].id is not None + assert parsed_calls[1].id is not None + assert parsed_calls[0].id != parsed_calls[1].id diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 3a52cf2e2b..a800e1270d 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -22,6 +22,7 @@ from vllm.entrypoints.chat_utils import ( validate_chat_template, ) from vllm.entrypoints.openai.models.protocol import LoRAModulePath +from vllm.logger import init_logger from vllm.tool_parsers import ToolParserManager from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName from vllm.utils.argparse_utils import FlexibleArgumentParser @@ -31,6 +32,8 @@ from .utils.constants import ( H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT, ) +logger = init_logger(__name__) + class LoRAParserAction(argparse.Action): def __call__( @@ -466,6 +469,14 @@ def validate_parsed_serve_args(args: argparse.Namespace): # Ensure that the chat template is valid; raises if it likely isn't validate_chat_template(args.chat_template) + if args.chat_template is not None and "hf" in ( + args.tool_call_parser, + getattr(args, "reasoning_parser", None), + ): + logger.warning( + "--chat-template is set; the hf parser still expects " + "the checkpoint's output format." + ) # Enable auto tool needs a tool call parser to be valid if args.enable_auto_tool_choice and not args.tool_call_parser: diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index 3fde958a3d..8c7ae37647 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -129,17 +129,6 @@ class OpenAIServingChatBatch(OpenAIServingChat): return render_result single_requests, all_conversations, engine_prompts = render_result - parser: Parser | None = None - if self.parser_cls is not None: - chat_template_kwargs = self._effective_chat_template_kwargs( - single_requests[0] - ) - parser = self.parser_cls( - tokenizer, - None, # tools - chat_template_kwargs=chat_template_kwargs, - ) - request_id = ( f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}" ) @@ -202,7 +191,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): all_conversations, tokenizer, request_metadata, - parser, + self.parser_cls, ) async def chat_completion_full_generator_batch( @@ -214,7 +203,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): all_conversations: list[list[ConversationMessage]], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, - parser: Parser | None = None, + parser_cls: type[Parser] | None = None, ) -> ErrorResponse | ChatCompletionResponse: """Handle batched (non-streaming) chat completions. @@ -273,10 +262,21 @@ class OpenAIServingChatBatch(OpenAIServingChat): else: logprobs = None - if parser is not None: + if parser_cls is not None: + single_request = request.to_chat_completion_request( + request.messages[prompt_idx] + ) + parser = parser_cls( + tokenizer, + None, + chat_template_kwargs=self._effective_chat_template_kwargs( + single_request + ), + ) + parser.set_prompt_token_ids(final_res.prompt_token_ids) reasoning, content, _ = parser.parse( output.text, - request=request, # type: ignore[arg-type] + request=single_request, model_output_token_ids=output.token_ids, ) if not request.include_reasoning: diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index f22fed1e75..060d22b031 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -165,6 +165,7 @@ class OpenAIServingChat(GenerateBaseServing): tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", + tokenizer=self.renderer.tokenizer, ) self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none @@ -974,6 +975,8 @@ class OpenAIServingChat(GenerateBaseServing): parser = self._make_parser( request, tokenizer, self._effective_chat_template_kwargs(request) ) + if parser is not None and final_res.prompt_token_ids is not None: + parser.set_prompt_token_ids(final_res.prompt_token_ids) token_ids = output.token_ids out_logprobs = output.logprobs diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 66e0f2640d..bd876e4916 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -141,6 +141,7 @@ class OpenAIServingResponses(GenerateBaseServing): tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", + tokenizer=self.renderer.tokenizer, ) self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage @@ -446,6 +447,12 @@ class OpenAIServingResponses(GenerateBaseServing): response_parser = self._make_response_parser( request, tokenizer, chat_template_kwargs ) + if response_parser is not None: + prompt_token_ids = self._extract_prompt_components( + engine_input + ).token_ids + if prompt_token_ids is not None: + response_parser.set_prompt_token_ids(prompt_token_ids) context: ConversationContext function_tool_names = extract_function_tool_names(request.tools) @@ -703,6 +710,12 @@ class OpenAIServingResponses(GenerateBaseServing): context.request, context.response_messages, ) + if context.response_parser is not None: + prompt_token_ids = self._extract_prompt_components( + engine_input + ).token_ids + if prompt_token_ids is not None: + context.response_parser.set_prompt_token_ids(prompt_token_ids) sampling_params.max_tokens = get_max_tokens( max_model_len, @@ -1187,7 +1200,9 @@ class OpenAIServingResponses(GenerateBaseServing): ) -> AsyncGenerator[StreamingResponsesResponse, None]: processor = SimpleStreamingEventProcessor(tools=request.tools) - hide_stream_metadata = not request.include_reasoning and self.parser is not None + hide_stream_metadata = ( + not request.include_reasoning and context.response_parser is not None + ) def _get_logprobs(output: CompletionOutput) -> list[Logprob]: if not request.is_include_output_logprobs(): diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 2a348697c7..eea957f90c 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -115,6 +115,7 @@ class Parser: tool_parser_cls: type[ToolParser] | None = None # Server-side floor for tool-call structural tags (--tool-strict-level). tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO + always_adjust_request: bool = False def __init__( self, @@ -206,6 +207,10 @@ class Parser: """ return request + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + """Provide the exact rendered prompt to parsers that need prefix state.""" + return + @abstractmethod def is_reasoning_end(self, input_ids: list[int]) -> bool: """Check if the reasoning content ends in the input_ids. diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index 41b27967a6..719b396130 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -11,10 +11,13 @@ from vllm.tool_parsers.tool_strict_level import ToolStrictLevel if TYPE_CHECKING: from vllm.parser.abstract_parser import Parser from vllm.reasoning import ReasoningParser + from vllm.tokenizers import TokenizerLike from vllm.tool_parsers import ToolParser logger = init_logger(__name__) +HF_PARSER = "hf" + class ParserManager: """Provides a unified Parser by composing reasoning and tool parser adapters.""" @@ -79,6 +82,7 @@ class ParserManager: model_name: str | None = None, is_harmony: bool = False, tool_strict_level: str = "auto", + tokenizer: TokenizerLike | None = None, ) -> type[Parser] | None: """Get a Parser that handles both reasoning and tool parsing. @@ -93,6 +97,8 @@ class ParserManager: If True, HarmonyParser is always returned. tool_strict_level: Server-side floor for tool-call structural tags (``--tool-strict-level``). + tokenizer: Tokenizer whose `response_template` metadata is + validated when the `hf` parser is selected. Returns: A Parser class, or None if neither parser is specified. @@ -119,6 +125,40 @@ class ParserManager: HarmonyParser.tool_strict_level = strict_level return HarmonyParser + if HF_PARSER in (reasoning_parser_name, tool_parser_name): + if {reasoning_parser_name, tool_parser_name} - { + HF_PARSER, + None, + "", + }: + raise TypeError( + "The hf parser cannot be combined with other " + "reasoning or tool call parsers" + ) + from vllm.parser.response_template import ( + ResponseTemplateParser, + validate_tokenizer_response_template, + ) + + if tokenizer is not None: + validate_tokenizer_response_template( + tokenizer, + reasoning=reasoning_parser_cls is not None, + tools=tool_parser_cls is not None, + ) + + r_cls = reasoning_parser_cls + t_cls = tool_parser_cls + auto_tools = enable_auto_tools + + class _ResponseTemplateParser(ResponseTemplateParser): + reasoning_parser_cls = r_cls + tool_parser_cls = t_cls + tool_strict_level = strict_level + _enable_auto_tools = auto_tools + + return _ResponseTemplateParser + if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3": from vllm.parser.kimi_k3 import KimiK3Parser diff --git a/vllm/parser/response_template.py b/vllm/parser/response_template.py new file mode 100644 index 0000000000..985b76308a --- /dev/null +++ b/vllm/parser/response_template.py @@ -0,0 +1,642 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unified vLLM parser backed by checkpoint `response_template` metadata.""" + +from __future__ import annotations + +import copy +import json +from collections.abc import Callable, Iterable, Sequence +from typing import TYPE_CHECKING, Any, cast + +from openai.types.responses import ToolChoiceFunction + +from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.entrypoints.generate.base.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + FunctionCall, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, +) +from vllm.exceptions import VLLMValidationError +from vllm.logger import init_logger +from vllm.parser.abstract_parser import Parser +from vllm.parser.chat_parsing import ResponseParser +from vllm.parser.chat_parsing.response_templates import ( + ResponseTemplate, + load_response_template, +) +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser +from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.utils import iter_response_function_tool_dicts + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +logger = init_logger(__name__) + +THINKING_FIELD = "thinking" +CONTENT_FIELD = "content" +TOOL_FIELD = "tool_calls" +_SUPPORTED_FIELDS = frozenset({THINKING_FIELD, CONTENT_FIELD, TOOL_FIELD}) + + +def _tool_dict(tool: Any) -> dict[str, Any]: + if hasattr(tool, "model_dump"): + tool = tool.model_dump(exclude_none=True) + return tool if isinstance(tool, dict) else {} + + +def _as_tool_dicts(tools: Sequence[Any] | None) -> list[dict[str, Any]]: + """Responses function tools with flattened namespaces, plus chat tools.""" + tools = list(tools or ()) + result = iter_response_function_tool_dicts(tools) + result.extend( + tool_dict + for tool_dict in map(_tool_dict, tools) + if isinstance(tool_dict.get("function"), dict) + ) + return result + + +def _tool_is_strict(tool: Any) -> bool: + tool_dict = _tool_dict(tool) + function = tool_dict.get("function") + return bool( + tool_dict.get("strict") + or (isinstance(function, dict) and function.get("strict")) + ) + + +def _decode(tokenizer: TokenizerLike, token_ids: Sequence[int]) -> str: + try: + return cast(Any, tokenizer.decode)( + token_ids, + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + spaces_between_special_tokens=False, + ) + except TypeError: + return tokenizer.decode(token_ids, skip_special_tokens=False) + + +def resolve_response_template( + tokenizer: Any | None, + template: dict[str, Any] | None = None, +) -> dict[str, Any] | None: + """Return `template` if given, otherwise the tokenizer's metadata.""" + if template is not None or tokenizer is None: + return template + template = getattr(tokenizer, "response_template", None) + if not isinstance(template, dict): + init_kwargs = getattr(tokenizer, "init_kwargs", None) or {} + template = init_kwargs.get("response_template") + return template if isinstance(template, dict) else None + + +def validate_response_template_for_serving( + template: dict[str, Any], +) -> ResponseTemplate: + """Validate the template and reject semantic fields serving cannot route.""" + loaded = load_response_template(template) + unsupported = (set(loaded.fields) - _SUPPORTED_FIELDS) | ( + set(loaded.defaults) - {"role"} + ) + if unsupported: + raise ValueError( + "response_template contains unsupported semantic fields: " + f"{sorted(unsupported)}. Supported fields are: " + f"{sorted(_SUPPORTED_FIELDS)}" + ) + for name in (THINKING_FIELD, CONTENT_FIELD): + field = loaded.fields.get(name) + if field is None: + continue + if ( + field.content != "text" + or field.transform is not None + or field.join is not None + ): + raise ValueError( + f"response_template field {name!r} uses semantics that cannot " + "be streamed by the OpenAI serving adapter" + ) + return loaded + + +def validate_tokenizer_response_template( + tokenizer: Any, + *, + reasoning: bool, + tools: bool, +) -> None: + """Fail at startup if the tokenizer cannot serve the requested parsing.""" + template = resolve_response_template(tokenizer) + if template is None: + raise TypeError( + "The hf parser requires `response_template` metadata " + "in the tokenizer configuration" + ) + try: + fields = validate_response_template_for_serving(template).fields + except (TypeError, ValueError) as exc: + raise TypeError(f"Invalid response_template metadata: {exc}") from exc + if reasoning and THINKING_FIELD not in fields: + raise TypeError( + "--reasoning-parser hf requires a response_template with a thinking field" + ) + if tools and TOOL_FIELD not in fields: + raise TypeError( + "--tool-call-parser hf requires a response_template with a tool_calls field" + ) + + +def _streaming_template(template: dict[str, Any]) -> ResponseTemplate: + result = copy.deepcopy(template) + fields = result["fields"] + if CONTENT_FIELD not in fields and all( + "open" in field or "open_pattern" in field for field in fields.values() + ): + fields[CONTENT_FIELD] = {} + for field in fields.values(): + field["optional"] = True + return load_response_template(result) + + +def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool: + return isinstance( + request.tool_choice, (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam) + ) + + +def _tool_parsing_enabled( + request: ChatCompletionRequest | ResponsesRequest, + *, + enable_auto_tools: bool, +) -> bool: + if request.tool_choice == "none": + return False + if request.tool_choice == "required" or _is_named_tool_choice(request): + return True + return enable_auto_tools and request.tool_choice in (None, "auto") + + +def _unsupported_tool_guarantee( + request: ChatCompletionRequest | ResponsesRequest, +) -> tuple[str, str] | None: + """The first requested tool guarantee this parser cannot enforce, with the + request parameter that asked for it.""" + if any(_tool_is_strict(tool) for tool in request.tools or ()): + return "strict tools", "tools" + if request.tool_choice == "required": + return "required tool choice", "tool_choice" + if _is_named_tool_choice(request): + return "named tool choice", "tool_choice" + if getattr(request, "parallel_tool_calls", True) is False: + return "parallel_tool_calls=False", "parallel_tool_calls" + return None + + +def _reasoning_has_ended( + parser: ResponseParser, + template: ResponseTemplate, + *, + thinking_disabled: bool = False, +) -> bool: + thinking = template.fields.get(THINKING_FIELD) + if thinking is None: + return True + if thinking.repeats: + return False + + active_field: str | None = None + thinking_closed = False + for event in parser.initial_events: + if event["type"] == "region_open": + active_field = event["field"] + elif event["type"] in {"region_close", "region_malformed"}: + thinking_closed |= event["field"] == THINKING_FIELD + active_field = None + if thinking_closed: + return True + if active_field in (CONTENT_FIELD, TOOL_FIELD): + return True + + # Resolve delimiters held at the prompt boundary. A NUL cannot occur in + # the model wire format and any implicit chunk containing it is a probe, + # not evidence that content generation has started. + for event in parser.feed("\0"): + if ( + event["type"] == "region_open" + and event["field"] in (CONTENT_FIELD, TOOL_FIELD) + and event["end"] > event["start"] + ): + return True + if event["field"] == THINKING_FIELD: + if event["type"] == "region_open": + return False + if event["type"] in {"region_close", "region_malformed"}: + return True + if active_field == THINKING_FIELD: + return False + return thinking_disabled + + +def _literal_ids( + tokenizer: TokenizerLike, vocab: dict[str, int], literals: Sequence[str] +) -> list[list[int]] | None: + """Token ids of each literal, or `None` when one encodes to nothing.""" + ids = [ + [vocab[literal]] + if literal in vocab + else tokenizer.encode(literal, add_special_tokens=False) + for literal in literals + ] + return ids if all(ids) else None + + +def _reasoning_end_ids( + tokenizer: TokenizerLike, vocab: dict[str, int], template: ResponseTemplate +) -> list[list[int]] | None: + """Token ids of each thinking closer, or `None` when they are not all + literal.""" + thinking = template.fields.get(THINKING_FIELD) + if thinking is None or thinking.repeats or not thinking.close_literals: + return None + return _literal_ids(tokenizer, vocab, thinking.close_literals) + + +def _ends_within(token_ids: Sequence[int], end_ids: list[int], num_new: int) -> bool: + """Whether `end_ids` completes within the last `num_new` of `token_ids`.""" + window = list(token_ids[max(0, len(token_ids) - num_new - len(end_ids) + 1) :]) + return any( + window[index : index + len(end_ids)] == end_ids + for index in range(len(window) - len(end_ids) + 1) + ) + + +def _serving_template( + tokenizer: Any, + template: dict[str, Any] | None, +) -> ResponseTemplate: + """The validated streaming form of `template` or the tokenizer's metadata.""" + template = resolve_response_template(tokenizer, template) + if template is None: + raise ValueError( + "The hf parser requires `response_template` " + "metadata in the tokenizer configuration" + ) + validate_response_template_for_serving(template) + return _streaming_template(template) + + +def _to_tool_calls(value: Any) -> list[tuple[str, str]]: + """`(name, arguments)` of each call in a tool region value, or nothing if + any call is incomplete.""" + values = value if isinstance(value, list) else [value] + calls: list[tuple[str, str]] = [] + for item in values: + function = item.get("function") if isinstance(item, dict) else None + if not isinstance(function, dict): + return [] + name = function.get("name") + arguments = function.get("arguments") + if not isinstance(name, str) or not name or arguments is None: + return [] + if not isinstance(arguments, str): + arguments = json.dumps(arguments, ensure_ascii=False) + calls.append((name, arguments)) + return calls + + +class ResponseTemplateReasoningParser(ReasoningParser): + """Reasoning end detection for `--reasoning-parser hf`. + + `ResponseTemplateParser` parses the output; this parser tells it and the + structured-output engine where reasoning ends. + """ + + def __init__( + self, + tokenizer: TokenizerLike, + *args, + response_template: dict[str, Any] | None = None, + **kwargs, + ) -> None: + super().__init__(tokenizer, *args, **kwargs) + self.response_template = _serving_template(tokenizer, response_template) + chat_template_kwargs = kwargs.get("chat_template_kwargs") or {} + self._thinking_disabled = not chat_template_kwargs.get("enable_thinking", True) + self._reasoning_end_ids = _reasoning_end_ids( + tokenizer, self.vocab, self.response_template + ) + + @property + def reasoning_start_str(self) -> str | None: + thinking = self.response_template.fields.get(THINKING_FIELD) + literals = thinking.open_literals if thinking else None + return literals[0] if literals else None + + @property + def reasoning_end_str(self) -> str | None: + thinking = self.response_template.fields.get(THINKING_FIELD) + literals = thinking.close_literals if thinking else None + return literals[0] if literals else None + + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + try: + parser = ResponseParser( + self.response_template, + prefix=_decode(self.model_tokenizer, input_ids), + ) + except Exception: + return False + return _reasoning_has_ended( + parser, + self.response_template, + thinking_disabled=self._thinking_disabled, + ) + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + if self._reasoning_end_ids is None: + return self.is_reasoning_end(input_ids) + num_new = len(list(delta_ids)) + return any( + _ends_within(input_ids, end_ids, num_new) + for end_ids in self._reasoning_end_ids + ) + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + raise NotImplementedError("ResponseTemplateParser extracts content") + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + raise NotImplementedError("ResponseTemplateParser extracts reasoning") + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + raise NotImplementedError("ResponseTemplateParser extracts reasoning") + + +class ResponseTemplateToolParser(ToolParser): + """Registers `--tool-call-parser hf`; `ResponseTemplateParser` extracts the + tool calls.""" + + +class _DeltaBuilder: + """Accumulates one `DeltaMessage`.""" + + def __init__(self) -> None: + self.reasoning: list[str] = [] + self.content: list[str] = [] + self.tool_calls: list[DeltaToolCall] = [] + + def build(self) -> DeltaMessage | None: + fields: dict[str, Any] = {} + if reasoning := "".join(self.reasoning): + fields["reasoning"] = reasoning + if content := "".join(self.content): + fields["content"] = content + if self.tool_calls: + fields["tool_calls"] = self.tool_calls + return DeltaMessage(**fields) if fields else None + + +class _ResponseStream: + """One generation fed through `ResponseParser`, routed to message deltas.""" + + def __init__( + self, + template: ResponseTemplate, + *, + prefix: str, + tools: list[dict[str, Any]], + parse_reasoning: bool, + include_reasoning: bool, + parse_tools: bool, + new_tool_call_id: Callable[[str], str], + ) -> None: + self.parse_reasoning = parse_reasoning + self.include_reasoning = include_reasoning + self.parse_tools = parse_tools + self.new_tool_call_id = new_tool_call_id + self.parser = ResponseParser(template, prefix=prefix, tools=tools) + self._prefix_end = len(self.parser.input_text) + self._next_tool_index = 0 + self._finalized = False + + def feed(self, text: str, *, finished: bool) -> DeltaMessage | None: + if self._finalized: + return None + delta = _DeltaBuilder() + self._route(self.parser.feed(text), delta) + if finished: + self._finalized = True + if len(self.parser.input_text) > self._prefix_end: + _, events = self.parser.finalize() + self._route(events, delta) + return delta.build() + + def _route(self, events: Sequence[dict[str, Any]], delta: _DeltaBuilder) -> None: + for event in events: + field = event.get("field") + if field == TOOL_FIELD: + self._route_tool_event(event, delta) + elif event["type"] == "region_chunk": + if field == THINKING_FIELD and self.parse_reasoning: + if self.include_reasoning: + delta.reasoning.append(event["text"]) + elif field in (THINKING_FIELD, CONTENT_FIELD): + delta.content.append(event["text"]) + + def _route_tool_event(self, event: dict[str, Any], delta: _DeltaBuilder) -> None: + """Emit the calls of a tool region once it ends and parses; a region cut + off at the end of the stream is parsed as generated. With tool parsing + disabled, the region passes through as content.""" + if not self.parse_tools: + if event["type"] == "region_chunk": + delta.content.append(event["text"]) + else: + start = max(event["start"], self._prefix_end) + delta.content.append(self.parser.input_text[start : event["end"]]) + return + if event["type"] == "region_malformed": + logger.warning("response_template: dropping a malformed tool call") + elif event["type"] == "region_close": + calls = _to_tool_calls(event.get("value")) + if not calls: + logger.warning("response_template: dropping an incomplete tool call") + for name, arguments in calls: + delta.tool_calls.append( + DeltaToolCall( + index=self._next_tool_index, + id=self.new_tool_call_id(name), + type="function", + function=DeltaFunctionCall(name=name, arguments=arguments), + ) + ) + self._next_tool_index += 1 + + +class ResponseTemplateParser(Parser): + """Parse reasoning, content, and tool calls with one response template.""" + + reasoning_parser_cls: type[ReasoningParser] | None = ResponseTemplateReasoningParser + tool_parser_cls: type[ToolParser] | None = ResponseTemplateToolParser + always_adjust_request = True + _enable_auto_tools = False + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + *, + response_template: dict[str, Any] | None = None, + enable_auto_tools: bool | None = None, + **kwargs, + ) -> None: + self.response_template = _serving_template(tokenizer, response_template) + super().__init__( + tokenizer, tools, response_template=response_template, **kwargs + ) + if enable_auto_tools is not None: + self._enable_auto_tools = enable_auto_tools + self._tools = _as_tool_dicts(tools) + self._prompt_token_ids: list[int] | None = None + self._prefix = "" + self._stream: _ResponseStream | None = None + + def set_prompt_token_ids(self, prompt_token_ids: Sequence[int]) -> None: + prompt_token_ids = list(prompt_token_ids) + if prompt_token_ids == self._prompt_token_ids: + return + self._prompt_token_ids = prompt_token_ids + self._prefix = _decode(self.model_tokenizer, prompt_token_ids) + self._stream = None + + def adjust_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + if getattr(request, "chat_template", None) is not None: + logger.warning_once( + "A request chat template is set; response_template parsing still " + "expects the checkpoint's output format." + ) + request.skip_special_tokens = False + if hasattr(request, "spaces_between_special_tokens"): + request.spaces_between_special_tokens = False + if ( + self._tool_parser is not None + and request.tools + and request.tool_choice != "none" + and (unsupported := _unsupported_tool_guarantee(request)) is not None + ): + guarantee, parameter = unsupported + raise VLLMValidationError( + f"response_template parsing cannot guarantee {guarantee} " + "without a format-compatible structural grammar", + parameter=parameter, + ) + return request + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + return ( + self._reasoning_parser is None + or self._reasoning_parser.is_reasoning_end(input_ids) + ) + + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + self._initialize_history_tool_call_cnt(request) + stream = self._new_stream( + request, + enable_auto_tools=enable_auto_tools, + include_reasoning=True, + ) + delta = stream.feed(model_output, finished=True) + if delta is None: + return None, None, None + tool_calls = [ + FunctionCall( + id=call.id, + name=call.function.name or "", + arguments=call.function.arguments or "{}", + ) + for call in delta.tool_calls + if call.function is not None + ] + return delta.reasoning, delta.content, tool_calls or None + + def parse_delta( + self, + delta_text: str, + delta_token_ids: list[int], + request: ChatCompletionRequest | ResponsesRequest, + prompt_token_ids: list[int] | None = None, + *, + finished: bool, + ) -> DeltaMessage | None: + self._initialize_history_tool_call_cnt(request) + if self._prompt_token_ids is None and prompt_token_ids is not None: + self.set_prompt_token_ids(prompt_token_ids) + if self._stream is None: + self._stream = self._new_stream( + request, + enable_auto_tools=self._enable_auto_tools, + include_reasoning=request.include_reasoning, + ) + return self._stream.feed(delta_text, finished=finished) + + def _new_stream( + self, + request: ChatCompletionRequest | ResponsesRequest, + *, + enable_auto_tools: bool, + include_reasoning: bool, + ) -> _ResponseStream: + return _ResponseStream( + self.response_template, + prefix=self._prefix, + tools=self._tools, + parse_reasoning=self._reasoning_parser is not None, + include_reasoning=include_reasoning, + parse_tools=self._tool_parser is not None + and _tool_parsing_enabled(request, enable_auto_tools=enable_auto_tools), + new_tool_call_id=self._new_tool_call_id, + ) + + def _new_tool_call_id(self, name: str) -> str: + state = self._stream_state + tool_call_id = make_tool_call_id( + id_type=state.tool_call_id_type, + func_name=name, + idx=state.history_tool_call_cnt, + ) + state.history_tool_call_cnt += 1 + return tool_call_id diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 9a4413cbd9..f9ab1dc9a1 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -80,6 +80,10 @@ _REASONING_PARSERS_TO_REGISTER = { "granite_thinking_engine_reasoning_parser", "GraniteThinkingParserReasoningAdapter", ), + "hf": ( + "response_template_reasoning_parser", + "ResponseTemplateReasoningParser", + ), "holo2": ( "deepseek_v3_reasoning_parser", "DeepSeekV3ReasoningWithThinkingParser", diff --git a/vllm/reasoning/response_template_reasoning_parser.py b/vllm/reasoning/response_template_reasoning_parser.py new file mode 100644 index 0000000000..757dd73417 --- /dev/null +++ b/vllm/reasoning/response_template_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.response_template import ResponseTemplateReasoningParser + +__all__ = ["ResponseTemplateReasoningParser"] diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index e1f67397ce..c1f175798d 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -79,6 +79,7 @@ class OnlineDerenderer: tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, + tokenizer=renderer.tokenizer, ) self.chat_template = chat_template @@ -160,6 +161,8 @@ class OnlineDerenderer: chat_template_kwargs=chat_template_kwargs, model_config=self.model_config, ) + if generate_response.prompt_token_ids is not None: + parser.set_prompt_token_ids(generate_response.prompt_token_ids) reasoning, content, tool_calls = parser.parse( decoded_text, chat_request, diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index ec8b58f73d..23352ef1cf 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -183,6 +183,7 @@ class OnlineRenderer: tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, + tokenizer=renderer.tokenizer, ) self.chat_template = chat_template @@ -825,7 +826,8 @@ class OnlineRenderer: and tokenizer.supports_grammar ) should_adjust_request = ( - parser.reasoning_parser_cls is not None + parser.always_adjust_request + or parser.reasoning_parser_cls is not None or tool_choice != "none" or is_mistral_grammar_eligible ) diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 5647a75291..8446aae815 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -86,6 +86,10 @@ _TOOL_PARSERS_TO_REGISTER = { "hermes_tool_parser", "Hermes2ProToolParser", ), + "hf": ( + "response_template_tool_parser", + "ResponseTemplateToolParser", + ), "poolside_v1": ( "poolside_v1_tool_parser", "PoolsideV1ToolParser", diff --git a/vllm/tool_parsers/response_template_tool_parser.py b/vllm/tool_parsers/response_template_tool_parser.py new file mode 100644 index 0000000000..32b4b44de9 --- /dev/null +++ b/vllm/tool_parsers/response_template_tool_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.response_template import ResponseTemplateToolParser + +__all__ = ["ResponseTemplateToolParser"] From ea106db806918f371b8f392cc896dc7a2d6677a1 Mon Sep 17 00:00:00 2001 From: Prudhvi Vuda <53619858+Prudhvivuda@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:51:34 -0400 Subject: [PATCH 236/248] [Bugfix][Bench] Clean up synthetic video files and writers (#59529) Signed-off-by: Prudhvivuda Co-authored-by: OpenAI Codex --- .../test_random_multimodal_dataset_video.py | 40 ++++++++++++++++++- vllm/benchmarks/datasets/datasets.py | 30 ++++++-------- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/tests/benchmarks/test_random_multimodal_dataset_video.py b/tests/benchmarks/test_random_multimodal_dataset_video.py index b394ea2c0d..ebe17989c1 100644 --- a/tests/benchmarks/test_random_multimodal_dataset_video.py +++ b/tests/benchmarks/test_random_multimodal_dataset_video.py @@ -2,8 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import os +import tempfile +from pathlib import Path from tempfile import NamedTemporaryFile from typing import Any, cast +from unittest.mock import Mock import cv2 import pybase64 as base64 @@ -61,11 +64,17 @@ def test_map_config_to_modality(video_dataset: RandomMultiModalDataset): @pytest.mark.benchmark -def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset): - """Test generating multimodal items for video configurations.""" +def test_generate_mm_item_video( + video_dataset: RandomMultiModalDataset, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +): + """Generate a decodable video without leaving temporary files behind.""" + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) # Test video item generation video_config = (64, 48, 8) # height, width, num_frames result = video_dataset.generate_mm_item(video_config) + assert not list(tmp_path.iterdir()) # Check the result structure matches OpenAI API format assert isinstance(result, dict) @@ -105,6 +114,33 @@ def test_generate_mm_item_video(video_dataset: RandomMultiModalDataset): os.unlink(temp_path) +@pytest.mark.benchmark +@pytest.mark.parametrize("writer_opened", [False, True], ids=["open", "write"]) +def test_generate_synthetic_video_cleans_up_on_error( + video_dataset: RandomMultiModalDataset, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + writer_opened: bool, +): + """Release the writer and remove temporary storage when encoding fails.""" + monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) + writer = Mock(spec=cv2.VideoWriter) + writer.isOpened.return_value = writer_opened + writer.write.side_effect = RuntimeError("Failed to write video frame") + monkeypatch.setattr(cv2, "VideoWriter", lambda *args, **kwargs: writer) + + error = ( + "Failed to write video frame" + if writer_opened + else "Failed to create video writer" + ) + with pytest.raises(RuntimeError, match=error): + video_dataset.generate_synthetic_video(48, 64, 8) + + assert not list(tmp_path.iterdir()) + writer.release.assert_called_once_with() + + @pytest.mark.benchmark def test_generate_mm_item_image(video_dataset: RandomMultiModalDataset): """Test generating multimodal items for image configurations.""" diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index 76de75a66f..abaf299a19 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -25,7 +25,7 @@ from dataclasses import dataclass, replace from functools import cache from io import BytesIO from pathlib import Path -from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory from typing import Any, cast import numpy as np @@ -975,32 +975,26 @@ class RandomMultiModalDataset(RandomDataset): dtype=np.uint8, ) - # Create a temporary video file in memory fourcc = cv2.VideoWriter_fourcc(*"mp4v") fps = 30 # frames per second - with NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file: - temp_path = temp_file.name + with TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "video.mp4" - # Create video writer video_writer = cv2.VideoWriter( - temp_path, fourcc=fourcc, fps=fps, frameSize=(width, height) + str(temp_path), fourcc=fourcc, fps=fps, frameSize=(width, height) ) - if not video_writer.isOpened(): - raise RuntimeError("Failed to create video writer") + try: + if not video_writer.isOpened(): + raise RuntimeError("Failed to create video writer") - for frame in random_pixels: - video_writer.write(frame) + for frame in random_pixels: + video_writer.write(frame) + finally: + video_writer.release() - video_writer.release() - temp_file.close() - - # Read the video file content - with open(temp_path, "rb") as f: - video_content = f.read() - - return {"bytes": video_content} + return {"bytes": temp_path.read_bytes()} def map_config_to_modality(self, config: tuple[int, int, int]) -> str: """Map the configuration to the modality.""" From 8faacd707f2f69b86ecd0bb1e7b5a705b473cb25 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:15:14 +0100 Subject: [PATCH 237/248] [Model] Migrate Glm, Arcee, CWM and Mellum to the Transformers modeling backend (#59679) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- docs/models/supported_models.md | 8 +- vllm/model_executor/models/arcee.py | 365 ------------------- vllm/model_executor/models/bert_with_rope.py | 1 - vllm/model_executor/models/glm.py | 24 -- vllm/model_executor/models/mellum.py | 253 ------------- vllm/model_executor/models/registry.py | 9 +- 6 files changed, 8 insertions(+), 652 deletions(-) delete mode 100644 vllm/model_executor/models/arcee.py delete mode 100644 vllm/model_executor/models/glm.py delete mode 100644 vllm/model_executor/models/mellum.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index df44a3e8b0..545890e152 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -374,7 +374,6 @@ th { | ------------ | ------ | ----------------- | -------------------- | ------------------------- | | `AfmoeForCausalLM` | Afmoe | TBA | ✅︎ | ✅︎ | | `ApertusForCausalLM` | Apertus | `swiss-ai/Apertus-8B-2509`, `swiss-ai/Apertus-70B-Instruct-2509`, etc. | ✅︎ | ✅︎ | -| `ArceeForCausalLM` | Arcee (AFM) | `arcee-ai/AFM-4.5B-Base`, etc. | ✅︎ | ✅︎ | | `AXK1ForCausalLM` | A.X-K1 | `skt/A.X-K1`, etc. | | ✅︎ | | `BailingMoeForCausalLM` | Ling | `inclusionAI/Ling-lite-1.5`, `inclusionAI/Ling-plus`, etc. | ✅︎ | ✅︎ | | `BailingMoeV2ForCausalLM` | Ling | `inclusionAI/Ling-mini-2.0`, etc. | ✅︎ | ✅︎ | @@ -384,7 +383,6 @@ th { | `ChatGLMModel`, `ChatGLMForConditionalGeneration` | ChatGLM | `zai-org/chatglm2-6b`, `zai-org/chatglm3-6b`, `thu-coai/ShieldLM-6B-chatglm3`, etc. | ✅︎ | ✅︎ | | `CohereForCausalLM`, `Cohere2ForCausalLM` | Command-R, Command-A | `CohereLabs/c4ai-command-r-v01`, `CohereLabs/c4ai-command-r7b-12-2024`, `CohereLabs/c4ai-command-a-03-2025`, `CohereLabs/command-a-reasoning-08-2025`, etc. | ✅︎ | ✅︎ | | `Cohere2MoeForCausalLM` | North-Mini-Code | `CohereLabs/North-Mini-Code`, etc. | ✅︎ | ✅︎ | -| `CwmForCausalLM` | CWM | `facebook/cwm`, etc. | ✅︎ | ✅︎ | | `DbrxForCausalLM` | DBRX | `databricks/dbrx-base`, `databricks/dbrx-instruct`, etc. | | ✅︎ | | `DeciLMForCausalLM` | DeciLM | `nvidia/Llama-3_3-Nemotron-Super-49B-v1`, etc. | ✅︎ | ✅︎ | | `DeepseekForCausalLM` | DeepSeek | `deepseek-ai/deepseek-llm-67b-base`, `deepseek-ai/deepseek-llm-7b-chat`, etc. | ✅︎ | ✅︎ | @@ -406,7 +404,6 @@ th { | `Gemma3ForCausalLM` | Gemma 3 | `google/gemma-3-1b-it`, etc. | ✅︎ | ✅︎ | | `Gemma3nForCausalLM` | Gemma 3n | `google/gemma-3n-E2B-it`, `google/gemma-3n-E4B-it`, etc. | | | | `Gemma4ForCausalLM` | Gemma 4 | `google/gemma-4-E2B-it`, etc. | ✅︎ | ✅︎ | -| `GlmForCausalLM` | GLM-4 | `zai-org/glm-4-9b-chat-hf`, etc. | ✅︎ | ✅︎ | | `Glm4ForCausalLM` | GLM-4-0414 | `zai-org/GLM-4-32B-0414`, etc. | ✅︎ | ✅︎ | | `Glm4MoeForCausalLM` | GLM-4.5, GLM-4.6, GLM-4.7 | `zai-org/GLM-4.5`, etc. | ✅︎ | ✅︎ | | `Glm4MoeLiteForCausalLM` | GLM-4.7-Flash | `zai-org/GLM-4.7-Flash`, etc. | ✅︎ | ✅︎ | @@ -439,7 +436,6 @@ th { | `LongcatFlashForCausalLM` | LongCat-Flash | `meituan-longcat/LongCat-Flash-Chat`, `meituan-longcat/LongCat-Flash-Chat-FP8` | ✅︎ | ✅︎ | | `MambaForCausalLM` | Mamba | `state-spaces/mamba-130m-hf`, `state-spaces/mamba-790m-hf`, `state-spaces/mamba-2.8b-hf`, etc. | | ✅︎ | | `Mamba2ForCausalLM` | Mamba2 | `mistralai/Mamba-Codestral-7B-v0.1`, etc. | | ✅︎ | -| `MellumForCausalLM` | Mellum 2 | `JetBrains/Mellum2-12B-A2.5B-Base`, etc. | | ✅︎ | | `MiMoForCausalLM` | MiMo | `XiaomiMiMo/MiMo-7B-RL`, etc. | ✅︎ | ✅︎ | | `MiMoV2FlashForCausalLM` | MiMoV2Flash | `XiaomiMiMo/MiMo-V2-Flash`, etc. | | ✅︎ | | `MiMoV2ForCausalLM` | MiMoV2Pro | `XiaomiMiMo/MiMo-V2.5-Pro`, etc. | | ✅︎ | @@ -486,10 +482,14 @@ Some models are supported only via the [Transformers modeling backend](#transfor | Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | | ------------ | ------ | ----------------- | -------------------- | ------------------------- | +| `ArceeForCausalLM` | Arcee (AFM) | `arcee-ai/AFM-4.5B-Base`, etc. | ✅︎ | ✅︎ | +| `CwmForCausalLM` | CWM | `facebook/cwm`, etc. | ✅︎ | ✅︎ | | `FlexOlmoForCausalLM` | FlexOlmo | `allenai/FlexOlmo-7x7B-1T`, `allenai/FlexOlmo-7x7B-1T-RT`, etc. | | ✅︎ | +| `GlmForCausalLM` | GLM-4 | `zai-org/glm-4-9b-chat-hf`, etc. | ✅︎ | ✅︎ | | `GPTBigCodeForCausalLM` | StarCoder, SantaCoder, WizardCoder | `bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, `WizardLM/WizardCoder-15B-V1.0`, etc. | ✅︎ | | | `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | | `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | +| `MellumForCausalLM` | Mellum 2 | `JetBrains/Mellum2-12B-A2.5B-Base`, etc. | | ✅︎ | | `NanbeigeForCausalLM` | Nanbeige4.2 | `Nanbeige/Nanbeige4.2-3B`, etc. | ✅︎ | ✅︎ | | `OlmoForCausalLM` | OLMo | `allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc. | ✅︎ | ✅︎ | | `Olmo2ForCausalLM` | OLMo2 | `allenai/OLMo-2-0425-1B`, etc. | ✅︎ | ✅︎ | diff --git a/vllm/model_executor/models/arcee.py b/vllm/model_executor/models/arcee.py deleted file mode 100644 index 32ad19a8e7..0000000000 --- a/vllm/model_executor/models/arcee.py +++ /dev/null @@ -1,365 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Copyright 2023-2025 vLLM Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Inference-only Arcee (AFM) model – adds support for ReLU^2 feed-forward -# activation. - -from collections.abc import Iterable -from itertools import islice -from typing import Any - -import torch -from torch import nn -from transformers import LlamaConfig - -from vllm.compilation.decorators import support_torch_compile -from vllm.distributed import get_pp_group -from vllm.model_executor.layers.activation import ReLUSquaredActivation -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.vocab_parallel_embedding import ( - ParallelLMHead, - VocabParallelEmbedding, -) -from vllm.sequence import IntermediateTensors - -from .interfaces import ( - EagleModelMixin, - SupportsEagle, - SupportsEagle3, - SupportsLoRA, - SupportsPP, -) -from .utils import ( - AutoWeightsLoader, - PPMissingLayer, - WeightsMapper, - make_empty_intermediate_tensors_factory, - make_layers, - maybe_prefix, -) - - -class ArceeMLP(nn.Module): - """Feed-forward layer for Arcee using ReLU^2 activation - (no gating as in LLaMA).""" - - def __init__( - self, - hidden_size: int, - intermediate_size: int, - hidden_act: str, - quant_config: Any | None = None, - bias: bool = False, - prefix: str = "", - reduce_results: bool = True, - ) -> None: - super().__init__() - # Single linear projection up to intermediate size - # (no separate gate projection) - self.up_proj = ColumnParallelLinear( - input_size=hidden_size, - output_size=intermediate_size, - bias=bias, - quant_config=quant_config, - prefix=f"{prefix}.up_proj", - ) - # Down projection back to hidden size - self.down_proj = RowParallelLinear( - input_size=intermediate_size, - output_size=hidden_size, - bias=bias, - quant_config=quant_config, - reduce_results=reduce_results, - prefix=f"{prefix}.down_proj", - ) - if hidden_act != "relu2": - raise ValueError( - f"Unsupported activation: {hidden_act}. " - "Only 'relu2' is supported for AFM." - ) - # Define ReLU^2 activation: (ReLU(x))^2 elementwise - self.act_fn = ReLUSquaredActivation() - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x, _ = self.up_proj(x) # Project to intermediate size - x = self.act_fn(x) # Apply ReLU^2 activation elementwise - x, _ = self.down_proj(x) # Project back down to hidden size - return x - - -class ArceeDecoderLayer(nn.Module): - """Transformer decoder block for Arcee, with self-attention and - ReLU^2 MLP.""" - - def __init__( - self, - config: LlamaConfig, - cache_config: Any | None = None, - quant_config: Any | None = None, - prefix: str = "", - ) -> None: - super().__init__() - self.hidden_size = config.hidden_size - max_position_embeddings = getattr(config, "max_position_embeddings", 8192) - # Determine if attention bias is needed (some variants use bias terms) - attention_bias = getattr(config, "attention_bias", False) or getattr( - config, "bias", False - ) - bias_o_proj = attention_bias - if hasattr(config, "qkv_bias"): - attention_bias = config.qkv_bias - - # Self-Attention (using LLaMA's attention structure) - from vllm.model_executor.models.llama import ( - LlamaAttention, # import here to avoid circular import - ) - - self.self_attn = LlamaAttention( - config=config, - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - num_kv_heads=getattr( - config, "num_key_value_heads", config.num_attention_heads - ), - max_position_embeddings=max_position_embeddings, - quant_config=quant_config, - bias=attention_bias, - bias_o_proj=bias_o_proj, - cache_config=cache_config, - prefix=f"{prefix}.self_attn", - attn_type=getattr( - config, "attn_type", "decoder" - ), # assume decoder (causal) unless specified - ) - # MLP with ReLU^2 activation - self.mlp = ArceeMLP( - hidden_size=self.hidden_size, - intermediate_size=config.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - bias=getattr(config, "mlp_bias", False), - prefix=f"{prefix}.mlp", - ) - # Layer normalization layers (RMSNorm as in LLaMA) - self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - def forward( - self, - positions: torch.Tensor, - hidden_states: torch.Tensor, - residual: torch.Tensor | None, - ) -> tuple[torch.Tensor, torch.Tensor]: - # Self-Attention block - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - # Fused residual add + layernorm if supported - hidden_states, residual = self.input_layernorm(hidden_states, residual) - hidden_states = self.self_attn(positions=positions, hidden_states=hidden_states) - # Feed-forward block - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual - - -@support_torch_compile -class ArceeModel(nn.Module, EagleModelMixin): - """The transformer model backbone for Arcee (embedding layer + stacked - decoder blocks + final norm).""" - - def __init__( - self, - *, - vllm_config, - prefix: str = "", - layer_type: type[nn.Module] = ArceeDecoderLayer, - ) -> None: - super().__init__() - config: LlamaConfig = vllm_config.model_config.hf_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - self.quant_config = quant_config - self.config = config - self.vocab_size = config.vocab_size - - # Word embeddings (parallelized if using pipeline parallel) - if get_pp_group().is_first_rank or ( - config.tie_word_embeddings and get_pp_group().is_last_rank - ): - self.embed_tokens = VocabParallelEmbedding( - self.vocab_size, - config.hidden_size, - quant_config=quant_config, - ) - else: - self.embed_tokens = PPMissingLayer() # placeholder on non-embedding ranks - - # Build decoder layers across pipeline ranks - self.start_layer, self.end_layer, self.layers = make_layers( - config.num_hidden_layers, - lambda prefix: layer_type( - config=config, - cache_config=cache_config, - quant_config=quant_config, - prefix=prefix, - ), - prefix=f"{prefix}.layers", - ) - # Final RMSNorm on the last pipeline stage - if get_pp_group().is_last_rank: - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - else: - self.norm = PPMissingLayer() - - # Prepare factory for empty intermediate tensors - # (for pipeline scheduling) - self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( - ["hidden_states", "residual"], config.hidden_size - ) - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.embed_tokens(input_ids) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: - # Embedding lookup (on first pipeline rank) - if get_pp_group().is_first_rank: - hidden_states = ( - inputs_embeds - if inputs_embeds is not None - else self.embed_input_ids(input_ids) - ) - residual = None - else: - assert intermediate_tensors is not None, ( - "IntermediateTensors must be provided for non-first pipeline ranks" - ) - hidden_states = intermediate_tensors["hidden_states"] - residual = intermediate_tensors["residual"] - - aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual) - for idx, layer in enumerate( - islice(self.layers, self.start_layer, self.end_layer) - ): - hidden_states, residual = layer(positions, hidden_states, residual) - self._maybe_add_hidden_state( - aux_hidden_states, idx + 1, hidden_states, residual - ) - - if not get_pp_group().is_last_rank: - # Send intermediate results to the next pipeline stage - return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} - ) - # On last rank: apply final layer norm - hidden_states, _ = self.norm(hidden_states, residual) - if len(aux_hidden_states) > 0: - return hidden_states, aux_hidden_states - return hidden_states - - -class ArceeForCausalLM( - nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3 -): - """Arcee Model for causal language modeling, integrated with vLLM - runtime.""" - - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_stacked={ - # weight_name: (param_name, shard_id) - ".q_proj": (".qkv_proj", "q"), - ".k_proj": (".qkv_proj", "k"), - ".v_proj": (".qkv_proj", "v"), - }, - orig_to_new_substr={ - "gate_proj": None, - }, - ) - # Map fused module names to their submodule components - # (for quantization and LoRA) - packed_modules_mapping = { - "qkv_proj": ["q_proj", "k_proj", "v_proj"], - } - - def __init__(self, *, vllm_config, prefix: str = "") -> None: - super().__init__() - config = vllm_config.model_config.hf_config - self.config = config - - # Initialize the inner Transformer model (ArceeModel) - self.model = ArceeModel( - vllm_config=vllm_config, - prefix=maybe_prefix(prefix, "model"), - ) - # On the last pipeline stage, set up the LM head and logits processor - if get_pp_group().is_last_rank: - # Determine vocabulary size (including any LoRA extra tokens - # for padded LM head) - - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=vllm_config.quant_config, - bias=getattr(config, "lm_head_bias", False), - prefix=maybe_prefix(prefix, "lm_head"), - ) - if config.tie_word_embeddings: - # Tie output weights with input embedding matrix - self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) - logit_scale = getattr(config, "logit_scale", 1.0) - self.logits_processor = LogitsProcessor( - config.vocab_size, scale=logit_scale - ) - else: - # Placeholder for lm_head on non-last ranks - self.lm_head = PPMissingLayer() - - self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors - ) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - ) -> torch.Tensor | IntermediateTensors: - model_output = self.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return model_output - - def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: - # Compute final logits from hidden states (last pipeline rank only) - logits = self.logits_processor(self.lm_head, hidden_states) - return logits - - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: - return self.model.embed_input_ids(input_ids) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load weights into the model (delegates to inner model and handles - tied embeddings).""" - loader = AutoWeightsLoader(self) - # AutoWeightLoader handles weight name remapping, including fusing - # separate q_proj, k_proj, v_proj into qkv_proj - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/bert_with_rope.py b/vllm/model_executor/models/bert_with_rope.py index cd35792b28..ed0ee78b2b 100644 --- a/vllm/model_executor/models/bert_with_rope.py +++ b/vllm/model_executor/models/bert_with_rope.py @@ -572,7 +572,6 @@ class GteNewModel(BertWithRope): super().__init__(vllm_config=vllm_config, prefix=prefix, **kwargs) # GteNewModel only gate_up_proj does not have bias. - # Hack method learned from vllm/model_executor/models/glm.py for layer in self.encoder.layers: layer.mlp.gate_up_proj.bias = None layer.mlp.gate_up_proj.skip_bias_add = True diff --git a/vllm/model_executor/models/glm.py b/vllm/model_executor/models/glm.py deleted file mode 100644 index 26d7c29aae..0000000000 --- a/vllm/model_executor/models/glm.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Inference-only HF format GLM-4 model compatible with THUDM weights.""" - -from vllm.config import VllmConfig -from vllm.model_executor.models.llama import LlamaForCausalLM - -from .utils import PPMissingLayer - - -class GlmForCausalLM(LlamaForCausalLM): - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - hf_config = vllm_config.model_config.hf_config - hf_config.rope_parameters["partial_rotary_factor"] = 0.5 - super().__init__(vllm_config=vllm_config, prefix=prefix) - # Hack Llama model to fit HF format GLM implementation - # Attention difference between GLM and Llama: - # 1. Half partial rotary_dim and no Neox style. - # 2. There is no bias for o_proj in attention - for layer in self.model.layers: - if not isinstance(layer, PPMissingLayer): - layer.self_attn.rotary_emb.is_neox_style = False - layer.self_attn.o_proj.bias = None - layer.self_attn.o_proj.skip_bias_add = True diff --git a/vllm/model_executor/models/mellum.py b/vllm/model_executor/models/mellum.py deleted file mode 100644 index a75e3b0a8e..0000000000 --- a/vllm/model_executor/models/mellum.py +++ /dev/null @@ -1,253 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any - -from torch import nn - -from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig -from vllm.distributed import get_tensor_model_parallel_world_size -from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.layernorm import RMSNorm -from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear -from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead - -from .qwen3_moe import ( - Qwen3MoeAttention, - Qwen3MoeDecoderLayer, - Qwen3MoeForCausalLM, - Qwen3MoeMLP, - Qwen3MoeModel, - Qwen3MoeSparseMoeBlock, -) -from .utils import PPMissingLayer, extract_layer_index, maybe_prefix - - -class MellumAttention(Qwen3MoeAttention): - """Differences from `Qwen3MoeAttention`: - - Supports `per_layer_sliding_window` for `Attention`. - """ - - def __init__( - self, - hidden_size: int, - num_heads: int, - num_kv_heads: int, - rope_parameters: dict[str, Any], - max_position_embeddings: int = 8192, - head_dim: int | None = None, - rms_norm_eps: float = 1e-06, - qkv_bias: bool = False, - cache_config: Any | None = None, - quant_config: Any | None = None, - prefix: str = "", - dual_chunk_attention_config: dict[str, Any] | None = None, - per_layer_sliding_window: int | None = None, - ) -> None: - nn.Module.__init__(self) - - self.hidden_size = hidden_size - tp_size = get_tensor_model_parallel_world_size() - self.total_num_heads = num_heads - assert self.total_num_heads % tp_size == 0 - self.num_heads = self.total_num_heads // tp_size - self.total_num_kv_heads = num_kv_heads - if self.total_num_kv_heads >= tp_size: - assert self.total_num_kv_heads % tp_size == 0 - else: - assert tp_size % self.total_num_kv_heads == 0 - self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) - self.head_dim = head_dim or (hidden_size // self.total_num_heads) - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_kv_heads * self.head_dim - self.scaling = self.head_dim**-0.5 - self.max_position_embeddings = max_position_embeddings - self.dual_chunk_attention_config = dual_chunk_attention_config - - self.qkv_proj = QKVParallelLinear( - hidden_size, - self.head_dim, - self.total_num_heads, - self.total_num_kv_heads, - bias=qkv_bias, - quant_config=quant_config, - prefix=f"{prefix}.qkv_proj", - ) - - self.o_proj = RowParallelLinear( - self.total_num_heads * self.head_dim, - hidden_size, - bias=False, - quant_config=quant_config, - prefix=f"{prefix}.o_proj", - ) - - self.rotary_emb = get_rope( - self.head_dim, - max_position=max_position_embeddings, - rope_parameters=rope_parameters, - dual_chunk_attention_config=dual_chunk_attention_config, - ) - if dual_chunk_attention_config: - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, - per_layer_sliding_window=per_layer_sliding_window, - prefix=f"{prefix}.attn", - layer_idx=extract_layer_index(prefix), - dual_chunk_attention_config=dual_chunk_attention_config, - ) - else: - self.attn = Attention( - self.num_heads, - self.head_dim, - self.scaling, - num_kv_heads=self.num_kv_heads, - cache_config=cache_config, - quant_config=quant_config, - per_layer_sliding_window=per_layer_sliding_window, - prefix=f"{prefix}.attn", - ) - - self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) - self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) - - -class MellumDecoderLayer(Qwen3MoeDecoderLayer): - """Differences from `Qwen3MoeDecoderLayer`: - - Supports interleaved SWA and per-layer RoPE scaling. - """ - - def __init__(self, vllm_config: VllmConfig, prefix: str = "") -> None: - nn.Module.__init__(self) - - config = vllm_config.model_config.hf_text_config - cache_config = vllm_config.cache_config - quant_config = vllm_config.quant_config - - self.hidden_size = config.hidden_size - max_position_embeddings = getattr(config, "max_position_embeddings", 8192) - dual_chunk_attention_config = getattr( - config, "dual_chunk_attention_config", None - ) - - layer_idx = extract_layer_index(prefix) - layer_type = config.layer_types[layer_idx] - if layer_type == "sliding_attention": - sliding_window = getattr(config, "sliding_window", None) - else: - sliding_window = None - rope_parameters = config.rope_parameters[layer_type] - - self.self_attn = MellumAttention( - hidden_size=self.hidden_size, - num_heads=config.num_attention_heads, - num_kv_heads=config.num_key_value_heads, - rope_parameters=rope_parameters, - max_position_embeddings=max_position_embeddings, - rms_norm_eps=config.rms_norm_eps, - qkv_bias=getattr(config, "attention_bias", False), - head_dim=getattr(config, "head_dim", None), - cache_config=cache_config, - quant_config=quant_config, - prefix=f"{prefix}.self_attn", - dual_chunk_attention_config=dual_chunk_attention_config, - per_layer_sliding_window=sliding_window, - ) - - if config.mlp_layer_types[layer_idx] == "sparse": - self.mlp = Qwen3MoeSparseMoeBlock( - vllm_config=vllm_config, prefix=f"{prefix}.mlp" - ) - else: - self.mlp = Qwen3MoeMLP( - hidden_size=config.hidden_size, - intermediate_size=config.intermediate_size, - hidden_act=config.hidden_act, - quant_config=quant_config, - prefix=f"{prefix}.mlp", - ) - - self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.post_attention_layernorm = RMSNorm( - config.hidden_size, eps=config.rms_norm_eps - ) - - -@support_torch_compile -class MellumModel(Qwen3MoeModel): - """Differences from `Qwen3MoeModel`: - - Uses `MellumDecoderLayer`. - """ - - def __init__( - self, - *, - vllm_config: VllmConfig, - prefix: str = "", - ): - super().__init__( - vllm_config=vllm_config, - prefix=prefix, - decoder_layer_type=MellumDecoderLayer, - ) - - -class MellumForCausalLM(Qwen3MoeForCausalLM): - """Differences from `Qwen3MoeForCausalLM`: - - Uses `MellumModel`. - """ - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - nn.Module.__init__(self) - config = vllm_config.model_config.hf_text_config - quant_config = vllm_config.quant_config - self.config = config - self.quant_config = quant_config - if "dense" in getattr(config, "mlp_layer_types", []): - self.packed_modules_mapping["gate_up_proj"] = ["gate_proj", "up_proj"] - self.model = MellumModel( - vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") - ) - self.lm_head = ParallelLMHead( - config.vocab_size, - config.hidden_size, - quant_config=quant_config, - prefix=maybe_prefix(prefix, "lm_head"), - ) - if self.config.tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) - self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( - self.model.make_empty_intermediate_tensors - ) - - self.moe_layers = [] - example_layer = None - for layer in self.model.layers: - if isinstance(layer, PPMissingLayer): - continue - - assert isinstance(layer, Qwen3MoeDecoderLayer) - if isinstance(layer.mlp, Qwen3MoeSparseMoeBlock): - example_layer = layer.mlp - self.moe_layers.append(layer.mlp.experts) - - if example_layer is None: - raise RuntimeError("No MoE layer found in the model.layers.") - - self.num_moe_layers = len(self.moe_layers) - self.num_expert_groups = 1 - self.num_shared_experts = 0 - self.num_logical_experts = example_layer.n_logical_experts - self.num_physical_experts = example_layer.n_physical_experts - self.num_local_physical_experts = example_layer.n_local_physical_experts - self.num_routed_experts = example_layer.n_routed_experts - self.num_redundant_experts = example_layer.n_redundant_experts diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index b435d77ad7..81763fed40 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -72,7 +72,6 @@ _TEXT_GENERATION_MODELS = { # [Decoder-only] "AfmoeForCausalLM": ("afmoe", "AfmoeForCausalLM"), "ApertusForCausalLM": ("apertus", "ApertusForCausalLM"), - "ArceeForCausalLM": ("arcee", "ArceeForCausalLM"), "AXK1ForCausalLM": ("AXK1", "AXK1ForCausalLM"), "BailingMoeForCausalLM": ("bailing_moe", "BailingMoeForCausalLM"), "BailingMoeV2ForCausalLM": ("bailing_moe", "BailingMoeV2ForCausalLM"), @@ -84,7 +83,6 @@ _TEXT_GENERATION_MODELS = { "CohereForCausalLM": ("commandr", "CohereForCausalLM"), "Cohere2ForCausalLM": ("commandr", "CohereForCausalLM"), "Cohere2MoeForCausalLM": ("cohere2_moe", "Cohere2MoeForCausalLM"), - "CwmForCausalLM": ("llama", "LlamaForCausalLM"), "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"), "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"), "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"), @@ -114,7 +112,6 @@ _TEXT_GENERATION_MODELS = { "vllm.models.qwen4_exp", "Qwen4ExpForCausalLM", ), - "GlmForCausalLM": ("glm", "GlmForCausalLM"), "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"), "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"), "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"), @@ -160,7 +157,6 @@ _TEXT_GENERATION_MODELS = { ), "MambaForCausalLM": ("mamba", "MambaForCausalLM"), "Mamba2ForCausalLM": ("mamba2", "Mamba2ForCausalLM"), - "MellumForCausalLM": ("mellum", "MellumForCausalLM"), "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"), "MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"), "MiniMaxM2ForCausalLM": ("minimax_m2", "MiniMaxM2ForCausalLM"), @@ -226,7 +222,6 @@ _EMBEDDING_MODELS = { "DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"), "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"), "Gemma3TextModel": ("gemma3", "Gemma3Model"), - "GlmForCausalLM": ("glm", "GlmForCausalLM"), "GteModel": ("bert_with_rope", "SnowflakeGteNewModel"), "GteNewModel": ("bert_with_rope", "GteNewModel"), "JinaEmbeddingsV5Model": ("jina", "JinaEmbeddingsV5Model"), @@ -709,10 +704,14 @@ _SPECULATIVE_DECODING_MODELS = { _TRANSFORMERS_SUPPORTED_MODELS = { # Text generation models + "ArceeForCausalLM": ("transformers", "TransformersForCausalLM"), + "CwmForCausalLM": ("transformers", "TransformersForCausalLM"), "FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"), + "GlmForCausalLM": ("transformers", "TransformersForCausalLM"), "GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"), "HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"), "HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"), + "MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"), "NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"), "OlmoForCausalLM": ("transformers", "TransformersForCausalLM"), "Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"), From fcfacc3499b3f45978fc92a1b300f817cd17bb77 Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Fri, 2 Oct 2026 04:17:56 +0100 Subject: [PATCH 238/248] [Bugfix][Frontend] Seed derender detokenization from the prompt (#59046) Signed-off-by: Martin Hickey --- docs/serving/online_serving/derenderer.md | 12 +- requirements/common.txt | 2 +- .../scale_out/derender/test_derender.py | 135 ++++++ .../derender/test_derender_stream.py | 429 ++++++++++++++---- .../entrypoints/scale_out/derender/serving.py | 33 +- .../scale_out/token_in_token_out/protocol.py | 48 +- vllm/renderers/online_derenderer.py | 247 +++++++--- vllm/tokenizers/detokenizer_utils.py | 4 +- vllm/v1/engine/detokenizer.py | 13 +- 9 files changed, 760 insertions(+), 163 deletions(-) diff --git a/docs/serving/online_serving/derenderer.md b/docs/serving/online_serving/derenderer.md index e6d1221fd4..cf57f5d8b2 100644 --- a/docs/serving/online_serving/derenderer.md +++ b/docs/serving/online_serving/derenderer.md @@ -23,6 +23,10 @@ endpoints. The derender step needs more than the engine's `token_ids`. It also consumes the original `chat_request`/`completion_request` and `prompt_tokens` carried over from the render step (see [Request format](#request-format)) so the tool and reasoning parsers have the context they need. The chat streaming path additionally needs `prompt_token_ids` (see [Streaming cost](#streaming-cost)) when a parser is configured. +All derender endpoints also accept an optional `prompt_token_ids` (the `token_ids` of the `GenerateRequest` returned by `/render`). It's used to seed detokenization from the end of the prompt, the same way the engine does. Without it, the first output token is decoded as if it started a new sequence. On SentencePiece tokenizers (Llama-2, Phi-3) that drops its leading space compared to `/v1/chat/completions` and `/v1/completions`. If it's omitted, derender falls back to the generate response's `prompt_token_ids` (set when the `GenerateRequest` has `return_token_ids`), then to decoding without prompt context. + +Seeding only reads the last few IDs (7 today), so a suffix of the prompt is enough and a long prompt doesn't need to travel back to derender. The exception is streaming chat with a tool or reasoning parser configured. There the parser also reads `prompt_token_ids` to decide whether the prompt left reasoning open and it can look back to any point in the prompt. In this case, send the full list (see [Streaming cost](#streaming-cost)). + ## API Reference - Chat Completions Derender API (`/v1/chat/completions/derender`) @@ -96,6 +100,7 @@ The server keeps no state between calls. Everything the next call needs is in `s - **`n > 1` is N streams.** Generate interleaves chunks for different choices and tags each with its `index`. Keep one `stream_state` per index and send each chunk with the state for its index. Each index gets its own `role` delta on its first chunk. - **Send every chunk through, including the last ones.** The finish reason usually arrives with the final tokens. A chunk with a `finish_reason` and no tokens is accepted too and flushes any buffered tool call arguments. With `stream_options: {"include_usage": true}`, generate ends with a usage only chunk (`choices: []`). Send it through derender like any other chunk to get the usage chunk for the response. `usage.prompt_tokens` is the request's `prompt_tokens` if set, otherwise the generate chunk's. - **Send the same context on every call.** `chat_request` and `prompt_token_ids` aren't kept between calls, so they go with every chunk including the usage chunk. Both are required when a tool or reasoning parser is configured. `prompt_token_ids` is the `token_ids` of the `GenerateRequest` returned by `/render`. +- **Send `prompt_token_ids` on the first chunk, even without a parser.** It seeds detokenization so the first token's leading space matches the coupled endpoint (see [Request format](#request-format)). Later chunks already carry that context in `stream_state`. - **Don't forward `[DONE]`.** It marks the end of the generate stream and isn't a chunk. Streaming derender resolves `logprobs` on the plain detokenization path, including `token_id:N` placeholders, same as the non-streaming endpoints (see [Streaming state and logprobs](#streaming-state-and-logprobs)). The parser path doesn't resolve them yet and drops them. @@ -110,7 +115,7 @@ When a tool or reasoning parser is configured, parser internal state (buffered m - **Transport**: `output_token_ids` and `output_chunk_lens` round-trip in full in both directions on every call. `output_chunk_lens` has one entry per chunk, which is one per token without speculative decoding. This means O(n) bytes per chunk, O(n²) bytes over a full generation. Bounded by `max_model_len`. `prompt_token_ids` is sent in full on every call too and it isn't trimmed as `output_token_ids` grows. This means that for most of a stream it dominates the per chunk payload. A 100k token prompt with 1k tokens of output means `prompt_token_ids` is ~99% of the request body on every chunk. - **Compute**: replay is O(n) `parse_delta` calls per chunk (O(n²) per generation). `parse_delta` itself is O(n) for parsers that re-scan accumulated text (e.g. Hermes tool-call JSON, DeepSeek-R1 reasoning). The per-generation cost is O(n³) character work, not O(n²). This is a deliberately minimal first implementation with no caching layer. -- The parser path also requires `prompt_token_ids` so `parse_delta` can settle whether the prompt left reasoning open or not. Since parser state can't be carried across calls, it re-scans the full prompt once per chunk. +- The parser path also requires `prompt_token_ids` so `parse_delta` can settle whether the prompt left reasoning open or not. Since parser state can't be carried across calls, it re-scans the full prompt once per chunk. Send the full list and not a suffix because the last reasoning marker can be anywhere in the prompt. - Replay runs off the event loop on the renderer's executor (`renderer_num_workers`, default `1`). Size it for the expected number of concurrent parser configured streams. `output_token_ids` and `prompt_token_ids` are both bounded by `max_model_len` but callers streaming long reasoning traces through a parser configured model should expect materially more state transport and CPU cost than the plain detokenization path. @@ -153,7 +158,7 @@ with httpx.Client(timeout=60.0) as client: generate_request = client.post( f"{RENDER}/v1/chat/completions/render", json=chat_request ).json() - prompt_tokens = len(generate_request["token_ids"]) + prompt_token_ids = generate_request["token_ids"] # 2. Generate: token IDs -> token IDs (token-in / token-out engine) generate_response = client.post( @@ -166,7 +171,8 @@ with httpx.Client(timeout=60.0) as client: json={ "model": MODEL, "generate_response": generate_response, - "prompt_tokens": prompt_tokens, + "prompt_tokens": len(prompt_token_ids), + "prompt_token_ids": prompt_token_ids, "chat_request": chat_request, }, ).json() diff --git a/requirements/common.txt b/requirements/common.txt index 70bfa93253..8e483cc712 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -9,7 +9,7 @@ blake3 py-cpuinfo transformers >= 5.16.1 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export -tokenizers >= 0.21.1 # Required for fast incremental detokenization. +tokenizers >= 0.22.0 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611 protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.*; platform_machine != "s390x" protobuf == 7.36.1; platform_machine == "s390x" diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index 80ecc0f49c..659aaed1aa 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -27,6 +27,11 @@ async def client(server): yield http_client +@pytest.fixture(scope="module") +def tokenizer(): + return get_tokenizer(MODEL_NAME) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -372,6 +377,58 @@ async def test_derender_chat_model_omitted_resolves_served_name(client): assert response.json()["model"] == MODEL_NAME +@pytest.mark.asyncio +async def test_derender_chat_leading_space_seeded_from_prompt(client, tokenizer): + """prompt_token_ids keeps the first token's leading space on this + Metaspace tokenizer, matching the coupled path.""" + gen_req = await _render_chat(client) + prompt_token_ids = gen_req["token_ids"] + output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False) + + unseeded = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(output_ids), + }, + ) + seeded = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(output_ids), + "prompt_token_ids": prompt_token_ids, + }, + ) + assert unseeded.status_code == 200 + assert seeded.status_code == 200 + unseeded_content = unseeded.json()["choices"][0]["message"]["content"] + seeded_content = seeded.json()["choices"][0]["message"]["content"] + + assert not unseeded_content.startswith(" ") + assert seeded_content.startswith(" ") + assert seeded_content.lstrip(" ") == unseeded_content + + +@pytest.mark.asyncio +async def test_derender_chat_oversized_prompt_token_ids_rejected(client): + """prompt_token_ids longer than max_model_len returns 400.""" + gen_req = await _render_chat(client) + synthetic_ids = gen_req["token_ids"][:3] + oversized_prompt_ids = [42] * 1_000_000 + + response = await client.post( + "/v1/chat/completions/derender", + json={ + "model": MODEL_NAME, + "generate_response": _make_generate_response(synthetic_ids), + "prompt_token_ids": oversized_prompt_ids, + }, + ) + assert response.status_code == 400 + assert "max_model_len" in response.json()["error"]["message"] + + # --------------------------------------------------------------------------- # Completion derender tests # --------------------------------------------------------------------------- @@ -486,6 +543,84 @@ async def test_derender_completion_prompt_tokens_length_mismatch(client): assert response.status_code == 400 +@pytest.mark.asyncio +async def test_derender_completion_prompt_token_ids_length_mismatch(client): + """len(prompt_token_ids) != len(generate_responses) returns 400, the + same as the existing prompt_tokens length check.""" + gr1 = await _render_completion(client, "Hello") + ids1 = gr1["token_ids"][:3] + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(ids1, gr1["request_id"]), + ], + "prompt_token_ids": [[1, 2], [3, 4]], + }, + ) + assert response.status_code == 400 + + +@pytest.mark.asyncio +async def test_derender_completion_leading_space_seeded_from_prompt(client, tokenizer): + """Completions counterpart of + test_derender_chat_leading_space_seeded_from_prompt.""" + gr1 = await _render_completion(client, "Hello world") + prompt_token_ids = gr1["token_ids"] + output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False) + + unseeded = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(output_ids, gr1["request_id"]), + ], + }, + ) + seeded = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(output_ids, gr1["request_id"]), + ], + "prompt_token_ids": [prompt_token_ids], + }, + ) + assert unseeded.status_code == 200 + assert seeded.status_code == 200 + unseeded_text = unseeded.json()["choices"][0]["text"] + seeded_text = seeded.json()["choices"][0]["text"] + + assert not unseeded_text.startswith(" ") + assert seeded_text.startswith(" ") + assert seeded_text.lstrip(" ") == unseeded_text + + +@pytest.mark.asyncio +async def test_derender_completion_oversized_prompt_token_ids_rejected(client): + """A prompt_token_ids entry longer than max_model_len returns 400.""" + gr1 = await _render_completion(client, "Hello") + ids1 = gr1["token_ids"][:3] + oversized_prompt_ids = [42] * 1_000_000 + + response = await client.post( + "/v1/completions/derender", + json={ + "model": MODEL_NAME, + "generate_responses": [ + _make_completion_generate_response(ids1, gr1["request_id"]), + ], + "prompt_token_ids": [oversized_prompt_ids], + }, + ) + assert response.status_code == 400 + assert "max_model_len" in response.json()["error"]["message"] + + @pytest.mark.asyncio async def test_derender_completion_empty_generate_responses(client): """Empty generate_responses list returns 400.""" diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index b1db1193a2..5862256835 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -20,11 +20,17 @@ Tests are split into two layers: import json import threading from concurrent.futures import ThreadPoolExecutor +from unittest.mock import AsyncMock, MagicMock +import httpx import pytest import pytest_asyncio +from pydantic import ValidationError +from tokenizers.decoders import DecodeStream +import vllm.renderers.online_derenderer as online_derenderer_module from tests.entrypoints.scale_out.derender.utils import stream_chat_derender +from tests.utils import RemoteLaunchRenderServer from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, @@ -32,7 +38,17 @@ from vllm.entrypoints.generate.base.protocol import ( PerRequestMetrics, SpeculativeDecodingMetrics, ) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionStreamResponse, +) +from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.scale_out.derender.serving import ServingDerender from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( + DerenderChatRequest, + DerenderChatStreamRequest, + DerenderCompletionRequest, + DerenderCompletionStreamRequest, DerenderStreamState, GenerateResponse, GenerateResponseChoice, @@ -41,6 +57,12 @@ from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( ) from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo from vllm.parser import Parser +from vllm.renderers.online_derenderer import ( + OnlineDerenderer, + _decode_params, + _seed_stream_state, +) +from vllm.tokenizers import get_tokenizer from vllm.utils import random_uuid MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM" @@ -170,6 +192,7 @@ def _make_stream_chunk( usage: dict | None = None, metrics: PerRequestMetrics | None = None, logprobs: dict | None = None, + prompt_token_ids: list[int] | None = None, ) -> GenerateStreamResponse: """Build a GenerateStreamResponse SSE chunk.""" return GenerateStreamResponse( @@ -184,6 +207,7 @@ def _make_stream_chunk( ], usage=UsageInfo(**usage) if usage else None, metrics=metrics, + prompt_token_ids=prompt_token_ids, ) @@ -228,18 +252,12 @@ def _make_usage_chunk( @pytest.fixture(scope="module") def tokenizer(): """Load the tiny tokenizer used across unit tests.""" - from vllm.tokenizers import get_tokenizer - return get_tokenizer(MODEL_NAME) @pytest.fixture(scope="module") def derenderer(tokenizer, request): """Construct a minimal OnlineDerenderer backed by a stub renderer.""" - from unittest.mock import MagicMock - - from vllm.renderers.online_derenderer import OnlineDerenderer - renderer = MagicMock() renderer.get_tokenizer.return_value = tokenizer @@ -273,10 +291,6 @@ def parsed_derenderer(tokenizer): `renderer._executor` must be a real `ThreadPoolExecutor`. `loop.run_in_executor` cannot submit work to a `MagicMock`. """ - from unittest.mock import MagicMock - - from vllm.renderers.online_derenderer import OnlineDerenderer - renderer = MagicMock() renderer.get_tokenizer.return_value = tokenizer renderer._executor = ThreadPoolExecutor(max_workers=2) @@ -433,6 +447,57 @@ class TestDetokenizeDelta: # Bounded by a small constant independent of len(token_ids) assert max_window <= 32 + def test_truncated_multibyte_char_is_held_back(self, derenderer, tokenizer): + """Output ending mid-character emits no U+FFFD: batch == stream == engine.""" + emoji_ids = tokenizer.encode("🌈", add_special_tokens=False) + byte_ids = emoji_ids[-4:] + assert len(byte_ids) == 4 + output_ids = byte_ids[:-1] + + batch, _ = derenderer._detokenize_delta( + tokenizer, output_ids, DerenderStreamState() + ) + streamed = self._chunked(derenderer, tokenizer, [[t] for t in output_ids]) + engine = _decode_stream_expected(tokenizer, [], output_ids) + + assert "�" not in batch + assert batch == streamed == engine + + full, _ = derenderer._detokenize_delta( + tokenizer, byte_ids, DerenderStreamState() + ) + assert full.endswith("🌈") + + @pytest.mark.parametrize( + ("skip", "spaces", "preserve", "fast", "expected"), + [ + (True, True, False, True, (True, True)), + (True, False, False, True, (True, True)), + (True, False, False, False, (True, False)), + (False, False, False, True, (False, False)), + (False, False, True, True, (False, False)), + (True, False, True, True, (False, False)), + (True, True, True, True, (False, True)), + ], + ) + def test_decode_params_mirror_engine( + self, monkeypatch, skip, spaces, preserve, fast, expected + ): + """Flags match the engine's fast or slow detokenizer for the tokenizer.""" + monkeypatch.setattr( + online_derenderer_module, "uses_fast_detokenizer", lambda _: fast + ) + request = CompletionRequest( + model=MODEL_NAME, + prompt="x", + skip_special_tokens=skip, + spaces_between_special_tokens=spaces, + ) + assert _decode_params(None, request, preserve_special=preserve) == expected + + def test_decode_params_default_without_request(self, tokenizer): + assert _decode_params(tokenizer, None) == (True, True) + def test_n_independent_streams_same_result(self, derenderer, tokenizer): """N parallel streams with the same token sequence give the same text.""" token_ids = tokenizer.encode("parallel streams")[:8] @@ -451,6 +516,242 @@ class TestDetokenizeDelta: assert results[0] == self._one_shot(tokenizer, token_ids) +# --------------------------------------------------------------------------- +# Prompt seeded detokenization (matches the engine's primed detokenizer) +# --------------------------------------------------------------------------- + + +def _decode_stream_expected( + tokenizer, prompt_ids: list[int], output_ids: list[int] +) -> str: + """Engine-equivalent decode via `DecodeStream`, primed by stepping the + prompt (`DecodeStream(ids=...)` needs tokenizers >= 0.22).""" + stream = DecodeStream(skip_special_tokens=True) + for tid in prompt_ids: + stream.step(tokenizer.backend_tokenizer, tid) + return "".join( + s + for tid in output_ids + if (s := stream.step(tokenizer.backend_tokenizer, tid)) is not None + ) + + +@pytest.fixture(scope="module") +def leading_space_ids(tokenizer): + """(prompt_ids, output_ids, expected) where seeded and unseeded decode + differ by the leading space.""" + prompt_ids = tokenizer.encode("[INST] Hi [/INST]", add_special_tokens=False) + output_ids = tokenizer.encode("Hello there, output", add_special_tokens=False) + expected = _decode_stream_expected(tokenizer, prompt_ids, output_ids) + unseeded = tokenizer.decode(output_ids, skip_special_tokens=True) + assert expected.startswith(" ") and not unseeded.startswith(" "), ( + "fixture no longer reproduces the leading-space divergence; " + "pick a different prompt/output pair" + ) + return prompt_ids, output_ids, expected + + +class TestSeedStreamState: + """`_seed_stream_state`: pure helper building the initial decode state.""" + + def test_seeds_prev_tokens_from_prompt_tail(self, tokenizer, leading_space_ids): + prompt_ids, _, _ = leading_space_ids + state = _seed_stream_state(tokenizer, prompt_ids, skip_special_tokens=True) + assert state.prev_tokens + assert state.read_offset == len(state.prev_tokens) + + def test_none_prompt_token_ids_returns_empty_state(self, tokenizer): + assert ( + _seed_stream_state(tokenizer, None, skip_special_tokens=True) + == DerenderStreamState() + ) + + def test_empty_prompt_token_ids_returns_empty_state(self, tokenizer): + assert ( + _seed_stream_state(tokenizer, [], skip_special_tokens=True) + == DerenderStreamState() + ) + + +class TestPromptSeededLeadingSpace: + """Every decode entry point keeps the first token's leading space when + given prompt_token_ids.""" + + @pytest.mark.asyncio + async def test_chat_batch_seeded_via_explicit_prompt_token_ids( + self, derenderer, leading_space_ids + ): + prompt_ids, output_ids, expected = leading_space_ids + choices = await derenderer.derender_chat( + GenerateResponse( + request_id="t", + choices=[ + GenerateResponseChoice( + index=0, token_ids=output_ids, finish_reason="stop" + ) + ], + ), + prompt_token_ids=prompt_ids, + ) + assert choices[0].message.content == expected + + @pytest.mark.asyncio + async def test_chat_batch_seeded_via_generate_response_fallback( + self, derenderer, leading_space_ids + ): + prompt_ids, output_ids, expected = leading_space_ids + choices = await derenderer.derender_chat( + GenerateResponse( + request_id="t", + choices=[ + GenerateResponseChoice( + index=0, token_ids=output_ids, finish_reason="stop" + ) + ], + prompt_token_ids=prompt_ids, + ), + ) + assert choices[0].message.content == expected + + @pytest.mark.asyncio + async def test_chat_batch_explicit_takes_precedence_over_fallback( + self, derenderer, tokenizer, leading_space_ids + ): + prompt_ids, output_ids, expected = leading_space_ids + wrong_ids = tokenizer.encode("a different prompt", add_special_tokens=False) + choices = await derenderer.derender_chat( + GenerateResponse( + request_id="t", + choices=[ + GenerateResponseChoice( + index=0, token_ids=output_ids, finish_reason="stop" + ) + ], + prompt_token_ids=wrong_ids, + ), + prompt_token_ids=prompt_ids, + ) + assert choices[0].message.content == expected + + @pytest.mark.asyncio + async def test_chat_batch_without_prompt_context_stays_unseeded( + self, derenderer, leading_space_ids + ): + _, output_ids, _ = leading_space_ids + choices = await derenderer.derender_chat( + GenerateResponse( + request_id="t", + choices=[ + GenerateResponseChoice( + index=0, token_ids=output_ids, finish_reason="stop" + ) + ], + ), + ) + assert not choices[0].message.content.startswith(" ") + + @pytest.mark.asyncio + async def test_completion_batch_seeded(self, derenderer, leading_space_ids): + prompt_ids, output_ids, expected = leading_space_ids + choices, _, _ = await derenderer.derender_completion( + [ + GenerateResponse( + request_id="t", + choices=[ + GenerateResponseChoice( + index=0, token_ids=output_ids, finish_reason="stop" + ) + ], + ) + ], + prompt_token_ids=[prompt_ids], + ) + assert choices[0].text == expected + + @pytest.mark.asyncio + @pytest.mark.parametrize("from_generate_chunk", [False, True]) + async def test_chat_stream_first_chunk_seeded( + self, derenderer, leading_space_ids, from_generate_chunk + ): + prompt_ids, output_ids, expected = leading_space_ids + mid = len(output_ids) // 2 + + chunk1, state = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + output_ids[:mid], + prompt_token_ids=prompt_ids if from_generate_chunk else None, + ), + prompt_token_ids=None if from_generate_chunk else prompt_ids, + ) + # Only needed on the first chunk; the carried state already holds + # the seeded window from here on. + chunk2, _ = await derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"), + state=state, + ) + + streamed = (chunk1.choices[0].delta.content or "") + ( + chunk2.choices[0].delta.content or "" + ) + assert streamed == expected + + @pytest.mark.asyncio + @pytest.mark.parametrize("from_generate_chunk", [False, True]) + async def test_completion_stream_first_chunk_seeded( + self, derenderer, leading_space_ids, from_generate_chunk + ): + prompt_ids, output_ids, expected = leading_space_ids + mid = len(output_ids) // 2 + + chunk1, state = await derenderer.derender_completion_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + output_ids[:mid], + prompt_token_ids=prompt_ids if from_generate_chunk else None, + ), + prompt_token_ids=None if from_generate_chunk else prompt_ids, + ) + chunk2, _ = await derenderer.derender_completion_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk(output_ids[mid:], finish_reason="stop"), + state=state, + ) + + assert chunk1.choices[0].text + chunk2.choices[0].text == expected + + @pytest.mark.asyncio + async def test_parser_path_ephemeral_state_seeded( + self, parsed_derenderer, monkeypatch, leading_space_ids + ): + """The parser path's per-call detok window is seeded too, so + `parse_delta` sees the same text as coupled serving.""" + + class _EchoParser(Parser): + def parse_delta( + self, + delta_text, + delta_token_ids, + request, + prompt_token_ids=None, + *, + finished, + ): + return DeltaMessage(content=delta_text) if delta_text else None + + monkeypatch.setattr(parsed_derenderer, "parser", _EchoParser) + prompt_ids, output_ids, expected = leading_space_ids + + chunk, _ = await parsed_derenderer.derender_chat_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk(output_ids, finish_reason="stop"), + chat_request=_chat_request(), + prompt_token_ids=prompt_ids, + ) + assert chunk.choices[0].delta.content == expected + + @pytest.mark.asyncio @pytest.mark.parametrize( "stream_method", @@ -460,8 +761,6 @@ async def test_streaming_detokenization_runs_off_event_loop( derenderer, monkeypatch, stream_method ): """Streaming detokenization runs on the renderer executor.""" - import vllm.renderers.online_derenderer as online_derenderer_module - event_loop_thread_id = threading.get_ident() executor_thread_id = derenderer.renderer._executor.submit( threading.get_ident @@ -561,8 +860,6 @@ class TestDerenderCompletionStream: @pytest.mark.asyncio async def test_skip_special_tokens_threaded(self, derenderer, tokenizer): """completion_request.skip_special_tokens is honored (not hardcoded True).""" - from vllm.entrypoints.openai.completion.protocol import CompletionRequest - eos = tokenizer.eos_token_id if eos is None: pytest.skip("tokenizer has no eos token to exercise special stripping") @@ -789,8 +1086,6 @@ class TestDerenderChatStream: def _chat_request(**kwargs): - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - kwargs.setdefault("messages", [{"role": "user", "content": "hi"}]) kwargs.setdefault("model", MODEL_NAME) return ChatCompletionRequest(**kwargs) @@ -1072,8 +1367,6 @@ def harmony_encode(): @pytest.fixture(scope="module") def harmony_tokenizer(): pytest.importorskip("openai_harmony") - from vllm.tokenizers import get_tokenizer - return get_tokenizer(HARMONY_MODEL, trust_remote_code=True) @@ -1085,10 +1378,7 @@ def harmony_derenderer(harmony_tokenizer): with a real tokenizer and parser, so the replay path is exercised against Harmony's actual channel grammar rather than a stub. """ - from unittest.mock import MagicMock - from vllm.parser.harmony import HarmonyParser - from vllm.renderers.online_derenderer import OnlineDerenderer renderer = MagicMock() renderer.get_tokenizer.return_value = harmony_tokenizer @@ -1224,20 +1514,14 @@ class TestDerenderStreamStateValidation: """DerenderStreamState rejects malformed caller supplied offsets/lengths.""" def test_negative_prefix_offset_rejected(self): - from pydantic import ValidationError - with pytest.raises(ValidationError): DerenderStreamState(prefix_offset=-1) def test_negative_read_offset_rejected(self): - from pydantic import ValidationError - with pytest.raises(ValidationError): DerenderStreamState(read_offset=-1) def test_prev_tokens_over_cap_rejected(self): - from pydantic import ValidationError - with pytest.raises(ValidationError): DerenderStreamState(prev_tokens=["a"] * 1025) @@ -1246,14 +1530,10 @@ class TestDerenderStreamStateValidation: assert len(state.prev_tokens) == 1024 def test_output_chunk_lens_mismatch_rejected(self): - from pydantic import ValidationError - with pytest.raises(ValidationError): DerenderStreamState(output_token_ids=[1, 2, 3], output_chunk_lens=[1, 1]) def test_output_chunk_lens_zero_entry_rejected(self): - from pydantic import ValidationError - with pytest.raises(ValidationError): DerenderStreamState(output_token_ids=[1, 2], output_chunk_lens=[0, 2]) @@ -1264,14 +1544,40 @@ class TestDerenderStreamStateValidation: assert state.output_chunk_lens == [1, 2] +_GEN = {"request_id": "t", "choices": []} +_GEN_BAD = {**_GEN, "prompt_token_ids": [-1]} + + +@pytest.mark.parametrize( + "model_cls,payload", + [ + (DerenderChatRequest, {"generate_response": _GEN, "prompt_token_ids": [-1]}), + (DerenderChatRequest, {"generate_response": _GEN_BAD}), + ( + DerenderCompletionRequest, + {"generate_responses": [_GEN], "prompt_token_ids": [[-1]]}, + ), + ( + DerenderChatStreamRequest, + {"stream": True, "generate_chunk": _GEN, "prompt_token_ids": [-1]}, + ), + ( + DerenderCompletionStreamRequest, + {"stream": True, "generate_chunk": _GEN_BAD}, + ), + ], +) +def test_negative_prompt_token_ids_rejected(model_cls, payload): + """Negative prompt ids would reach convert_ids_to_tokens, so they get a + 400 at parse time like `GenerateRequest.token_ids` does.""" + with pytest.raises(ValidationError, match="prompt_token_ids"): + model_cls.model_validate(payload) + + class TestServingDerenderStreamErrorHandling: """Malformed stream_state must surface as 400 and not an unhandled 500.""" def _make_serving(self, side_effect: Exception): - from unittest.mock import AsyncMock, MagicMock - - from vllm.entrypoints.scale_out.derender.serving import ServingDerender - models = MagicMock() models.is_base_model.return_value = True models.model_config = MagicMock() @@ -1289,10 +1595,6 @@ class TestServingDerenderStreamErrorHandling: @pytest.mark.asyncio @pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")]) async def test_completion_stream_bad_state_returns_400(self, exc): - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderCompletionStreamRequest, - ) - serving = self._make_serving(exc) request = DerenderCompletionStreamRequest( stream=True, @@ -1307,10 +1609,6 @@ class TestServingDerenderStreamErrorHandling: @pytest.mark.asyncio @pytest.mark.parametrize("exc", [KeyError("bad byte"), IndexError("oob")]) async def test_chat_stream_bad_state_returns_400(self, exc): - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(exc) request = DerenderChatStreamRequest( stream=True, @@ -1329,13 +1627,6 @@ class TestServingDerenderStreamValidation: before touching the tokenizer).""" def _make_serving(self, *, parser_configured: bool, max_model_len: int = 100_000): - from unittest.mock import AsyncMock, MagicMock - - from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionStreamResponse, - ) - from vllm.entrypoints.scale_out.derender.serving import ServingDerender - models = MagicMock() models.is_base_model.return_value = True models.model_config = MagicMock() @@ -1353,10 +1644,6 @@ class TestServingDerenderStreamValidation: @pytest.mark.asyncio async def test_missing_chat_request_with_parser_rejected(self): - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=True) request = DerenderChatStreamRequest( stream=True, @@ -1371,10 +1658,6 @@ class TestServingDerenderStreamValidation: @pytest.mark.asyncio async def test_missing_chat_request_without_parser_ok(self): - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=False) request = DerenderChatStreamRequest( stream=True, @@ -1390,10 +1673,6 @@ class TestServingDerenderStreamValidation: the same way it rejects a missing chat_request. Without it, parse_delta cannot tell whether the prompt left reasoning open and would misclassify reasoning content as plain content.""" - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=True) request = DerenderChatStreamRequest( stream=True, @@ -1409,10 +1688,6 @@ class TestServingDerenderStreamValidation: @pytest.mark.asyncio async def test_prompt_token_ids_present_with_parser_ok(self): - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=True) request = DerenderChatStreamRequest( stream=True, @@ -1426,10 +1701,6 @@ class TestServingDerenderStreamValidation: @pytest.mark.asyncio async def test_oversized_output_token_ids_rejected(self): - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=False, max_model_len=4) request = DerenderChatStreamRequest( stream=True, @@ -1451,10 +1722,6 @@ class TestServingDerenderStreamValidation: parser configured deployment rescans it in full on every chunk (is_reasoning_end / adjust_initial_state_from_prompt), so it must be bounded the same way output_token_ids is.""" - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=True, max_model_len=4) request = DerenderChatStreamRequest( stream=True, @@ -1475,10 +1742,6 @@ class TestServingDerenderStreamValidation: DerenderStreamState is threaded through every choice). The check could never fire since derender_chat_stream itself rejects anything above 1 first.""" - from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( - DerenderChatStreamRequest, - ) - serving = self._make_serving(parser_configured=False) two_choices = GenerateStreamResponse( request_id="t", @@ -1503,16 +1766,12 @@ class TestServingDerenderStreamValidation: @pytest.fixture(scope="module") def server(): - from tests.utils import RemoteLaunchRenderServer - with RemoteLaunchRenderServer(MODEL_NAME, []) as remote_server: yield remote_server @pytest_asyncio.fixture async def client(server): - import httpx - async with httpx.AsyncClient( base_url=server.url_for(""), timeout=30.0 ) as http_client: @@ -1776,8 +2035,6 @@ _PARSER_TOOLS = [ @pytest.fixture(scope="module") def parser_server(): - from tests.utils import RemoteLaunchRenderServer - args = [ "--enable-auto-tool-choice", "--tool-call-parser", @@ -1791,8 +2048,6 @@ def parser_server(): @pytest_asyncio.fixture async def parser_client(parser_server): - import httpx - async with httpx.AsyncClient( base_url=parser_server.url_for(""), timeout=60.0 ) as http_client: @@ -1801,8 +2056,6 @@ async def parser_client(parser_server): @pytest.fixture(scope="module") def parser_tokenizer(): - from vllm.tokenizers import get_tokenizer - return get_tokenizer(PARSER_MODEL) diff --git a/vllm/entrypoints/scale_out/derender/serving.py b/vllm/entrypoints/scale_out/derender/serving.py index 5da7366403..0783d3a78f 100644 --- a/vllm/entrypoints/scale_out/derender/serving.py +++ b/vllm/entrypoints/scale_out/derender/serving.py @@ -117,6 +117,20 @@ class ServingDerender(BaseServing): return None + def _validate_prompt_token_ids( + self, + prompt_token_ids: list[list[int] | None], + ) -> ErrorResponse | None: + """Reject caller-supplied prompt_token_ids longer than max_model_len.""" + max_model_len = self.model_config.max_model_len + for ids in prompt_token_ids: + if ids is not None and len(ids) > max_model_len: + return self.create_error_response( + f"prompt_token_ids length ({len(ids)}) exceeds " + f"max_model_len ({max_model_len})." + ) + return None + async def derender_chat_response( self, request: DerenderChatRequest, @@ -138,6 +152,10 @@ class ServingDerender(BaseServing): if bounds_error is not None: return bounds_error + prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids]) + if prompt_ids_error is not None: + return prompt_ids_error + if self.online_derenderer.parser is not None and request.chat_request is None: return self.create_error_response( "chat_request is required when a tool or reasoning parser is " @@ -147,7 +165,9 @@ class ServingDerender(BaseServing): try: choices = await self.online_derenderer.derender_chat( - request.generate_response, request.chat_request + request.generate_response, + request.chat_request, + request.prompt_token_ids, ) except ValueError as exc: return self.create_error_response(str(exc)) @@ -203,6 +223,11 @@ class ServingDerender(BaseServing): if bounds_error is not None: return bounds_error + if request.prompt_token_ids is not None: + prompt_ids_error = self._validate_prompt_token_ids(request.prompt_token_ids) + if prompt_ids_error is not None: + return prompt_ids_error + ( choices, total_prompt_tokens, @@ -211,6 +236,7 @@ class ServingDerender(BaseServing): request.generate_responses, request.prompt_tokens, completion_request=request.completion_request, + prompt_token_ids=request.prompt_token_ids, ) first = request.generate_responses[0] @@ -367,6 +393,10 @@ class ServingDerender(BaseServing): if error_check_ret is not None: return error_check_ret + prompt_ids_error = self._validate_prompt_token_ids([request.prompt_token_ids]) + if prompt_ids_error is not None: + return prompt_ids_error + model_name = request.model or self.models.model_name() try: ( @@ -378,6 +408,7 @@ class ServingDerender(BaseServing): state=request.stream_state, prompt_tokens=request.prompt_tokens, completion_request=request.completion_request, + prompt_token_ids=request.prompt_token_ids, ) except ValueError as exc: return self.create_error_response(str(exc)) diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 94604b013f..60cffd07c2 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -5,6 +5,7 @@ from typing import Annotated, Any, Literal, TypeAlias from pydantic import ( BaseModel, Field, + NonNegativeInt, PrivateAttr, field_validator, model_validator, @@ -362,7 +363,7 @@ class GenerateStreamResponse(BaseModel): ) choices: list[GenerateResponseStreamChoice] usage: UsageInfo | None = Field(default=None) - prompt_token_ids: list[int] | None = None + prompt_token_ids: list[NonNegativeInt] | None = None mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None metrics: PerRequestMetrics | None = None @@ -382,7 +383,7 @@ class GenerateResponse(BaseModel): usage: UsageInfo | None = Field(default=None) prompt_logprobs: list[dict[int, Logprob] | None] | None = None prompt_token_id_logprobs: str | None = None - prompt_token_ids: list[int] | None = None + prompt_token_ids: list[NonNegativeInt] | None = None mm_placeholders: dict[str, list[PlaceholderRangeInfo]] | None = None metrics: PerRequestMetrics | None = None @@ -421,6 +422,15 @@ class DerenderChatRequest(BaseModel): len(GenerateRequest.token_ids) from the render step. """ + prompt_token_ids: list[NonNegativeInt] | None = None + """Prompt token IDs (`GenerateRequest.token_ids` from /render). Seeds + detokenization from the prompt tail so the first output token keeps its + leading space on SentencePiece tokenizers. Falls back to + `generate_response.prompt_token_ids`, then to unseeded decoding. + + Only the last few IDs are read, so a suffix of the prompt is enough. + """ + chat_request: ChatCompletionRequest | None = None """The original (post-adjust_request) ChatCompletionRequest from /render. @@ -455,6 +465,14 @@ class DerenderCompletionRequest(BaseModel): If provided, len(prompt_tokens) must equal len(generate_responses). """ + prompt_token_ids: list[list[NonNegativeInt] | None] | None = None + """One prompt token ID list per response, used to seed detokenization. + See `DerenderChatRequest.prompt_token_ids`. + + If provided, len(prompt_token_ids) must equal len(generate_responses). + A `None` entry falls back to `generate_responses[i].prompt_token_ids`. + """ + completion_request: CompletionRequest | None = None """The original (post-adjust_request) CompletionRequest from /render. @@ -472,6 +490,13 @@ class DerenderCompletionRequest(BaseModel): f"prompt_tokens length ({len(self.prompt_tokens)}) must equal " f"generate_responses length ({len(self.generate_responses)})" ) + if self.prompt_token_ids is not None and len(self.prompt_token_ids) != len( + self.generate_responses + ): + raise ValueError( + f"prompt_token_ids length ({len(self.prompt_token_ids)}) must " + f"equal generate_responses length ({len(self.generate_responses)})" + ) return self @@ -627,7 +652,7 @@ class DerenderChatStreamRequest(BaseModel): prompt_tokens: int | None = None """Prompt token count for usage. Forwarded from the render step.""" - prompt_token_ids: list[int] | None = None + prompt_token_ids: list[NonNegativeInt] | None = None """Prompt token IDs. Required by the parser path's `parse_delta` to settle its initial reasoning state (e.g. chat templates that pre-open ````). `prompt_tokens` is a usage count and cannot serve this @@ -636,8 +661,15 @@ class DerenderChatStreamRequest(BaseModel): Rejected with a 400 (by `ServingDerender`) when a tool or reasoning parser is configured and this is omitted. Without it, `parse_delta` cannot tell whether the prompt left reasoning open and would silently - misclassify reasoning content as plain content. Unused on the plain - detokenization path. + misclassify reasoning content as plain content. On all paths it also + seeds detokenization on the first chunk, falling back to + `generate_chunk.prompt_token_ids` when omitted (see + `DerenderChatRequest.prompt_token_ids`). + + With a parser configured, send the full list. Parsers look back to the + last reasoning marker which can be anywhere in the prompt, so a suffix + can flip the initial reasoning state. Without a parser, a suffix is + enough. """ chat_request: ChatCompletionRequest | None = None @@ -666,6 +698,12 @@ class DerenderCompletionStreamRequest(BaseModel): prompt_tokens: int | None = None """Prompt token count for usage.""" + prompt_token_ids: list[NonNegativeInt] | None = None + """Prompt token IDs, used on the first chunk to seed detokenization. + Falls back to `generate_chunk.prompt_token_ids`. See + `DerenderChatRequest.prompt_token_ids`. + """ + completion_request: CompletionRequest | None = None """The original (post adjust_request) CompletionRequest from /render.""" # --8<-- [end:derender-completion-stream-request] diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index c1f175798d..0261401dd4 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -40,9 +40,14 @@ from vllm.logger import init_logger from vllm.parser import Parser, ParserManager from vllm.renderers import BaseRenderer from vllm.tokenizers import TokenizerLike -from vllm.tokenizers.detokenizer_utils import detokenize_incrementally +from vllm.tokenizers.detokenizer_utils import ( + convert_prompt_ids_to_tokens, + detokenize_incrementally, + get_leading_space_marker, +) from vllm.utils import random_uuid from vllm.utils.async_utils import make_async +from vllm.v1.engine.detokenizer import uses_fast_detokenizer logger = init_logger(__name__) @@ -115,17 +120,37 @@ class OnlineDerenderer: self, generate_response: GenerateResponse, chat_request: ChatCompletionRequest | None = None, + prompt_token_ids: list[int] | None = None, ) -> list[ChatCompletionResponseChoice]: - return await self._derender_chat_async(generate_response, chat_request) + return await self._derender_chat_async( + generate_response, chat_request, prompt_token_ids + ) def _derender_chat( self, generate_response: GenerateResponse, chat_request: ChatCompletionRequest | None = None, + prompt_token_ids: list[int] | None = None, ) -> list[ChatCompletionResponseChoice]: tokenizer = self.renderer.get_tokenizer() choices: list[ChatCompletionResponseChoice] = [] + has_parser = self.parser is not None and chat_request is not None + skip_special, spaces_between = _decode_params( + tokenizer, chat_request, preserve_special=has_parser + ) + seed_ids = ( + prompt_token_ids + if prompt_token_ids is not None + else generate_response.prompt_token_ids + ) + # Choices continue the same prompt, so they share one seeded window. + seed_state = _seed_stream_state( + tokenizer, + seed_ids, + skip_special_tokens=skip_special, + ) + for choice in generate_response.choices: if not choice.token_ids: raise ValueError(f"choice {choice.index} has empty or null token_ids") @@ -136,12 +161,17 @@ class OnlineDerenderer: else None ) - if self.parser is not None and chat_request is not None: + if has_parser: + assert self.parser is not None and chat_request is not None # Parser path: decode with special tokens preserved # so the parser can see markers like , # , or Harmony channel tokens. - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=False + decoded_text, _ = self._detokenize_delta( + tokenizer, + choice.token_ids, + seed_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) chat_template_kwargs: dict[str, Any] = {} @@ -201,13 +231,12 @@ class OnlineDerenderer: else: # No parser: plain detokenization honouring the request's # skip_special_tokens (default True when no request was given). - skip_special = ( - chat_request.skip_special_tokens - if chat_request is not None - else True - ) - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=skip_special + decoded_text, _ = self._detokenize_delta( + tokenizer, + choice.token_ids, + seed_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) message = ChatMessage(role="assistant", content=decoded_text) @@ -239,7 +268,7 @@ class OnlineDerenderer: (tracked by ``read_offset``). The delta tokens are fed straight onto it. The window is bounded. ``detokenize_incrementally`` never reads before - ``prefix_offset``, so after processing we trim ``prev_tokens`` to that + ``prefix_offset``, so after each token we trim ``prev_tokens`` to that tail and rebase the offsets to it. State transport therefore stays O(window) per chunk instead of re-sending the full token history. @@ -273,18 +302,20 @@ class OnlineDerenderer: skip_special_tokens=skip_special_tokens, spaces_between_special_tokens=spaces_between_special_tokens, ) - prev_tokens = prev_tokens + new_toks + # Trim to the tail still readable by detokenize_incrementally + # (everything before prefix_offset is dead) and rebase the + # offsets, so the window stays bounded within a long batch + # decode as well as across chunks. + prev_tokens = (prev_tokens + new_toks)[prefix_offset:] + read_offset -= prefix_offset + prefix_offset = 0 text_parts.append(text) - # Trim to the tail still readable by detokenize_incrementally - # (everything before prefix_offset is dead) and rebase the offsets so - # the carried window stays bounded regardless of generation length. - trimmed = prev_tokens[prefix_offset:] updated_state = state.model_copy( update={ - "prev_tokens": trimmed, - "prefix_offset": 0, - "read_offset": read_offset - prefix_offset, + "prev_tokens": prev_tokens, + "prefix_offset": prefix_offset, + "read_offset": read_offset, } ) return "".join(text_parts), updated_state @@ -315,20 +346,16 @@ class OnlineDerenderer: (validated by the caller — see `ServingDerender`) because plain detokenization would leak raw parser markup into `content`. prompt_tokens: Prompt token count for the usage chunk. - prompt_token_ids: Prompt token IDs. Parser path only. Lets - `parse_delta` settle its initial reasoning state. Required - when a reasoning or tool parser is configured (validated by - the caller — see ``ServingDerender``). Without it reasoning - left open by the prompt would be misclassified as content. + prompt_token_ids: Prompt token IDs. Required on the parser path + (validated by the caller) and optional otherwise, falling + back to ``generate_chunk.prompt_token_ids``. See + `DerenderChatStreamRequest.prompt_token_ids`. Returns: (chunk, updated_state) — the derendered SSE chunk and the state the client must pass to the next call. """ - if state is None: - state = DerenderStreamState() - # A single DerenderStreamState is threaded through every choice in # this chunk. Correct only when there is at most one choice per SSE # event (n=1, one call per index), as the streaming derender @@ -353,23 +380,34 @@ class OnlineDerenderer: parser_cls, model, generate_chunk, - state, + state if state is not None else DerenderStreamState(), chat_request, prompt_tokens, prompt_token_ids, ) tokenizer = self.renderer.get_tokenizer() - skip_special = ( - chat_request.skip_special_tokens if chat_request is not None else True - ) + skip_special, spaces_between = _decode_params(tokenizer, chat_request) + # Seed on the first chunk only. A carried state already has the prompt. + if state is None: + state = _seed_stream_state( + tokenizer, + prompt_token_ids + if prompt_token_ids is not None + else generate_chunk.prompt_token_ids, + skip_special_tokens=skip_special, + ) stream_choices: list[ChatCompletionResponseStreamChoice] = [] updated_state = state for choice in generate_chunk.choices: delta_tids = choice.token_ids or [] new_text, updated_state = await self._detokenize_delta_async( - tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special + tokenizer, + delta_tids, + updated_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) # NOTE: parser-configured servers dispatch to @@ -454,7 +492,8 @@ class OnlineDerenderer: `state.output_chunk_lens`, so the rebuilt parser state matches standard serving, which calls `parse_delta` once per engine step. Text comes from a fresh incremental detokenizer with special tokens - preserved (``skip_special_tokens=False``). + preserved (``skip_special_tokens=False``), seeded from the prompt + tail on every call since replay starts from scratch. """ tokenizer = self.renderer.get_tokenizer() @@ -480,7 +519,12 @@ class OnlineDerenderer: # across both the replay and current chunk phases (via `nonlocal`) # so multi-byte characters split across that boundary still decode # correctly. Discarded once the call returns. - detok_state = DerenderStreamState() + skip_special, spaces_between = _decode_params( + tokenizer, chat_request, preserve_special=True + ) + detok_state = _seed_stream_state( + tokenizer, prompt_token_ids, skip_special_tokens=skip_special + ) def _replay(token_ids: list[int], chunk_lens: list[int]) -> None: """Replay prior chunks through `parse_delta` to rebuild parser @@ -493,7 +537,11 @@ class OnlineDerenderer: chunk = token_ids[start : start + chunk_len] start += chunk_len text, detok_state = self._detokenize_delta( - tokenizer, chunk, detok_state, skip_special_tokens=False + tokenizer, + chunk, + detok_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) parser.parse_delta( text, @@ -530,7 +578,11 @@ class OnlineDerenderer: # granularity), not one per token. See the granularity note # in this method's docstring. text, detok_state = self._detokenize_delta( - tokenizer, delta_tids, detok_state, skip_special_tokens=False + tokenizer, + delta_tids, + detok_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) delta_message = parser.parse_delta( text, @@ -633,9 +685,10 @@ class OnlineDerenderer: generate_responses: list[GenerateResponse], prompt_tokens: list[int] | None = None, completion_request: CompletionRequest | None = None, + prompt_token_ids: list[list[int] | None] | None = None, ) -> tuple[list[CompletionResponseChoice], int, int]: return await self._derender_completion_async( - generate_responses, prompt_tokens, completion_request + generate_responses, prompt_tokens, completion_request, prompt_token_ids ) def _derender_completion( @@ -643,24 +696,34 @@ class OnlineDerenderer: generate_responses: list[GenerateResponse], prompt_tokens: list[int] | None = None, completion_request: CompletionRequest | None = None, + prompt_token_ids: list[list[int] | None] | None = None, ) -> tuple[list[CompletionResponseChoice], int, int]: n = len(generate_responses) prompt_tokens_list: list[int] = ( prompt_tokens if prompt_tokens is not None else [0] * n ) - - skip_special = ( - completion_request.skip_special_tokens - if completion_request is not None - else True + prompt_token_ids_list: list[list[int] | None] = ( + prompt_token_ids if prompt_token_ids is not None else [None] * n ) + tokenizer = self.renderer.get_tokenizer() + skip_special, spaces_between = _decode_params(tokenizer, completion_request) choices: list[CompletionResponseChoice] = [] total_prompt_tokens = 0 total_completion_tokens = 0 index = 0 - for gen, pt in zip(generate_responses, prompt_tokens_list): + for gen, pt, seed_ids_override in zip( + generate_responses, prompt_tokens_list, prompt_token_ids_list + ): + seed_ids = ( + seed_ids_override + if seed_ids_override is not None + else gen.prompt_token_ids + ) + seed_state = _seed_stream_state( + tokenizer, seed_ids, skip_special_tokens=skip_special + ) for choice in gen.choices: if not choice.token_ids: raise ValueError( @@ -668,8 +731,12 @@ class OnlineDerenderer: "has empty or null token_ids" ) - decoded_text = tokenizer.decode( - choice.token_ids, skip_special_tokens=skip_special + decoded_text, _ = self._detokenize_delta( + tokenizer, + choice.token_ids, + seed_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) completion_logprobs = None if choice.logprobs is not None: @@ -698,6 +765,7 @@ class OnlineDerenderer: state: DerenderStreamState | None = None, prompt_tokens: int | None = None, completion_request: CompletionRequest | None = None, + prompt_token_ids: list[int] | None = None, ) -> tuple[CompletionStreamResponse, DerenderStreamState]: """Process one GenerateStreamResponse chunk for streaming completions. @@ -716,14 +784,13 @@ class OnlineDerenderer: prompt_tokens: Prompt token count for usage (from the render step). completion_request: Original CompletionRequest from ``/render``; supplies ``skip_special_tokens``. + prompt_token_ids: Seeds the first chunk's decode. Falls back to + ``generate_chunk.prompt_token_ids``. Returns: (chunk, updated_state) — the derendered chunk and updated state. """ - if state is None: - state = DerenderStreamState() - # See the equivalent check in derender_chat_stream: a single # DerenderStreamState is threaded through every choice in this # chunk, so more than one choice per chunk would corrupt the @@ -734,18 +801,27 @@ class OnlineDerenderer: ) tokenizer = self.renderer.get_tokenizer() - skip_special = ( - completion_request.skip_special_tokens - if completion_request is not None - else True - ) + skip_special, spaces_between = _decode_params(tokenizer, completion_request) + # Seed on the first chunk only. A carried state already has the prompt. + if state is None: + state = _seed_stream_state( + tokenizer, + prompt_token_ids + if prompt_token_ids is not None + else generate_chunk.prompt_token_ids, + skip_special_tokens=skip_special, + ) stream_choices: list[CompletionResponseStreamChoice] = [] updated_state = state for choice in generate_chunk.choices: delta_tids = choice.token_ids or [] new_text, updated_state = await self._detokenize_delta_async( - tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special + tokenizer, + delta_tids, + updated_state, + skip_special_tokens=skip_special, + spaces_between_special_tokens=spaces_between, ) completion_logprobs = None @@ -810,6 +886,65 @@ def _logprob_context_tail( return (list(context_token_ids) + list(delta_token_ids))[-_LOGPROB_CONTEXT_WINDOW:] +def _decode_params( + tokenizer: TokenizerLike, + request: ChatCompletionRequest | CompletionRequest | None, + preserve_special: bool = False, +) -> tuple[bool, bool]: + """Derive `(skip_special_tokens, spaces_between_special_tokens)` the way + the engine does (`IncrementalDetokenizer`). + + Args: + tokenizer: Picks the engine's fast or slow detokenizer rules. + request: The original request, if the caller supplied one. + preserve_special: Keep special tokens so a parser can see markers. + The serving side does this via `adjust_request`. + + """ + skip_special = ( + False + if preserve_special + else (request.skip_special_tokens if request is not None else True) + ) + spaces_between = ( + request.spaces_between_special_tokens if request is not None else True + ) + # Only the fast detokenizer forces spaces on when skipping special tokens. + if skip_special and uses_fast_detokenizer(tokenizer): + spaces_between = True + return skip_special, spaces_between + + +def _seed_stream_state( + tokenizer: TokenizerLike, + prompt_token_ids: list[int] | None, + skip_special_tokens: bool, +) -> DerenderStreamState: + """Build the initial decode state from the prompt tail, the same way the + engine primes its incremental detokenizer. Without it, Metaspace + tokenizers drop the first output token's leading space. + + Returns an empty state when `prompt_token_ids` is empty or omitted. + """ + if not prompt_token_ids: + if prompt_token_ids is None and get_leading_space_marker(tokenizer) is not None: + logger.warning_once( + "derender got no prompt_token_ids, so the first output token " + "may lose its leading space compared to the coupled endpoint. " + "Pass prompt_token_ids from the render step to avoid this." + ) + return DerenderStreamState() + + prev_tokens, prefix_offset, read_offset = convert_prompt_ids_to_tokens( + tokenizer, prompt_token_ids, skip_special_tokens=skip_special_tokens + ) + return DerenderStreamState( + prev_tokens=prev_tokens, + prefix_offset=prefix_offset, + read_offset=read_offset, + ) + + def _parse_token_id_placeholder(token: str) -> int | None: """Extract token ID from a 'token_id:N' placeholder string.""" if not token.startswith("token_id:"): diff --git a/vllm/tokenizers/detokenizer_utils.py b/vllm/tokenizers/detokenizer_utils.py index 35e73bc410..ddaf512442 100644 --- a/vllm/tokenizers/detokenizer_utils.py +++ b/vllm/tokenizers/detokenizer_utils.py @@ -63,7 +63,7 @@ _CACHED_MARKER_KEY = "_vllm_space_marker_cache" _NOT_CACHED = "__not_computed__" -def _get_leading_space_marker(tokenizer: TokenizerLike) -> str | None: +def get_leading_space_marker(tokenizer: TokenizerLike) -> str | None: """Read the space marker from the tokenizer's pre_tokenizer config. Only Metaspace pre_tokenizers (used by SentencePiece-based models like @@ -160,7 +160,7 @@ def convert_ids_list_to_tokens( """ if not token_ids: return [] - marker = _get_leading_space_marker(tokenizer) + marker = get_leading_space_marker(tokenizer) if marker is None: return [tokenizer.decode([tid]) or "" for tid in token_ids] raw_tokens = tokenizer.convert_ids_to_tokens(token_ids) diff --git a/vllm/v1/engine/detokenizer.py b/vllm/v1/engine/detokenizer.py index 9658c030ef..c27e01895c 100644 --- a/vllm/v1/engine/detokenizer.py +++ b/vllm/v1/engine/detokenizer.py @@ -3,9 +3,7 @@ import sys from abc import ABC, abstractmethod -import tokenizers import tokenizers.decoders -from packaging import version from tokenizers import Tokenizer from transformers import TokenizersBackend @@ -20,14 +18,15 @@ from vllm.v1.engine import EngineCoreRequest logger = init_logger(__name__) -# Only tokenizers >= 0.22.0 supports DecodeStream with native prefill -# (ids parameter) used for FastIncrementalDetokenizer. -USE_FAST_DETOKENIZER = version.parse(tokenizers.__version__) >= version.parse("0.22.0") - # Error string from https://github.com/huggingface/tokenizers/blob/909fdde2a4ffedd9295206f705eb612be2a91b12/tokenizers/src/tokenizer/mod.rs#L1042 INVALID_PREFIX_ERR_MSG = "Invalid prefix encountered" +def uses_fast_detokenizer(tokenizer: TokenizerLike) -> bool: + """Whether the engine detokenizes with `FastIncrementalDetokenizer`.""" + return isinstance(tokenizer, TokenizersBackend) + + class IncrementalDetokenizer: def __init__(self): self.token_ids: list[int] = [] @@ -58,7 +57,7 @@ class IncrementalDetokenizer: # No tokenizer => skipping detokenization. return IncrementalDetokenizer() - if USE_FAST_DETOKENIZER and isinstance(tokenizer, TokenizersBackend): + if uses_fast_detokenizer(tokenizer): # Fast tokenizer => use tokenizers library DecodeStream. return FastIncrementalDetokenizer(tokenizer, request) From a810316c0894b8a43e53458ae5b14c9ad0b18b5d Mon Sep 17 00:00:00 2001 From: Guanxin Li <38149783+guanxingithub@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:19:28 -0700 Subject: [PATCH 239/248] [Bugfix] Keep batch-invariance NCCL pins out of the weight-transfer group (#59500) Signed-off-by: aoshen524 Co-authored-by: aoshen524 Co-authored-by: Claude Opus 5.5 Co-authored-by: aoshen02 --- tests/distributed/test_weight_transfer.py | 48 ++++++++++++-- .../weight_transfer/nccl_common.py | 17 +++-- .../determinism/batch_invariant.py | 37 ++++++----- vllm/utils/nccl.py | 65 +++++++++++++++++++ 4 files changed, 140 insertions(+), 27 deletions(-) diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index 3af7b7aeb9..85ae7f58c7 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -68,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import ( SparseWeightPatch, ) from vllm.platforms import current_platform +from vllm.utils.nccl import _nccl_has_no_cache from vllm.utils.network_utils import get_open_port @@ -433,13 +434,15 @@ def trainer_broadcast_tensor( return True -@ray.remote(num_gpus=1) +# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process. +@ray.remote(num_gpus=1, max_calls=1) def inference_receive_tensor( master_address: str, master_port: int, world_size: int, tensor_shape: list[int], tensor_dtype: str, + batch_invariant: bool = False, ) -> dict: """Inference task that receives tensor via NCCLWeightTransferEngine.""" import contextlib @@ -448,6 +451,21 @@ def inference_receive_tensor( import torch _set_ray_assigned_device() + if batch_invariant: + from vllm.model_executor.determinism.batch_invariant import ( + override_envs_for_invariance, + ) + + override_envs_for_invariance() + # Like vLLM's own groups, a pinned communicator makes NCCL read the + # pins before the transfer group exists. + torch.distributed.init_process_group( + "nccl", + init_method=f"tcp://127.0.0.1:{get_open_port()}", + rank=0, + world_size=1, + ) + torch.distributed.all_reduce(torch.ones(1, device="cuda")) from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig @@ -534,11 +552,25 @@ def inference_receive_tensor( torch.accelerator.device_count() < 2, reason="Need at least 2 GPUs to run NCCL weight transfer test.", ) -def test_nccl_weight_transfer_between_processes(): +@pytest.mark.parametrize( + "batch_invariant", + [ + pytest.param(False, id="default"), + pytest.param( + True, + id="batch-invariant-worker", + marks=pytest.mark.skipif( + not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7." + ), + ), + ], +) +def test_nccl_weight_transfer_between_processes(batch_invariant): """Test NCCL weight transfer from trainer to inference process using Ray. This test verifies that the NCCLWeightTransferEngine can receive - tensors broadcast by a trainer process via NCCL. + tensors broadcast by a trainer process via NCCL, including when the + worker runs with batch-invariance NCCL pins the trainer does not have. """ _init_ray_for_weight_transfer() @@ -550,13 +582,19 @@ def test_nccl_weight_transfer_between_processes(): tensor_dtype = "float32" inference_future = inference_receive_tensor.remote( - master_address, master_port, world_size, tensor_shape, tensor_dtype + master_address, + master_port, + world_size, + tensor_shape, + tensor_dtype, + batch_invariant, ) trainer_future = trainer_broadcast_tensor.remote( master_address, master_port, world_size, tensor_shape, tensor_dtype ) - trainer_result, result = ray.get([trainer_future, inference_future]) + # A mismatched NCCL config deadlocks instead of failing. + trainer_result, result = ray.get([trainer_future, inference_future], timeout=300) assert trainer_result, "Trainer should complete successfully" assert result["success"], ( diff --git a/vllm/distributed/weight_transfer/nccl_common.py b/vllm/distributed/weight_transfer/nccl_common.py index a329e456d5..1cd026fb07 100644 --- a/vllm/distributed/weight_transfer/nccl_common.py +++ b/vllm/distributed/weight_transfer/nccl_common.py @@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.packed_tensor import ( DEFAULT_PACKED_BUFFER_SIZE_BYTES, DEFAULT_PACKED_NUM_BUFFERS, ) +from vllm.utils.nccl import unpinned_nccl_env def decode_nccl_unique_id( @@ -154,7 +155,8 @@ def stateless_init_process_group( pg = StatelessProcessGroup.create( host=master_address, port=master_port, rank=rank, world_size=world_size ) - return PyNcclCommunicator(pg, device=device) + with unpinned_nccl_env(): + return PyNcclCommunicator(pg, device=device) def uid_init_process_group( @@ -170,12 +172,13 @@ def uid_init_process_group( """ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - return PyNcclCommunicator.from_unique_id_bytes( - nccl_unique_id_bytes, - rank=rank, - world_size=world_size, - device=device, - ) + with unpinned_nccl_env(): + return PyNcclCommunicator.from_unique_id_bytes( + nccl_unique_id_bytes, + rank=rank, + world_size=world_size, + device=device, + ) def worker_init_process_group( diff --git a/vllm/model_executor/determinism/batch_invariant.py b/vllm/model_executor/determinism/batch_invariant.py index 688e14d709..6490e69a60 100644 --- a/vllm/model_executor/determinism/batch_invariant.py +++ b/vllm/model_executor/determinism/batch_invariant.py @@ -16,6 +16,7 @@ from vllm.model_executor.determinism.batch_invariant_configs import ( from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.mem_utils import get_max_shared_memory_bytes +from vllm.utils.nccl import pin_nccl_env from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import is_torch_equal_or_newer @@ -1144,22 +1145,28 @@ def override_envs_for_invariance(): os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - # NCCL determinism settings + # NCCL determinism settings. NCCL keeps the launch mode process-wide, so, + # unlike the pins below, it also applies to communicators shared with other + # processes. os.environ["NCCL_LAUNCH_MODE"] = "GROUP" - os.environ["NCCL_COLLNET_ENABLE"] = "0" - os.environ["NCCL_NVLS_ENABLE"] = "0" - os.environ["NCCL_P2P_NET_DISABLE"] = "1" - os.environ["NCCL_MIN_NCHANNELS"] = "1" - os.environ["NCCL_MAX_NCHANNELS"] = "1" - os.environ["NCCL_PROTO"] = "Simple" - # NCCL >= 2.31 zero-fills the algorithm table of every collective when - # NCCL_ALGO is set and re-enables only the named ones; together with the - # NCCL_PROTO above, collectives not named here end up with no algorithm - # and fail with ncclInvalidUsage. Re-enable Ring and Tree for all - # collectives, then pin AllReduce to Tree for determinism. - os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree" - os.environ["NCCL_NTHREADS"] = "1" - os.environ["NCCL_SOCKET_NTHREADS"] = "1" + pin_nccl_env( + { + "NCCL_COLLNET_ENABLE": "0", + "NCCL_NVLS_ENABLE": "0", + "NCCL_P2P_NET_DISABLE": "1", + "NCCL_MIN_NCHANNELS": "1", + "NCCL_MAX_NCHANNELS": "1", + "NCCL_PROTO": "Simple", + # NCCL >= 2.31 zero-fills the algorithm table of every collective + # when NCCL_ALGO is set and re-enables only the named ones; together + # with the NCCL_PROTO above, collectives not named here end up with + # no algorithm and fail with ncclInvalidUsage. Re-enable Ring and + # Tree for all collectives, then pin AllReduce to Tree. + "NCCL_ALGO": "ring,tree;allreduce:tree", + "NCCL_NTHREADS": "1", + "NCCL_SOCKET_NTHREADS": "1", + } + ) # torch.compile settings os.environ["VLLM_USE_AOT_COMPILE"] = "0" diff --git a/vllm/utils/nccl.py b/vllm/utils/nccl.py index 90064d4184..1136661f6e 100644 --- a/vllm/utils/nccl.py +++ b/vllm/utils/nccl.py @@ -3,9 +3,11 @@ from __future__ import annotations +import contextlib import ctypes import importlib.util import os +from collections.abc import Iterator, Mapping import torch @@ -129,3 +131,66 @@ def query_nccl_gin_type( logger.warning("ncclCommQueryProperties returned error %d", result) return None return props.railedGinType if railed else props.ginType + + +# Values the variables set by `pin_nccl_env` had before it: None until something +# is pinned, empty if NCCL cannot re-read them. +_unpinned_env: dict[str, str | None] | None = None + + +def pin_nccl_env(pins: dict[str, str]) -> None: + """Set NCCL variables for this process's own communicators. + + NCCL never reconciles these across ranks, so communicators shared with a + process that does not set them are created without them, see + `unpinned_nccl_env`. NCCL caches most variables on first read; listing the + pins in NCCL_NO_CACHE makes it re-read them per communicator. NCCL parses + that list once, so this must run before the process's first NCCL call. + """ + global _unpinned_env + launch_env = _swap_env(pins) + if _unpinned_env is not None: + return + _unpinned_env = launch_env if _nccl_has_no_cache() else {} + if _unpinned_env: + no_cache = os.environ.get("NCCL_NO_CACHE") + os.environ["NCCL_NO_CACHE"] = ",".join(filter(None, [no_cache, *pins])) + + +@contextlib.contextmanager +def unpinned_nccl_env() -> Iterator[None]: + """Create communicators shared with other processes without the pins.""" + if _unpinned_env == {}: + logger.warning_once( + "NCCL_NO_CACHE needs CUDA NCCL >= 2.29.7, so the NCCL settings vLLM " + "pinned also apply here; a process sharing this communicator must " + "set the same NCCL_* variables or it will hang." + ) + pinned = _swap_env(_unpinned_env or {}) + try: + yield + finally: + _swap_env(pinned) + + +def _nccl_has_no_cache() -> bool: + # NCCL_NO_CACHE arrived in NCCL 2.29.7; ask the library vLLM loads. + if torch.version.cuda is None: + return False + version = ctypes.c_int() + try: + ctypes.CDLL(find_nccl_library()).ncclGetVersion(ctypes.byref(version)) + except Exception: + return False + return version.value >= 22907 + + +def _swap_env(values: Mapping[str, str | None]) -> dict[str, str | None]: + """Set (None: unset) the given variables and return their previous values.""" + old = {name: os.environ.get(name) for name in values} + for name, value in values.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + return old From 8486f1ebfe6e39a5513a8636e7f07ed9746f8be5 Mon Sep 17 00:00:00 2001 From: Qiu Chunshuo Date: Fri, 2 Oct 2026 09:20:13 +0530 Subject: [PATCH 240/248] [Perf][PCP] Shard decode requests across PCP ranks (#52162) Signed-off-by: QiuChunshuo Co-authored-by: Lucas Wilkinson --- .../layers/test_mla_short_prefill_indexer.py | 2 + .../test_indexer_deepseek_v4_slot_mapping.py | 73 ++++++ .../v1/attention/test_sparse_mla_backends.py | 1 + tests/v1/cudagraph/test_cudagraph_manager.py | 1 + tests/v1/worker/test_gpu_pcp_manager.py | 230 +++++++++++++++++- vllm/config/parallel.py | 13 + .../layers/attention/mla_attention.py | 2 + .../layers/sparse_attn_indexer.py | 6 + vllm/models/deepseek_v32/attention.py | 2 + .../attention/backends/mla/flashmla_sparse.py | 9 +- vllm/v1/attention/backends/mla/indexer.py | 39 ++- vllm/v1/attention/ops/pcp.py | 32 ++- vllm/v1/worker/gpu/pcp_manager.py | 29 ++- 13 files changed, 406 insertions(+), 33 deletions(-) diff --git a/tests/model_executor/layers/test_mla_short_prefill_indexer.py b/tests/model_executor/layers/test_mla_short_prefill_indexer.py index 2e4d4d3d67..ba4ff895cb 100644 --- a/tests/model_executor/layers/test_mla_short_prefill_indexer.py +++ b/tests/model_executor/layers/test_mla_short_prefill_indexer.py @@ -205,6 +205,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision( topk_indices, False, False, + False, MLA_LAYER, ) @@ -259,6 +260,7 @@ def test_skipped_k_cache_insert_accepts_no_k( topk_indices, True, False, + False, "", ) diff --git a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py index bf775078c8..5bb6eb4c32 100644 --- a/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py +++ b/tests/v1/attention/test_indexer_deepseek_v4_slot_mapping.py @@ -15,6 +15,7 @@ from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend, ) from vllm.v1.attention.backend import CommonAttentionMetadata +from vllm.v1.attention.backends.mla import indexer as indexer_module from vllm.v1.attention.backends.mla.compressor_utils import ( CompressedSlotMappingKernel, get_compressed_slot_mapping, @@ -295,6 +296,78 @@ def test_index_conversion_warmup_uses_physical_block_stride(): assert {key.block_stride_rows for key in keys} == {4096} +def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypatch): + builder = object.__new__(DeepseekV32IndexerMetadataBuilder) + builder.compress_ratio = 4 + builder.use_pcp = True + builder.pcp_world_size = 4 + builder.pcp_rank = 0 + builder.kernel_block_size = None + builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64) + builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64) + + def fake_get_compressed_slot_mapping( + num_tokens, + slot_mapping, + query_start_loc, + seq_lens, + block_table, + block_size, + compress_ratio, + out, + ): + assert num_tokens == 0 + assert block_size == 64 + assert compress_ratio == 4 + out.fill_(-1) + return out[:num_tokens] + + gathered_slot_mapping = torch.tensor([123, -1, -1, -1], dtype=torch.int64) + + class FakePCPGroup: + def __init__(self): + self.calls = 0 + + def all_gather(self, tensor, dim=0): + self.calls += 1 + assert dim == 0 + torch.testing.assert_close(tensor, torch.tensor([-1])) + return gathered_slot_mapping + + fake_pcp_group = FakePCPGroup() + monkeypatch.setattr( + indexer_module, + "get_compressed_slot_mapping", + fake_get_compressed_slot_mapping, + ) + monkeypatch.setattr(indexer_module, "get_pcp_group", lambda: fake_pcp_group) + + query_start_loc = torch.tensor([0, 0], dtype=torch.int32) + seq_lens = torch.tensor([16], dtype=torch.int32) + common = CommonAttentionMetadata( + query_start_loc=query_start_loc, + query_start_loc_cpu=query_start_loc, + seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens, + num_reqs=1, + num_actual_tokens=0, + max_query_len=0, + max_seq_len=16, + block_table_tensor=torch.tensor([[0]], dtype=torch.int32), + slot_mapping=torch.tensor([11, 22, 33, 44], dtype=torch.int64), + causal=True, + ) + + metadata = builder.build(common_prefix_len=0, common_attn_metadata=common) + + assert fake_pcp_group.calls == 1 + assert metadata.num_decodes == 0 + assert metadata.num_decode_tokens == 0 + assert metadata.num_prefills == 0 + assert metadata.num_prefill_tokens == 0 + torch.testing.assert_close(metadata.slot_mapping, gathered_slot_mapping) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_indexer_builder_deepseek_v4_compressed_slot_mapping_uses_num_states(): """Regression test: DeepseekV4 compression path must compute slot_mapping from diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 048eb40586..a6517430ad 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -2499,6 +2499,7 @@ def test_hisparse_kv_update_writes_resident_and_staging_caches(): layer = SimpleNamespace( hisparse_cache=cache_handle, use_pcp=False, + pcp_shard_decode_requests=False, impl=impl, ) mla_attention.MLAAttention.update_kv_cache( diff --git a/tests/v1/cudagraph/test_cudagraph_manager.py b/tests/v1/cudagraph/test_cudagraph_manager.py index 366db7b025..8f58462d6b 100644 --- a/tests/v1/cudagraph/test_cudagraph_manager.py +++ b/tests/v1/cudagraph/test_cudagraph_manager.py @@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings(): pcp_world_size=pcp_world_size, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=False, max_num_reqs=num_reqs, max_num_tokens=num_tokens, block_tables=pcp_block_tables, diff --git a/tests/v1/worker/test_gpu_pcp_manager.py b/tests/v1/worker/test_gpu_pcp_manager.py index 4612d49d92..d4b464e014 100644 --- a/tests/v1/worker/test_gpu_pcp_manager.py +++ b/tests/v1/worker/test_gpu_pcp_manager.py @@ -7,8 +7,9 @@ import numpy as np import pytest import torch -from vllm.config import CUDAGraphMode -from vllm.v1.attention.backends.utils import get_dcp_local_seq_lens +from vllm.config import CUDAGraphMode, ParallelConfig +from vllm.v1.attention.backends.utils import PAD_SLOT_ID, get_dcp_local_seq_lens +from vllm.v1.attention.ops import pcp as attention_pcp from vllm.v1.worker.gpu import cp_utils as gpu_cp_utils from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor @@ -53,6 +54,7 @@ def _make_capture_manager(block_table: torch.Tensor): pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, block_tables=block_tables, @@ -93,11 +95,12 @@ def test_validate_config_dcp_for_dspark(draft_uses_mla, dcp_size): PCPManager.validate_config(config, supports_mm_inputs=False) -def test_replicated_decode_piecewise_graph_padding(monkeypatch): +def test_sharded_decode_piecewise_graph_padding(monkeypatch): manager = PCPManager( pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=True, dcp_world_size=1, ) monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) @@ -110,19 +113,19 @@ def test_replicated_decode_piecewise_graph_padding(monkeypatch): padded_num_tokens=4, ) - assert per_rank_num_tokens == [3, 3] + assert per_rank_num_tokens == [2, 1] request_indices = [ [segment.global_batch_req_idx for segment in rank] for rank in segments_by_rank ] - assert request_indices == [[0, 1, 2], [0, 1, 2]] - assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 1, 2])) + assert request_indices == [[0, 2], [1]] + assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 4, 1])) assert torch.equal( manager._padded_gather_idx, - torch.tensor([0, 1, 2, 0, 0, 1, 2, 0]), + torch.tensor([0, 2, 0, 0, 1, 0, 0, 0]), ) assert torch.equal( manager._gathered_kv_write_mask, - torch.tensor([True, True, True, False, False, False, False, False]), + torch.tensor([True, True, False, False, True, False, False, False]), ) @@ -131,6 +134,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture(): pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, ) @@ -148,7 +152,7 @@ def test_input_buffers_are_exposed_for_cudagraph_capture(): (2, [7], [True], 4), (2, [3], [False], 3), (2, [3, 8], [False, True], 7), - (4, [2, 9], [False, True], 5), + (4, [2, 9], [False, True], 4), ], ) def test_num_tokens_for_dispatch_uses_largest_pcp_rank( @@ -158,6 +162,7 @@ def test_num_tokens_for_dispatch_uses_largest_pcp_rank( pcp_world_size=pcp_world_size, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=True, ) actual = manager.get_num_tokens_for_dispatch( @@ -173,6 +178,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): pcp_world_size=2, pcp_rank=0, device=torch.device("cpu"), + shard_decode_requests=True, dcp_world_size=1, ) monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) @@ -183,7 +189,7 @@ def test_graph_padding_cannot_be_smaller_than_largest_pcp_rank(monkeypatch): num_computed_tokens=np.full(3, 16, dtype=np.int32), is_prefilling=np.zeros(3, dtype=np.bool_), query_start_loc_np=np.arange(4, dtype=np.int32), - padded_num_tokens=2, + padded_num_tokens=1, ) @@ -207,6 +213,7 @@ def test_partition_padding_is_derived_from_batch_descriptor( pcp_world_size=2, pcp_rank=0, device=device, + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, ) @@ -288,6 +295,189 @@ def test_dummy_context_updates_pcp_local_block_tables(): assert torch.all(global_block_table == -1) +def _rank_request_ids( + manager: PCPManager, + rank: int, + req_ids: list[str], + *, + is_prefilling: np.ndarray | None = None, +) -> list[str]: + num_reqs = len(req_ids) + if is_prefilling is None: + is_prefilling = np.zeros(num_reqs, dtype=np.bool_) + segments = manager._get_rank_segments( + rank=rank, + num_scheduled_tokens=np.ones(num_reqs, dtype=np.int32), + is_prefilling=is_prefilling, + query_start_loc_np=np.arange(num_reqs + 1, dtype=np.int32), + ) + return [req_ids[segment.global_batch_req_idx] for segment in segments] + + +def test_pcp_only_decode_requests_are_round_robin_balanced_each_step(): + manager = PCPManager( + pcp_world_size=4, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + req_ids = [f"request-{idx}" for idx in range(18)] + + owners: dict[str, int] = {} + for rank in range(manager.pcp_world_size): + for req_id in _rank_request_ids(manager, rank, req_ids): + assert req_id not in owners + owners[req_id] = rank + + assert owners == { + req_id: index % manager.pcp_world_size for index, req_id in enumerate(req_ids) + } + counts = [list(owners.values()).count(rank) for rank in range(4)] + assert max(counts) - min(counts) == 1 + + reordered_req_ids = req_ids[::2] + req_ids[1::2] + reordered_owners = { + req_id: rank + for rank in range(manager.pcp_world_size) + for req_id in _rank_request_ids(manager, rank, reordered_req_ids) + } + assert reordered_owners == { + req_id: index % manager.pcp_world_size + for index, req_id in enumerate(reordered_req_ids) + } + assert reordered_owners != owners + + +def test_decode_requests_remain_replicated_when_dcp_is_enabled(): + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=False, + dcp_world_size=2, + ) + req_ids = ["request-a", "request-b", "request-c"] + + assert _rank_request_ids(manager, 0, req_ids) == req_ids + assert _rank_request_ids(manager, 1, req_ids) == req_ids + + +def test_decode_sharding_allows_ranks_with_no_owned_request(): + manager = PCPManager( + pcp_world_size=4, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + req_ids = ["request-a"] + + assert _rank_request_ids(manager, 0, req_ids) == req_ids + assert _rank_request_ids(manager, 1, req_ids) == [] + assert _rank_request_ids(manager, 2, req_ids) == [] + assert _rank_request_ids(manager, 3, req_ids) == [] + + +def test_prefill_partitioning_is_preserved_with_sharded_decode(): + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + segments_by_rank = [ + manager._get_rank_segments( + rank=rank, + num_scheduled_tokens=np.array([8, 1, 0, 1], dtype=np.int32), + is_prefilling=np.array([True, False, False, False]), + query_start_loc_np=np.array([0, 8, 9, 9, 10], dtype=np.int32), + ) + for rank in range(manager.pcp_world_size) + ] + + prefill_tokens = sorted( + token_idx + for segments in segments_by_rank + for segment in segments + if segment.global_batch_req_idx == 0 + for token_idx in range( + segment.global_batch_slice.start, segment.global_batch_slice.stop + ) + ) + decode_owners = { + segment.global_batch_req_idx: rank + for rank, segments in enumerate(segments_by_rank) + for segment in segments + if segment.global_batch_req_idx in (1, 3) + } + + assert prefill_tokens == list(range(8)) + assert decode_owners == {1: 0, 3: 1} + + +def test_sharded_decode_layout_selects_owner_kv_for_replication(monkeypatch): + manager = PCPManager( + pcp_world_size=2, + pcp_rank=0, + device=torch.device("cpu"), + shard_decode_requests=True, + dcp_world_size=1, + ) + monkeypatch.setattr(pcp_manager_module, "async_tensor_h2d", _copy_to_cpu) + manager._build_batch_layout( + num_scheduled_tokens=np.array([1, 1, 1], dtype=np.int32), + num_computed_tokens=np.array([16, 16, 16], dtype=np.int32), + is_prefilling=np.array([False, False, False]), + query_start_loc_np=np.array([0, 1, 2, 3], dtype=np.int32), + ) + + gathered_slot_mapping = manager._convert_to_gathered_slot_mappings( + torch.tensor([[123, 456, 789]], dtype=torch.int64) + ) + assert torch.equal( + gathered_slot_mapping, + torch.tensor([[123, 789, 456, PAD_SLOT_ID]], dtype=torch.int64), + ) + assert torch.equal(manager._hidden_restore_idx, torch.tensor([0, 2, 1])) + + class FakePCPGroup: + world_size = 2 + + def all_gather(self, tensor: torch.Tensor, dim: int = 0) -> torch.Tensor: + assert dim == 0 + assert tensor.shape == (2, 1) + return torch.tensor([[11.0], [33.0], [22.0], [0.0]]) + + monkeypatch.setattr(attention_pcp, "get_pcp_group", FakePCPGroup) + (gathered_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs( + (torch.tensor([[11.0], [33.0]]),), + gathered_slot_mapping[0], + num_decode_tokens=0, + shard_decode_requests=True, + ) + + assert torch.equal(gathered_kv, torch.tensor([[11.0], [33.0], [22.0], [0.0]])) + assert torch.equal(cache_slot_mapping, torch.tensor([123, 789, 456, PAD_SLOT_ID])) + + +def test_replicated_draft_cache_inputs_keep_unexpanded_slot_mapping(monkeypatch): + def unexpected_gather(): + pytest.fail("replicated draft decode must not gather PCP cache inputs") + + monkeypatch.setattr(attention_pcp, "get_pcp_group", unexpected_gather) + kv = torch.tensor([[11.0], [22.0], [0.0]]) + slot_mapping = torch.tensor([123, 456]) + + (cache_kv,), cache_slot_mapping = attention_pcp._gather_prefill_cache_inputs( + (kv,), slot_mapping, num_decode_tokens=2, shard_decode_requests=True + ) + + assert torch.equal(cache_kv, kv[:2]) + assert torch.equal(cache_slot_mapping, slot_mapping) + + def _rank_rows( pcp_rank: int, pcp_world_size: int, @@ -300,6 +490,7 @@ def _rank_rows( pcp_world_size=pcp_world_size, pcp_rank=pcp_rank, device=torch.device("cpu"), + shard_decode_requests=False, dcp_world_size=2, ) query_start_loc_np = np.concatenate([[0], np.cumsum(num_scheduled_tokens)]).astype( @@ -393,6 +584,7 @@ def test_dcp_replicates_prefills_too_short_to_split(query_len): pcp_world_size=pcp_world_size, pcp_rank=rank, device=torch.device("cpu"), + shard_decode_requests=False, dcp_world_size=2, ) assert manager.replicated_requests(num_scheduled_tokens, is_prefilling)[0] @@ -418,6 +610,7 @@ def test_pcp_first_chunk_row_is_never_short(pcp_world_size, query_len): pcp_world_size=pcp_world_size, pcp_rank=rank, device=torch.device("cpu"), + shard_decode_requests=False, dcp_world_size=pcp_world_size, ) chunk_lens = [ @@ -493,6 +686,7 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch(): pcp_world_size=2, pcp_rank=0, device=device, + shard_decode_requests=False, max_num_reqs=4, max_num_tokens=8, dcp_world_size=2, @@ -541,3 +735,19 @@ def test_partition_defers_dcp_metadata_to_post_partition_batch(): ) assert local_batch.dcp_local_seq_lens is not None assert torch.equal(local_batch.dcp_local_seq_lens.cpu(), expected) + + +@pytest.mark.parametrize( + ("pcp_world_size", "dcp_world_size", "expected"), + [(1, 1, False), (2, 1, True), (2, 2, False)], +) +def test_parallel_config_manages_decode_sharding( + pcp_world_size: int, dcp_world_size: int, expected: bool +): + parallel_config = ParallelConfig( + prefill_context_parallel_size=pcp_world_size, + decode_context_parallel_size=dcp_world_size, + distributed_executor_backend="ray", + ) + + assert parallel_config.pcp_shard_decode_requests is expected diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index f4b825eb33..53a28fa691 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -599,6 +599,19 @@ class ParallelConfig: """Process world size across TP, PCP, PP, and DP.""" return self.world_size * self.data_parallel_size + @property + def pcp_shard_decode_requests(self) -> bool: + """Whether PCP can shard decode requests across its ranks. + + PCP-only execution replicates the KV cache, so each decode request can + have a single PCP owner. DCP shards the KV cache and therefore requires + every decode request to run on every participating DCP rank. + """ + return ( + self.prefill_context_parallel_size > 1 + and self.decode_context_parallel_size == 1 + ) + @property def use_ubatching(self) -> bool: return self.enable_dbo or self.ubatch_size > 1 diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index e016e5daff..b952a36821 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -609,6 +609,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): vllm_config = get_current_vllm_config() parallel_config = vllm_config.parallel_config self.use_pcp = parallel_config.prefill_context_parallel_size > 1 + self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") @@ -784,6 +785,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): slot_mapping, attn_metadata.num_decode_tokens if attn_metadata is not None else None, self.use_pcp, + pcp_shard_decode_requests=self.pcp_shard_decode_requests, ) assert slot_mapping is not None if cache is not None: diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index ae45b77dce..f8fedf046a 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -335,6 +335,7 @@ def sparse_attn_indexer( topk_indices_buffer: torch.Tensor, skip_k_cache_insert: bool, use_pcp: bool, + pcp_shard_decode_requests: bool, dense_mha_metadata_layer_name: LayerNameType, use_fp4_cache: bool = False, dcp_rank: int = 0, @@ -404,6 +405,7 @@ def sparse_attn_indexer( topk_indices_buffer, skip_k_cache_insert, use_pcp, + pcp_shard_decode_requests, dense_mha_metadata_layer_name, use_fp4_cache, candidate_blocks=candidate_blocks, @@ -442,6 +444,7 @@ def sparse_attn_indexer( slot_mapping, num_decode_tokens, use_pcp, + pcp_shard_decode_requests=pcp_shard_decode_requests, ) # scale_fmt can be None, but the function expects str assert scale_fmt is not None @@ -817,6 +820,7 @@ def sparse_attn_indexer_fake( topk_indices_buffer: torch.Tensor | None, skip_k_cache_insert: bool, use_pcp: bool, + pcp_shard_decode_requests: bool, dense_mha_metadata_layer_name: LayerNameType, use_fp4_cache: bool = False, dcp_rank: int = 0, @@ -898,6 +902,7 @@ class SparseAttnIndexer(CustomOp): self.dcp_world_size = parallel_config.decode_context_parallel_size self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_world_size > 1 else 0 self.use_pcp = parallel_config.prefill_context_parallel_size > 1 + self.pcp_shard_decode_requests = parallel_config.pcp_shard_decode_requests self._cp_kv_cache_interleave_size: int | None = None if current_platform.is_cuda() and not has_deep_gemm(): raise RuntimeError( @@ -994,6 +999,7 @@ class SparseAttnIndexer(CustomOp): self.topk_indices_buffer, self.skip_k_cache_insert, self.use_pcp, + self.pcp_shard_decode_requests, _encode_layer_name(self.dense_mha_metadata_layer_name), self.use_fp4_cache, self.dcp_rank, diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index d352002009..384b96dbf8 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -500,6 +500,7 @@ class DeepseekV32Attention(MLAAttention): self.topk_indices_buffer, skip_k_cache_insert=not self.use_pcp, use_pcp=self.use_pcp, + pcp_shard_decode_requests=self.pcp_shard_decode_requests, dense_mha_metadata_layer_name=self._dense_mha_metadata_layer_name, dcp_rank=( self.dcp_manager.group.rank_in_group @@ -535,6 +536,7 @@ class DeepseekV32Attention(MLAAttention): layer_slot_mapping, attn_metadata.num_decode_tokens, True, + pcp_shard_decode_requests=self.pcp_shard_decode_requests, ) ) self.impl.do_kv_cache_update( # type: ignore[attr-defined] diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 8371fb8b85..1c3756e6c4 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -489,6 +489,14 @@ class FlashMLASparseMetadataBuilder( num_tokens - metadata.num_decode_tokens, ) + FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode + # PCP decode sharding can leave a rank with only its collective-padding + # row when the global decode batch is smaller than the PCP world size. + # The row has no actual query tokens and must not be interpreted as a + # zero-length decode request by the sparse FP8 metadata builder. + if num_tokens == 0: + return FP8Meta() + decode_query_len = 0 active_num_decodes = num_decodes if num_decodes > 0: @@ -498,7 +506,6 @@ class FlashMLASparseMetadataBuilder( active_num_decodes = num_decode_tokens // decode_query_len assert active_num_decodes * decode_query_len == num_decode_tokens - FP8Meta = FlashMLASparseMetadata.FP8SeparatePrefillDecode fp8_metadata = FP8Meta( num_decodes=active_num_decodes, num_prefills=num_prefills, diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index afaae5e35f..20a52af82b 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -1298,17 +1298,6 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): slot_mapping = common_attn_metadata.slot_mapping block_table = common_attn_metadata.block_table_tensor dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens - num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( - split_decodes_and_prefills( - common_attn_metadata, - decode_threshold=self.decode_threshold, - require_uniform=not (self.use_flattening or self.supports_varlen), - treat_short_extends_as_decodes=not self.use_pcp, - ) - ) - - assert num_decodes + num_prefills == num_reqs - assert num_decode_tokens + num_prefill_tokens == num_tokens compressed_slot_mapping = slot_mapping indexer_block_table = block_table @@ -1347,6 +1336,34 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): dim=0, ) + # PCP decode sharding keeps a zero-token placeholder on ranks that own + # no request in a step so collectives retain a uniform rank shape. Do + # not turn that placeholder into a zero-length indexer decode request. + if num_tokens == 0: + return DeepseekV32IndexerMetadata( + seq_lens=seq_lens, + max_seq_len=common_attn_metadata.max_seq_len, + slot_mapping=compressed_slot_mapping, + num_decodes=0, + num_decode_tokens=0, + num_prefills=0, + num_prefill_tokens=0, + prefill=None, + decode=None, + ) + + num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( + split_decodes_and_prefills( + common_attn_metadata, + decode_threshold=self.decode_threshold, + require_uniform=not (self.use_flattening or self.supports_varlen), + treat_short_extends_as_decodes=not self.use_pcp, + ) + ) + + assert num_decodes + num_prefills == num_reqs + assert num_decode_tokens + num_prefill_tokens == num_tokens + prefill_metadata = None if num_prefills > 0: compressed_seq_lens = ( diff --git a/vllm/v1/attention/ops/pcp.py b/vllm/v1/attention/ops/pcp.py index 7e7dbe2245..97aeba038d 100644 --- a/vllm/v1/attention/ops/pcp.py +++ b/vllm/v1/attention/ops/pcp.py @@ -12,29 +12,42 @@ def _gather_prefill_cache_inputs( tensors: tuple[torch.Tensor, ...], slot_mapping: torch.Tensor, num_decode_tokens: int, + shard_decode_requests: bool = False, ) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]: - """Keep replicated decode writes local and gather partitioned prefills.""" + """Gather PCP cache inputs while preserving replicated KV-cache state. + + PCP-only execution shards decode requests across ranks. In that mode each + rank must gather the other owners' decode KV as well as partitioned prefill + KV so every PCP rank retains a complete cache replica. DCP execution keeps + decode requests replicated and uses the legacy prefill-only gather. + """ local_num_tokens = tensors[0].shape[0] assert all(tensor.shape[0] == local_num_tokens for tensor in tensors) assert 0 <= num_decode_tokens <= local_num_tokens # Replicated draft decodes use unexpanded slot mappings, even with DP padding. - if ( - num_decode_tokens == local_num_tokens - or slot_mapping.shape[0] <= local_num_tokens - ): + if slot_mapping.shape[0] <= local_num_tokens: return ( tuple(tensor[:num_decode_tokens] for tensor in tensors), slot_mapping[:num_decode_tokens], ) pcp_group = get_pcp_group() + pcp_size = pcp_group.world_size + gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens] + if shard_decode_requests: + gathered_inputs = tuple( + pcp_group.all_gather(tensor.contiguous(), dim=0) for tensor in tensors + ) + return gathered_inputs, gathered_slot_mapping + + if num_decode_tokens == local_num_tokens: + return tensors, slot_mapping[:num_decode_tokens] + gathered_prefills = tuple( pcp_group.all_gather(tensor[num_decode_tokens:].contiguous(), dim=0) for tensor in tensors ) - pcp_size = pcp_group.world_size - gathered_slot_mapping = slot_mapping[: pcp_size * local_num_tokens] if num_decode_tokens == 0: return gathered_prefills, gathered_slot_mapping @@ -58,6 +71,7 @@ def maybe_gather_mla_latent_cache_inputs( slot_mapping: torch.Tensor | None, num_decode_tokens: int | None, use_pcp: bool, + pcp_shard_decode_requests: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None]: if not use_pcp or num_decode_tokens is None: return kv_c_normed, k_pe, slot_mapping @@ -68,6 +82,7 @@ def maybe_gather_mla_latent_cache_inputs( (kv_c_normed, k_pe_flat), slot_mapping, num_decode_tokens, + pcp_shard_decode_requests, ) cache_k_pe = cache_k_pe_flat.view(-1, *k_pe.shape[1:]) return cache_kv_c, cache_k_pe, cache_slot_mapping @@ -78,11 +93,12 @@ def maybe_gather_indexer_k( slot_mapping: torch.Tensor, num_decode_tokens: int, use_pcp: bool, + pcp_shard_decode_requests: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: if not use_pcp: return k, slot_mapping (cache_k,), cache_slot_mapping = _gather_prefill_cache_inputs( - (k,), slot_mapping, num_decode_tokens + (k,), slot_mapping, num_decode_tokens, pcp_shard_decode_requests ) return cache_k, cache_slot_mapping diff --git a/vllm/v1/worker/gpu/pcp_manager.py b/vllm/v1/worker/gpu/pcp_manager.py index 89e6b1064b..6aec96bfe1 100644 --- a/vllm/v1/worker/gpu/pcp_manager.py +++ b/vllm/v1/worker/gpu/pcp_manager.py @@ -49,6 +49,7 @@ class PCPManager: pcp_world_size: int, pcp_rank: int, device: torch.device, + shard_decode_requests: bool, max_num_reqs: int | None = None, max_num_tokens: int | None = None, block_tables: BlockTables | None = None, @@ -62,6 +63,7 @@ class PCPManager: self.dcp_world_size = dcp_world_size self.dcp_rank = dcp_rank self.cp_interleave = cp_interleave + self.shard_decode_requests = shard_decode_requests self._global_batch: InputBatch | None = None self._local_batch: InputBatch | None = None @@ -248,6 +250,7 @@ class PCPManager: Decodes, and prefills too short to fill all eight, are replicated instead. """ + decode_ordinal = 0 num_chunks = 2 * self.pcp_world_size replicated = self.replicated_requests(num_scheduled_tokens, is_prefilling) for global_batch_req_idx, num_tokens in enumerate(num_scheduled_tokens): @@ -258,7 +261,16 @@ class PCPManager: if not replicated[global_batch_req_idx]: chunk_size = (query_len + num_chunks - 1) // num_chunks chunk_indices = (rank, num_chunks - 1 - rank) - else: # decodes, and short prefills under DCP, are replicated + elif self.shard_decode_requests: + chunk_size = query_len + # KV and hidden states are gathered back to every PCP rank, so + # decode ownership does not need to persist across steps. Use a + # compact decode-only ordinal to keep each step exactly + # balanced even when prefills and zero-token rows are present. + owner_rank = decode_ordinal % self.pcp_world_size + decode_ordinal += 1 + chunk_indices = (0,) if rank == owner_rank else () + else: # DCP requires decode queries on every participating rank. chunk_size = query_len chunk_indices = (0,) @@ -346,8 +358,18 @@ class PCPManager: segment.global_batch_slice.stop, dtype=np.int64, ) - # Cache insertion pairs one slot entry with each rank's local decode. - if replicated[segment.global_batch_req_idx] and rank != 0: + # Replicated rows contain identical KV, so rank 0 is the + # canonical writer. A sharded decode has exactly one owner and + # therefore needs no de-duplication here. + is_sharded_decode = ( + not bool(is_prefilling[segment.global_batch_req_idx]) + and self.shard_decode_requests + ) + if ( + replicated[segment.global_batch_req_idx] + and not is_sharded_decode + and rank != 0 + ): continue gathered_kv_write_mask[padded_gathered_slice] = True hidden_restore_idx[segment.global_batch_slice] = np.arange( @@ -829,6 +851,7 @@ def maybe_build_pcp_manager( pcp_world_size=pcp_size, pcp_rank=pcp_rank, device=device, + shard_decode_requests=parallel_config.pcp_shard_decode_requests, max_num_reqs=vllm_config.scheduler_config.max_num_seqs, max_num_tokens=vllm_config.scheduler_config.max_num_batched_tokens, block_tables=block_tables, From 1a9eaa3a686233610b75b6cc7605500f8903c9b9 Mon Sep 17 00:00:00 2001 From: Teng Ma Date: Fri, 2 Oct 2026 12:10:33 +0800 Subject: [PATCH 241/248] [Doc] Document ECMooncakeConnector for EPD disaggregation (#59196) Signed-off-by: Teng Ma Co-authored-by: Teng Ma --- docs/features/disagg_encoder.md | 7 +- docs/features/disagg_prefill.md | 2 +- docs/features/mooncake_connector_usage.md | 3 + docs/features/mooncake_ec_connector_usage.md | 158 ++++++++++++++++++ .../disaggregated_encoder/README.md | 9 +- 5 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 docs/features/mooncake_ec_connector_usage.md diff --git a/docs/features/disagg_encoder.md b/docs/features/disagg_encoder.md index ac304345fd..84debbacaa 100644 --- a/docs/features/disagg_encoder.md +++ b/docs/features/disagg_encoder.md @@ -45,9 +45,10 @@ The following scripts demonstrate the workflow with **ExampleConnector**: ### ECMooncakeConnector **ECMooncakeConnector** transfers encoder outputs using the Mooncake TransferEngine. -See the [Mooncake integration example](../../tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh) -for a complete 1 Encoder instance + 1 PD instance setup, including the producer and -consumer `--ec-transfer-config` settings and proxy configuration. +See the [ECMooncakeConnector Usage Guide](mooncake_ec_connector_usage.md) for +producer/consumer `--ec-transfer-config`, proxy `--ec-consumer-zmq-addrs`, and +transport options. The [Mooncake integration example](../../tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh) +is a complete 1 Encoder instance + 1 PD instance setup. With vLLM and Mooncake installed, run the example from the repository root: diff --git a/docs/features/disagg_prefill.md b/docs/features/disagg_prefill.md index e6a5797f26..013d57fa87 100644 --- a/docs/features/disagg_prefill.md +++ b/docs/features/disagg_prefill.md @@ -27,7 +27,7 @@ Now supports 9 types of connectors: --kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both", "kv_buffer_device":"cuda", "kv_connector_extra_config":{"backends":["UCX", "GDS"]}}' ``` -- **MooncakeConnector**: refer to [examples/disaggregated/mooncake_connector/run_mooncake_connector.sh](../../examples/disaggregated/mooncake_connector/run_mooncake_connector.sh) for the example usage of MooncakeConnector disaggregated prefilling. For detailed usage guide, see [MooncakeConnector Usage Guide](mooncake_connector_usage.md). +- **MooncakeConnector**: refer to [examples/disaggregated/mooncake_connector/run_mooncake_connector.sh](../../examples/disaggregated/mooncake_connector/run_mooncake_connector.sh) for the example usage of MooncakeConnector disaggregated prefilling. For detailed usage guide, see [MooncakeConnector Usage Guide](mooncake_connector_usage.md). For multimodal encoder-cache transfer, see [ECMooncakeConnector Usage Guide](mooncake_ec_connector_usage.md). - **MoRIIOConnector** (ROCm only): see [MoRI-IO Usage Guide](moriio_connector_usage.md) for example usage and detailed documentation. - **MultiConnector**: take advantage of the kv_connector_extra_config: dict[str, Any] already present in KVTransferConfig to stash all the connectors we want in an ordered list of kwargs.such as: diff --git a/docs/features/mooncake_connector_usage.md b/docs/features/mooncake_connector_usage.md index 701a1d0049..866d6d6bd4 100644 --- a/docs/features/mooncake_connector_usage.md +++ b/docs/features/mooncake_connector_usage.md @@ -38,6 +38,9 @@ python examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py --p Now you can send requests to the proxy server through port 8000. +For multimodal Encode-Prefill-Decode (encoder embeddings rather than KV +cache), see [ECMooncakeConnector Usage Guide](mooncake_ec_connector_usage.md). + ## Environment Variables - `VLLM_MOONCAKE_BOOTSTRAP_PORT`: Port for Mooncake bootstrap server diff --git a/docs/features/mooncake_ec_connector_usage.md b/docs/features/mooncake_ec_connector_usage.md new file mode 100644 index 0000000000..a710cbdfb6 --- /dev/null +++ b/docs/features/mooncake_ec_connector_usage.md @@ -0,0 +1,158 @@ +# ECMooncakeConnector Usage Guide + +`ECMooncakeConnector` moves multimodal **encoder embeddings** (encoder cache) +between a disaggregated encoder instance and a language-model instance over +[Mooncake Transfer Engine](https://github.com/kvcache-ai/Mooncake). Use it +when the vision encoder should scale independently of prefill and decode. + +For KV-cache transfer between prefiller and decoder, see +[MooncakeConnector Usage Guide](mooncake_connector_usage.md). That path uses +`MooncakeConnector`, not `ECMooncakeConnector`. + +Added in [PR #41567](https://github.com/vllm-project/vllm/pull/41567). +Overview of disaggregated encoding: +[Disaggregated Encoder](disagg_encoder.md). + +!!! important + Static proxy flags (`--ec-consumer-zmq-addrs`) support Mooncake EC only in + **1E + 1PD** mode (`--prefill-servers-urls disable`). For 1E + 1P + 1D, + register `ec_zmq_addrs` on the **prefill** consumer through the proxy's + dynamic registration API instead of the static CLI flag. See + `examples/disaggregated/disaggregated_encoder/README.md`. + +## Prerequisites + +Install Mooncake Transfer Engine: + +```bash +# CUDA 13 (vLLM default) +uv pip install mooncake-transfer-engine-cuda13 + +# CUDA 12 +uv pip install mooncake-transfer-engine +``` + +The two wheels are the same release built against different CUDA majors. +Importing the wrong one fails with +`libcudart.so.: cannot open shared object file`. + +## 1E + 1PD + +One encoder producer and one combined prefill/decode consumer. The consumer +listens on `ec_ip` + `ec_port` (ZMQ control). The proxy must advertise that +address with `--ec-consumer-zmq-addrs`. + +### Encoder (producer) + +The producer must run with `tensor_parallel_size=1`, +`pipeline_parallel_size=1`, and `data_parallel_size=1`. + +```bash +CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen2.5-VL-3B-Instruct \ + --port 19534 \ + --gpu-memory-utilization 0.35 \ + --enable-request-id-headers \ + --no-enable-prefix-caching \ + --max-num-batched-tokens 16384 \ + --allowed-local-media-path /path/to/media \ + --ec-transfer-config '{ + "ec_connector": "ECMooncakeConnector", + "ec_role": "ec_producer", + "ec_connector_extra_config": { + "mooncake_protocol": "rdma" + } + }' +``` + +### Prefill+Decode (consumer) + +```bash +CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen2.5-VL-3B-Instruct \ + --port 19537 \ + --gpu-memory-utilization 0.75 \ + --enable-mm-embeds \ + --enable-request-id-headers \ + --no-enable-prefix-caching \ + --allowed-local-media-path /path/to/media \ + --ec-transfer-config '{ + "ec_connector": "ECMooncakeConnector", + "ec_role": "ec_consumer", + "ec_ip": "127.0.0.1", + "ec_port": 19019, + "ec_connector_extra_config": { + "mooncake_protocol": "rdma" + } + }' +``` + +`--enable-mm-embeds` lets the PD instance inject transferred embeddings +instead of re-running the vision encoder. + +### Proxy + +```bash +python examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py \ + --host 0.0.0.0 \ + --port 10002 \ + --encode-servers-urls http://127.0.0.1:19534 \ + --prefill-servers-urls disable \ + --decode-servers-urls http://127.0.0.1:19537 \ + --ec-consumer-zmq-addrs tcp://127.0.0.1:19019 +``` + +`--ec-consumer-zmq-addrs` must align with `--decode-servers-urls` (the +combined PD instances). For data-parallel consumers, list each replica +consecutively and set `--ec-consumer-dp-size`. + +### Correctness script + +```bash +# TCP (no RDMA device required; CI default) +MOONCAKE_EC_PROTOCOL=tcp \ + bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh + +# RDMA +MOONCAKE_EC_PROTOCOL=rdma \ + bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh +``` + +The TCP path sets `MC_FORCE_TCP=1` so Transfer Engine does not auto-select +RDMA when an HCA is present. + +## Configuration + +Producer and consumer must use the same `mooncake_protocol`. + +| Setting | Where | Default | Notes | +| --------- | ------- | --------- | ------- | +| `mooncake_protocol` | `ec_connector_extra_config` | `rdma` | Set `tcp` when no verbs device is available. With TCP, also export `MC_FORCE_TCP=1`. | +| `ec_ip` / `ec_port` | consumer `ECTransferConfig` | `127.0.0.1` / `14579` | First ZMQ control port. Tensor-parallel rank `r` listens on `ec_port + dp_index * tp_size + r`. | +| `ec_buffer_size` | `ECTransferConfig` | `1e9` | Registered receive arena on the consumer, in bytes. | +| `control_timeout_s` | `ec_connector_extra_config` | `30` | Reservation RPC timeout. | +| `push_wait_timeout_s` | `ec_connector_extra_config` | `60` | How long the consumer waits for a matching encoder push. | +| `WITH_NVIDIA_PEERMEM` | process environment | `1` | Read by Mooncake. Set `0` on hosts without `nvidia-peermem` (DMA-BUF path). | +| `--ec-consumer-zmq-addrs` | proxy | empty | `tcp://:` for each PD replica. Required for Mooncake EC. | + +## 1E + 1P + 1D + +The filesystem example is `examples/disaggregated/disaggregated_encoder/disagg_1e1p1d_example.sh` +(`ECExampleConnector` for encoder cache, `NixlConnector` for KV). P→D KV can +use `MooncakeConnector` instead; see +[MooncakeConnector Usage Guide](mooncake_connector_usage.md). + +Do **not** pass `--ec-consumer-zmq-addrs` together with independent prefill +URLs: the static proxy flag is E+PD only. For Mooncake EC with a dedicated +prefill, use dynamic registration and set `ec_zmq_addrs` on the prefill +consumer as described in +`examples/disaggregated/disaggregated_encoder/README.md`. + +## Troubleshooting + +| Symptom | Likely cause | +| --------- | ---------------- | +| `Install mooncake-transfer-engine` / `libcudart.so.*` | Wrong or missing Mooncake wheel for this CUDA major. | +| `Failed to register memory ... Bad address [14]` | GPUDirect registration failed. Set `WITH_NVIDIA_PEERMEM=0`. | +| Proxy startup error about E+PD only | `--ec-consumer-zmq-addrs` used with a real `--prefill-servers-urls`. | +| Consumer waits then errors; encoder never finishes the push | `--ec-consumer-zmq-addrs` does not match consumer `ec_ip`/`ec_port`, or producer/consumer protocols differ. | +| TCP transfers stall on a host that also has RDMA | Export `MC_FORCE_TCP=1`. | +| Producer rejected at startup | Encoder used tensor, pipeline, or data parallelism. Keep the producer unsharded. | diff --git a/examples/disaggregated/disaggregated_encoder/README.md b/examples/disaggregated/disaggregated_encoder/README.md index ec10f23fc0..b2679d6cd1 100644 --- a/examples/disaggregated/disaggregated_encoder/README.md +++ b/examples/disaggregated/disaggregated_encoder/README.md @@ -57,7 +57,12 @@ The vllm instances and `disagg_encoder_proxy` supports local URIs with ```{"url" ## EC connector and KV transfer -The `ECExampleonnector` is used to store the encoder cache on local disk and facilitate transfer. To enable the encoder disaggregation feature, add the following configuration: +`ECExampleConnector` stores encoder cache on local disk. For RDMA/TCP transfer +without a shared filesystem, use `ECMooncakeConnector` (1E + 1PD only through +this proxy). See +[ECMooncakeConnector Usage Guide](../../../docs/features/mooncake_ec_connector_usage.md). + +To enable filesystem encoder disaggregation, add: ```bash # Add to encoder instance: @@ -104,6 +109,8 @@ If you enable prefill instance (`--prefill-servers-urls` not disabled), you will | `--encode-servers-urls` | Comma-separated list of encoder endpoints. Every multimodal item extracted from the request is fanned out to one of these URLs in a round-robin fashion. | | `--prefill-servers-urls` | Comma-separated list of prefill endpoints. Set to `disable`, `none`, or `""` to skip the dedicated prefill phase and run E+PD (encoder + combined prefill/decode). | | `--decode-servers-urls` | Comma-separated list of decode endpoints. Non-stream and stream paths both round-robin over this list. | +| `--ec-consumer-zmq-addrs` | Mooncake EC consumer control addresses, aligned with `--decode-servers-urls`. Required for `ECMooncakeConnector`. Supported only in E+PD mode (`--prefill-servers-urls disable`). | +| `--ec-consumer-dp-size` | Data-parallel replicas per EC consumer (default 1). | | `--host`, `--port` | Bind address for the proxy itself (defaults: `0.0.0.0:8000`). | ### Dynamic registration From f7999d2e4489126f1c21868699f3890021f01b73 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Fri, 2 Oct 2026 00:11:55 -0400 Subject: [PATCH 242/248] [Bugfix][HiSparse] Size the KV cache from the groups HiSparse allocates (#59450) Signed-off-by: Matthew Bonanni Signed-off-by: Lucas Wilkinson Co-authored-by: Claude Opus 5.5 Co-authored-by: Lucas Wilkinson --- tests/v1/core/test_kv_cache_utils.py | 138 ++++++++++++++++++++++----- tests/v1/worker/test_attn_utils.py | 3 + vllm/v1/attention/backends/utils.py | 5 +- vllm/v1/core/kv_cache_utils.py | 18 ++-- vllm/v1/hisparse/binding.py | 26 +++-- vllm/v1/hisparse/layout.py | 57 ++++++----- vllm/v1/kv_cache_interface.py | 6 +- vllm/v1/worker/gpu/attn_utils.py | 5 +- 8 files changed, 187 insertions(+), 71 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index fdc9198fff..7923e34d0f 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -59,7 +59,7 @@ from vllm.v1.core.kv_cache_utils import ( ) from vllm.v1.hisparse.layout import ( create_hisparse_layout, - get_hisparse_gpu_memory_usage, + get_hisparse_kv_cache_groups, ) from vllm.v1.kv_cache_interface import ( ChunkedLocalAttentionSpec, @@ -118,9 +118,6 @@ def test_hisparse_hma_uses_resolved_gpu_block_size( cache_role=SparseCacheRole.INDEXER, ), } - group_spec = UniformTypeKVCacheSpecs.from_specs(specs) - assert group_spec is not None - group = KVCacheGroupSpec(list(specs), group_spec) config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), model_config=SimpleNamespace( @@ -142,14 +139,9 @@ def test_hisparse_hma_uses_resolved_gpu_block_size( get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC, ), ) - indexer_spec = specs["model.layers.0.self_attn.indexer"] - assert get_hisparse_gpu_memory_usage(config, [group]) == ( - indexer_spec.max_memory_usage_bytes(config) - ) - monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30) cache_config = kv_cache_utils.get_kv_cache_config_from_groups( - config, [group], available_memory=2**30 + config, get_hisparse_kv_cache_groups(config, specs), available_memory=2**30 ) assert cache_config.num_blocks == 7 assert cache_config.hisparse_host_num_blocks is not None @@ -228,8 +220,6 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok cache_role=SparseCacheRole.INDEXER, ), } - group_spec = UniformTypeKVCacheSpecs.from_specs(specs) - assert group_spec is not None config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), model_config=SimpleNamespace( @@ -240,14 +230,123 @@ def test_hisparse_host_pool_must_fit_max_model_len(monkeypatch, extra_blocks, ok ) host_page = specs["model.layers.0.self_attn"].page_size_bytes host_budget = (4 + extra_blocks) * host_page - group = KVCacheGroupSpec(list(specs), group_spec) + groups = get_hisparse_kv_cache_groups(config, specs) if ok: - layout = create_hisparse_layout(config, [group], host_budget=host_budget) + layout = create_hisparse_layout(config, groups, host_budget=host_budget) assert layout.host_num_blocks == 4 + extra_blocks else: with pytest.raises(ValueError, match="increase host_pool_gib"): - create_hisparse_layout(config, [group], host_budget=host_budget) + create_hisparse_layout(config, groups, host_budget=host_budget) + + +@pytest.mark.parametrize( + "max_model_len,num_gpu_blocks,ok", + [ + (8192, 160, False), + (8192, 1000, True), + (32768, 656, False), + (32768, 657, True), + (-1, 657, True), + ], +) +def test_hisparse_pool_must_fit_max_model_len( + monkeypatch, max_model_len, num_gpu_blocks, ok +): + """Each resident group takes its own blocks from HiSparse's shared GPU pool, + so a pool that fits only the indexer pages must be rejected at startup. + Resident pages are capped at the in-flight window: at 32768 tokens with 2048 + in flight, one request needs 512 indexer + 4 x 32 resident + 16 hot blocks, + plus the null block. Auto-fit (-1) must pick a length that passes the same + check.""" + monkeypatch.setattr( + hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True + ) + monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30) + specs: dict[str, KVCacheSpec] = {} + for i in range(4): + specs[f"model.layers.{i}.self_attn"] = MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + is_index_group_leader=True, + ) + specs[f"model.layers.{i}.self_attn.indexer"] = MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + cache_role=SparseCacheRole.INDEXER, + ) + config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len)) + config.scheduler_config.max_num_batched_tokens = 2048 + # Async scheduling doubles the in-flight window, and with it the bound. + config.scheduler_config.async_scheduling = False + config.attention_config.hisparse_config = HiSparseConfig() + config.model_config.hf_config.index_topk = 128 + config.cache_config.num_gpu_blocks_override = num_gpu_blocks + config.cache_config.kv_cache_layout = "BLHNC" + # Workers report the resident/hot caches HiSparse derives. + specs |= { + name: group.kv_cache_spec + for group in get_hisparse_kv_cache_groups(config, specs) or [] + if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec)) + for name in group.layer_names + } + + if ok: + kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34]) + else: + with pytest.raises(ValueError, match="max seq len"): + kv_cache_utils.get_kv_cache_configs(config, [specs], [2**34]) + + +def test_hisparse_derived_specs_do_not_affect_kv_cache_layout(monkeypatch): + """Workers report HiSparse's resident/hot caches, which lay out their own raw + backing; KV cache layout resolution must skip them.""" + from vllm.v1.attention.backends.utils import resolve_kv_cache_layout + + monkeypatch.setattr( + hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True + ) + monkeypatch.delenv("VLLM_KV_CACHE_LAYOUT", raising=False) + specs: dict[str, KVCacheSpec] = { + "model.layers.0.self_attn": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=576, + dtype=torch.bfloat16, + is_index_group_leader=True, + ), + "model.layers.0.self_attn.indexer": MLAAttentionSpec( + block_size=64, + num_kv_heads=1, + head_size=128, + dtype=torch.bfloat16, + cache_role=SparseCacheRole.INDEXER, + ), + } + config = VllmConfig(model_config=ModelConfig(max_model_len=1024)) + config.attention_config.hisparse_config = HiSparseConfig() + config.model_config.hf_config.index_topk = 128 + reported = specs | { + name: group.kv_cache_spec + for group in get_hisparse_kv_cache_groups(config, specs) or [] + if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec)) + for name in group.layer_names + } + assert len(reported) > len(specs) + + layouts = [] + for worker_specs in (specs, reported): + config.cache_config.kv_cache_layout = None + layouts.append( + resolve_kv_cache_layout( + config, [["BLHNC", "LBNHC"]], list(worker_specs.values()) + ) + ) + assert layouts[0] == layouts[1] def test_hisparse_rejects_deepseek_v4(): @@ -260,9 +359,6 @@ def test_hisparse_rejects_deepseek_v4(): model_version="deepseek_v4", ) } - full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs) - assert full_uniform is not None - group = KVCacheGroupSpec(list(full_specs), full_uniform) config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()), model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)), @@ -270,11 +366,7 @@ def test_hisparse_rejects_deepseek_v4(): ) with pytest.raises(ValueError, match="does not support DeepSeek V4"): - create_hisparse_layout( - config, - [group], - host_budget=2**30, - ) + get_hisparse_kv_cache_groups(config, full_specs) @pytest.fixture(autouse=True) diff --git a/tests/v1/worker/test_attn_utils.py b/tests/v1/worker/test_attn_utils.py index 2166141638..0e6089d529 100644 --- a/tests/v1/worker/test_attn_utils.py +++ b/tests/v1/worker/test_attn_utils.py @@ -86,6 +86,9 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size( get_attn_backend=lambda backend=backend: backend, ) monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers) + monkeypatch.setattr( + attn_utils_module, "get_hisparse_kv_cache_groups", lambda *_: [] + ) config = SimpleNamespace( attention_config=SimpleNamespace(hisparse_config=object() if enabled else None) ) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index eb28fd1e78..529d816c06 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -291,7 +291,10 @@ def resolve_kv_cache_layout( # specs can re-interpret HNC with different sizes as long as the total number of # bytes is the same. If not block-compact, each spec must agree on HNC to alias # the same page (this aliasing is done by the Hybrid Memory Allocator, HMA). - kv_cache_specs = tuple(kv_cache_specs or ()) + # Specs without per-layer views lay out their own raw backing tensor. + kv_cache_specs = tuple( + spec for spec in kv_cache_specs or () if spec.has_layer_views + ) hnc_shapes = { (spec.num_heads, spec.num_states, spec.page_size_bytes) for spec in kv_cache_specs diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 4c8b807d16..46934d530b 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -21,7 +21,6 @@ from vllm.utils.math_utils import cdiv, round_up from vllm.utils.mem_utils import format_gib from vllm.utils.torch_utils import get_dtype_size from vllm.v1.hisparse.layout import ( - get_hisparse_gpu_memory_usage, get_hisparse_host_pool_bytes, get_hisparse_kv_cache_config, get_hisparse_kv_cache_groups, @@ -1155,7 +1154,9 @@ def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int: `available_memory` into `num_blocks`. Used to compute the effective KV cache capacity once `num_gpu_blocks_override` is applied. """ - return _get_kv_cache_bytes_per_block(kv_cache_groups) + return _get_kv_cache_bytes_per_block( + [group for group in kv_cache_groups if not group.host_resident] + ) def get_uniform_page_size(kv_cache_specs: Iterable[KVCacheSpec]) -> int: @@ -2500,12 +2501,10 @@ def _max_memory_usage_bytes_from_groups( Each group independently claims blocks from the shared pool, so a request consumes the sum of the per-group block counts, i.e. ``bytes_per_block * total_blocks``. """ + kv_cache_groups = [group for group in kv_cache_groups if not group.host_resident] if not kv_cache_groups: return 0 - if vllm_config.attention_config.hisparse_config is not None: - return get_hisparse_gpu_memory_usage(vllm_config, kv_cache_groups) - if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None: ( attn_group, @@ -2563,12 +2562,11 @@ def _estimate_max_model_len_from_groups( vllm_config.model_config.max_model_len = model_len if hisparse_enabled: try: - config = get_kv_cache_config_from_groups( + get_kv_cache_config_from_groups( vllm_config, kv_cache_groups, available_memory ) except ValueError: return False - return get_max_concurrency_for_kv_cache_config(vllm_config, config) >= 1 return ( _max_memory_usage_bytes_from_groups(vllm_config, kv_cache_groups) <= available_memory @@ -2678,6 +2676,9 @@ def _project_kv_cache_groups_to_worker( worker_layer_names = [ layer_name for layer_name in group.layer_names if layer_name in worker_spec ] + if len(worker_layer_names) == len(group.layer_names): + projected_groups.append(group) + continue group_spec = group.kv_cache_spec if worker_layer_names and isinstance(group_spec, UniformTypeKVCacheSpecs): group_spec = UniformTypeKVCacheSpecs( @@ -2793,9 +2794,6 @@ def get_kv_cache_configs( adjusted_memory.append(override * bytes_per_block) available_memory = adjusted_memory - if vllm_config.attention_config.hisparse_config is not None: - available_memory = [min(available_memory)] * len(available_memory) - # Reserve the null block BlockPool permanently holds back, so auto-fit and # the capacity check both plan against usable blocks. Allocation below # still uses the full memory. diff --git a/vllm/v1/hisparse/binding.py b/vllm/v1/hisparse/binding.py index 716211fbb3..88c8976b56 100644 --- a/vllm/v1/hisparse/binding.py +++ b/vllm/v1/hisparse/binding.py @@ -12,6 +12,7 @@ from vllm.config import VllmConfig from vllm.v1.hisparse.layout import ( HISPARSE_HOT_SUFFIX, HISPARSE_RESIDENT_SUFFIX, + get_hisparse_kv_cache_groups, ) from vllm.v1.hisparse.runtime import ( HiSparseCacheHandle, @@ -36,21 +37,20 @@ if TYPE_CHECKING: from vllm.v1.worker.gpu.block_table import BlockTables -def resolve_hisparse_block_size( +def resolve_hisparse_specs( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec], attn_layers: Mapping[str, "AttentionLayerBase"], -) -> None: - """Resolve a common kernel block size in-place for HiSparse MLA specs.""" - if vllm_config.attention_config.hisparse_config is None: - return +) -> dict[str, KVCacheSpec]: + """Resolve a common kernel block size for HiSparse MLA specs, and add the + resident/hot specs HiSparse derives from them.""" mla_specs = { name: spec for name, spec in kv_cache_spec.items() if isinstance(spec, MLAAttentionSpec) } if not mla_specs: - return + return kv_cache_spec block_sizes = {spec.block_size for spec in mla_specs.values()} if len(block_sizes) != 1: raise ValueError("HiSparse requires one scheduler block size.") @@ -62,10 +62,18 @@ def resolve_hisparse_block_size( "HiSparse requires a GPU block size supported by every sparse " f"attention and indexer backend: {error}" ) from error - kv_cache_spec.update( - (name, spec.copy_with_new_block_size(block_size)) + kv_cache_spec = kv_cache_spec | { + name: spec.copy_with_new_block_size(block_size) for name, spec in mla_specs.items() - ) + } + groups = get_hisparse_kv_cache_groups(vllm_config, kv_cache_spec) + assert groups is not None + return kv_cache_spec | { + layer_name: group.kv_cache_spec + for group in groups + if isinstance(group.kv_cache_spec, (HiSparseResidentSpec, HiSparseHotSpec)) + for layer_name in group.layer_names + } def allocate_hisparse_kv_caches( diff --git a/vllm/v1/hisparse/layout.py b/vllm/v1/hisparse/layout.py index e177768fc9..8ebc63c0ed 100644 --- a/vllm/v1/hisparse/layout.py +++ b/vllm/v1/hisparse/layout.py @@ -47,6 +47,10 @@ class HiSparseLayout: def get_hisparse_kv_cache_groups( vllm_config: VllmConfig, kv_cache_spec: dict[str, KVCacheSpec] ) -> list[KVCacheGroupSpec] | None: + """The groups HiSparse allocates: the host source group, then the GPU + groups sharing one device pool. Derived resident/hot specs in + `kv_cache_spec` (see `resolve_hisparse_specs`) are laid out again + from the attention specs they belong to.""" attention_config = getattr(vllm_config, "attention_config", None) if attention_config is None or attention_config.hisparse_config is None: return None @@ -59,9 +63,11 @@ def get_hisparse_kv_cache_groups( other_specs = { name: spec for name, spec in kv_cache_spec.items() - if not isinstance(spec, MLAAttentionSpec) + if not isinstance( + spec, (MLAAttentionSpec, HiSparseResidentSpec, HiSparseHotSpec) + ) } - if not mla_specs or not other_specs: + if not mla_specs: return None from vllm.v1.core.kv_cache_utils import get_kv_cache_groups @@ -69,7 +75,10 @@ def get_hisparse_kv_cache_groups( mla_group_spec = UniformTypeKVCacheSpecs.from_specs(mla_specs) assert mla_group_spec is not None mla_group = KVCacheGroupSpec(list(mla_specs), mla_group_spec) - return [mla_group, *get_kv_cache_groups(vllm_config, other_specs)] + regular_groups = ( + get_kv_cache_groups(vllm_config, other_specs) if other_specs else [] + ) + return _lay_out_hisparse_groups(vllm_config, [mla_group, *regular_groups]) def get_hisparse_host_pool_bytes(vllm_config: VllmConfig) -> int: @@ -112,24 +121,9 @@ def _partition_hisparse_specs( return source_specs, indexer_specs -def get_hisparse_gpu_memory_usage( - vllm_config: VllmConfig, - kv_cache_groups: list[KVCacheGroupSpec], -) -> int: - _, indexer_specs = _partition_hisparse_specs(kv_cache_groups) - return sum( - spec.max_memory_usage_bytes(vllm_config) for spec in indexer_specs.values() - ) + sum( - group.kv_cache_spec.max_memory_usage_bytes(vllm_config) - for group in kv_cache_groups[1:] - ) - - -def create_hisparse_layout( - vllm_config: VllmConfig, - groups: list[KVCacheGroupSpec], - host_budget: int, -) -> HiSparseLayout: +def _lay_out_hisparse_groups( + vllm_config: VllmConfig, groups: list[KVCacheGroupSpec] +) -> list[KVCacheGroupSpec]: source_specs, indexer_specs = _partition_hisparse_specs(groups) block_sizes = { spec.block_size for spec in (*source_specs.values(), *indexer_specs.values()) @@ -225,11 +219,25 @@ def create_hisparse_layout( enable_kv_transfer=True, ) regular_groups = groups[1:] - gpu_groups = [indexer_group, *resident_groups, *hot_groups, *regular_groups] + return [ + source_group, + indexer_group, + *resident_groups, + *hot_groups, + *regular_groups, + ] + +def create_hisparse_layout( + vllm_config: VllmConfig, + groups: list[KVCacheGroupSpec], + host_budget: int, +) -> HiSparseLayout: + """Size the host pool for groups laid out by `get_hisparse_kv_cache_groups`.""" + (source_group,) = [group for group in groups if group.host_resident] shared_host_pool = use_shared_hisparse_host_pool(vllm_config) host_block_stride = get_hisparse_host_block_stride( - sum(spec.page_size_bytes for spec in source_specs.values()), + source_group.kv_cache_spec.page_size_bytes, use_shared_host_pool=shared_host_pool, ) host_num_blocks = host_budget // host_block_stride @@ -237,6 +245,7 @@ def create_hisparse_layout( raise ValueError("HiSparse has no allocatable host blocks.") # Every computed page needs a host block, so one request at max_model_len # must fit alongside the pool's null block and a copy-on-write tail. + gpu_block_size = source_group.kv_cache_spec.block_size min_host_blocks = cdiv(vllm_config.model_config.max_model_len, gpu_block_size) + 2 if host_num_blocks < min_host_blocks: raise ValueError( @@ -246,7 +255,7 @@ def create_hisparse_layout( return HiSparseLayout( source_group=source_group, - device_groups=gpu_groups, + device_groups=[group for group in groups if not group.host_resident], host_num_blocks=host_num_blocks, host_block_stride=host_block_stride, shared_host_pool=shared_host_pool, diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 248f03dd02..846e6bcad0 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -472,9 +472,11 @@ class HiSparseResidentSpec(KVCacheSpec): return cdiv(num_tokens, self.block_size) def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int: - return cdiv(vllm_config.model_config.max_model_len, self.block_size) * ( - self.page_size + max_blocks = self.max_admission_blocks_per_request( + max_in_flight_tokens=vllm_config.max_in_flight_tokens, + max_model_len=vllm_config.model_config.max_model_len, ) + return max_blocks * self.page_size @property def has_layer_views(self) -> bool: diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 19de125a9b..9e9ffd7be1 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -23,7 +23,7 @@ from vllm.v1.attention.backend import ( from vllm.v1.attention.backends.utils import create_fast_prefill_custom_backend from vllm.v1.hisparse.binding import ( init_hisparse_kv_cache, - resolve_hisparse_block_size, + resolve_hisparse_specs, ) from vllm.v1.kv_cache_interface import ( AttentionSpec, @@ -142,7 +142,8 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]: if isinstance(spec, AttentionSpec): spec = attn_module.get_attn_backend().customize_spec(spec) kv_cache_spec[layer_name] = spec - resolve_hisparse_block_size(vllm_config, kv_cache_spec, attn_layers) + if vllm_config.attention_config.hisparse_config is not None: + kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers) return kv_cache_spec From 0cbac6cd1305f710e12193596b27488397bcb205 Mon Sep 17 00:00:00 2001 From: Mohit Singla <140096022+mohit-sarvam@users.noreply.github.com> Date: Fri, 2 Oct 2026 09:58:19 +0530 Subject: [PATCH 243/248] [Model][Spec Decode] Enable EAGLE3/DSpark pipeline parallelism for Sarvam MLA (#55902) Signed-off-by: mohit-sarvam Co-authored-by: Codex Co-authored-by: Thien Tran --- vllm/model_executor/models/sarvam.py | 36 ++++++++++++++++++---------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/vllm/model_executor/models/sarvam.py b/vllm/model_executor/models/sarvam.py index e7db1d6801..d0e2336a83 100644 --- a/vllm/model_executor/models/sarvam.py +++ b/vllm/model_executor/models/sarvam.py @@ -73,6 +73,7 @@ from .utils import ( make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, + spec_decode_needs_target_embed, ) @@ -460,7 +461,9 @@ class SarvamMLABlock(nn.Module): } ) class SarvamMLAModel(nn.Module, EagleModelMixin): - """Sarvam MLA backbone with stage-local EAGLE3 auxiliary capture.""" + """Sarvam MLA backbone with EAGLE3 auxiliary capture across pipeline stages.""" + + supports_aux_hidden_states_over_pp = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_stacked={ @@ -487,8 +490,10 @@ class SarvamMLAModel(nn.Module, EagleModelMixin): self.vocab_size = config.vocab_size self.embed_dim = config.hidden_size self.tie_word_embeddings = getattr(config, "tie_word_embeddings", False) - if get_pp_group().is_first_rank or ( - self.tie_word_embeddings and get_pp_group().is_last_rank + if ( + get_pp_group().is_first_rank + or (self.tie_word_embeddings and get_pp_group().is_last_rank) + or spec_decode_needs_target_embed(vllm_config) ): self.embed_tokens = VocabParallelEmbedding( self.vocab_size, @@ -530,13 +535,10 @@ class SarvamMLAModel(nn.Module, EagleModelMixin): ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: """Run this stage and optionally return its auxiliary hidden states. - Auxiliary captures are local to this stage, matching Qwen3 MoE. - Pipeline transport carries only hidden states and residual; EAGLE3 - capture across pipeline stages is not supported by this model. - Returns: - Intermediate tensors on non-final stages. On the final stage, - normalized hidden states, paired with auxiliary states if captured. + Intermediate tensors with local auxiliary states on non-final stages. + On the final stage, normalized hidden states paired with the ordered + auxiliary states from all stages if captured. """ if get_pp_group().is_first_rank: @@ -551,9 +553,12 @@ class SarvamMLAModel(nn.Module, EagleModelMixin): hidden_states = intermediate_tensors["hidden_states"] residual = intermediate_tensors["residual"] - aux_hidden_states = self._maybe_add_hidden_state( - [], self.start_layer, hidden_states, residual - ) + remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors) + aux_hidden_states: list[torch.Tensor] = [] + if get_pp_group().is_first_rank: + self._maybe_add_hidden_state( + aux_hidden_states, self.start_layer, hidden_states, residual + ) for layer_idx, layer in enumerate( islice(self.layers, self.start_layer, self.end_layer), start=self.start_layer, @@ -569,13 +574,18 @@ class SarvamMLAModel(nn.Module, EagleModelMixin): if not get_pp_group().is_last_rank: return IntermediateTensors( - {"hidden_states": hidden_states, "residual": residual} + { + "hidden_states": hidden_states, + "residual": residual, + **self.pack_local_aux_hidden_states(aux_hidden_states), + } ) if residual is None: hidden_states = self.norm(hidden_states) else: hidden_states, _ = self.norm(hidden_states, residual) + aux_hidden_states = remote_aux + aux_hidden_states if len(aux_hidden_states) > 0: return hidden_states, aux_hidden_states return hidden_states From 4056c8ac1f8a7e8fb50cf1c56ff96649fcadccc6 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 2 Oct 2026 02:04:21 -0400 Subject: [PATCH 244/248] [Bugfix][HiSparse] Fix MTP acceptance collapse under FULL graphs with a saturated GPU pool (#59309) Signed-off-by: Lucas Wilkinson Signed-off-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Matthew Bonanni --- docs/design/hisparse.md | 5 +- .../v1/attention/test_sparse_mla_backends.py | 137 +++++++++--- tests/v1/e2e/general/test_hisparse.py | 77 ++++--- .../unit/test_hisparse_connector.py | 202 ++++++++++++++++++ tests/v1/worker/test_utils.py | 26 +-- .../kv_connector/v1/hisparse/worker.py | 43 +++- .../layers/attention_layer_base.py | 2 + vllm/v1/attention/backends/mla/index_group.py | 22 +- vllm/v1/hisparse/binding.py | 1 + vllm/v1/hisparse/runtime.py | 21 +- vllm/v1/worker/gpu/spec_decode/speculator.py | 12 +- 11 files changed, 426 insertions(+), 122 deletions(-) diff --git a/docs/design/hisparse.md b/docs/design/hisparse.md index 2c72ddcbcc..8651b3a06d 100644 --- a/docs/design/hisparse.md +++ b/docs/design/hisparse.md @@ -133,7 +133,10 @@ duplicate mapping. Speculative decoding resolves all verification rows of a request in one pass: one block resolves the union of the rows' top-k against the request's hot-cache state, so rows that select the same host row share its hot row and no row -evicts a hot row another row of the step still reads. +evicts a hot row another row of the step still reads. Draft layers write their +rows after the target forward, so their host mirror runs at the start of the +next step, after the drafter, and the step's page transfers are submitted +behind it. ## P/D import target diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index a6517430ad..b41db5a0f1 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -1943,6 +1943,10 @@ def _make_hisparse_cache_handle( return HiSparseCacheHandle(runtime) +def _valid_rows(num_rows: int) -> torch.Tensor: + return torch.tensor([num_rows], dtype=torch.int32, device=DEVICE_TYPE) + + @requires_hisparse_ops def test_hisparse_uses_graph_stable_request_state_mapping(): device = torch.device(DEVICE_TYPE) @@ -1969,6 +1973,7 @@ def test_hisparse_uses_graph_stable_request_state_mapping(): block_table=torch.tensor([[1]], dtype=torch.int32, device=device), logical_topk_indices=torch.tensor([[0]], dtype=torch.int32, device=device), block_size=block_size, + num_valid_rows=_valid_rows(1), ) torch.accelerator.synchronize() @@ -2011,6 +2016,7 @@ def test_hisparse_maps_speculative_rows_through_request_state(): block_table=torch.zeros((1, 1), dtype=torch.int32, device=device), logical_topk_indices=torch.zeros((4, 1), dtype=torch.int32, device=device), block_size=block_size, + num_valid_rows=_valid_rows(4), ) torch.accelerator.synchronize() @@ -2072,6 +2078,7 @@ def test_hisparse_speculative_rows_resolve_host_misses_consistently( block_table=block_table, logical_topk_indices=topk.clone(), block_size=block_size, + num_valid_rows=_valid_rows(num_rows), ) torch.accelerator.synchronize() @@ -2115,12 +2122,99 @@ def test_hisparse_speculative_rows_count_a_shared_load_once(): block_table=block_table, logical_topk_indices=topk, block_size=block_size, + num_valid_rows=_valid_rows(2), ) torch.accelerator.synchronize() assert runtime.index_group.swap_stats.cpu().tolist() == [top_k, top_k] +@requires_hisparse_ops +def test_hisparse_resolver_reads_current_request_ids(): + """The resolver must see this call's request ids, not the previous call's. + + It runs on the copy stream after waiting only on ``logical_topk_ready``, + which the indexer records before attention writes the ids. An MTP drafter + alternates four rows per request with one, so stale ids resolve rows + against other requests' KV. Stalling the compute stream exposes the race. + """ + device = torch.device(DEVICE_TYPE) + block_size, row_width, top_k, blocks_per_req, num_reqs = 64, 16, 64, 4, 2 + kv_pool = torch.randn( + (num_reqs * blocks_per_req + 1, block_size, row_width), dtype=torch.float32 + ).pin_memory() + runtime = _make_hisparse_runtime( + top_k=top_k, + device_buffer_size=4 * top_k, + max_num_reqs=num_reqs, + row_width=row_width, + block_size=block_size, + max_swap_rows=4 * num_reqs, + ) + runtime.bind_source_cache(kv_pool) + ready = torch.Event() + runtime.index_group.logical_topk_ready = ready + cache = HiSparseCacheHandle(runtime) + cache.all_context_pages_resident = False + cache.source_block_table = torch.arange( + 1, num_reqs * blocks_per_req + 1, dtype=torch.int32, device=device + ).view(num_reqs, blocks_per_req) + + index_group = object.__new__(HiSparseMLAIndexGroup) + index_group.caches = [cache] + index_group.physical_topk_indices = torch.empty( + (4 * num_reqs + 1, top_k), dtype=torch.int32, device=device + ) + + # Four rows per request, then one per request plus a CUDA-graph padding + # row, which maps to request 0 and must be skipped. + for rows_per_req, num_padding in ((4, 0), (1, 1)): + num_rows = num_reqs * rows_per_req + req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device) + req_ids = torch.cat( + [ + req_ids.repeat_interleave(rows_per_req), + torch.zeros(num_padding, dtype=torch.int32, device=device), + ] + ) + topk = torch.stack( + [ + torch.randperm(blocks_per_req * block_size, device=device)[:top_k] + for _ in range(req_ids.numel()) + ] + ).to(torch.int32) + query_start_loc = torch.arange( + 0, num_rows + 1, rows_per_req, dtype=torch.int32, device=device + ) + attn_metadata = SimpleNamespace( + req_id_per_token=req_ids, + query_start_loc=torch.cat([query_start_loc, query_start_loc[-1:]]), + block_size=block_size, + ) + runtime.begin_forward() + ready.record(current_stream()) + torch.cuda._sleep(200_000_000) + hot_indices = index_group.convert_logical_to_physical_topk( + 0, topk, attn_metadata, block_stride_rows=None, return_valid_counts=False + ) + torch.accelerator.synchronize() + + assert (hot_indices[num_rows:] == -1).all() + global_ref = _triton_convert_reference_impl( + req_ids[:num_rows], + cache.source_block_table, + topk[:num_rows], + block_size, + top_k, + ) + gathered = runtime.hot.attention_cache.reshape(-1, row_width)[ + hot_indices[:num_rows].to(torch.long) + ].cpu() + torch.testing.assert_close( + gathered, kv_pool.reshape(-1, row_width)[global_ref.cpu().to(torch.long)] + ) + + @requires_hisparse_ops def test_hisparse_union_table_scales_with_union_bound(): """Shared memory follows the per-request union bound, not the hot buffer. @@ -2157,6 +2251,7 @@ def test_hisparse_union_table_scales_with_union_bound(): block_table=block_table, logical_topk_indices=topk, block_size=block_size, + num_valid_rows=_valid_rows(req_ids.numel()), ) torch.accelerator.synchronize() @@ -2217,6 +2312,7 @@ def test_hisparse_resident_rows_bypass_hot_lru(): block_table=source_table, logical_topk_indices=topk, block_size=block_size, + num_valid_rows=_valid_rows(request_ids.numel()), return_valid_counts=True, ) torch.accelerator.synchronize() @@ -2280,6 +2376,7 @@ def test_hisparse_swap_in_preserves_rows_across_eviction(): block_table=block_table, logical_topk_indices=topk.clone(), block_size=block_size, + num_valid_rows=_valid_rows(req_ids.numel()), return_valid_counts=True, ) torch.accelerator.synchronize() @@ -2361,6 +2458,7 @@ def test_hisparse_multi_step_swaps_match_independent(): req_id_per_token=req_ids, block_table=block_table, block_size=block_size, + num_valid_rows=_valid_rows(req_ids.numel()), ) producer_indices = [ producer.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks @@ -2435,6 +2533,7 @@ def test_hisparse_multi_step_writes_request_major_output(): block_table, logical[:, step], block_size=block_size, + num_valid_rows=_valid_rows(request_ids.numel()), return_valid_counts=True, attention_indices_out=physical[:, step], valid_counts_out=valid_counts[:, step], @@ -2619,6 +2718,7 @@ def test_hisparse_remaps_strided_hma_rows_for_attention(): [[0, 1, 2, 3]], dtype=torch.int32, device=device ), block_size=block_size, + num_valid_rows=_valid_rows(1), ) assert runtime.hot.attention_cache is not None @@ -2752,6 +2852,7 @@ def test_hisparse_newest_write_and_recycled_slot_invalidation(): block_table=block_table, logical_topk_indices=topk, block_size=block_size, + num_valid_rows=_valid_rows(req_ids.numel()), ) torch.accelerator.synchronize() stale_hot_slot = hot_indices[0, 0].item() @@ -2767,6 +2868,7 @@ def test_hisparse_newest_write_and_recycled_slot_invalidation(): block_table=block_table, logical_topk_indices=topk, block_size=block_size, + num_valid_rows=_valid_rows(req_ids.numel()), ) torch.accelerator.synchronize() idx = hot_indices.cpu().tolist()[0][0] @@ -3200,8 +3302,6 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention(): index_group.physical_topk_indices = torch.empty( (num_tokens + 1, 4), dtype=torch.int32, device=device ) - index_group.row_indices = torch.arange(num_tokens, dtype=torch.int32, device=device) - index_group.request_ids = torch.empty_like(index_group.row_indices) impl = SimpleNamespace( kv_lora_rank=1, index_group=index_group, @@ -3255,39 +3355,6 @@ def test_hisparse_keeps_speculative_rows_as_decodes(): assert builder.reorder_batch_threshold == 4 -def test_hisparse_decode_skips_padding_rows(): - """Padding rows are masked, and the ids stay in the group's persistent buffer. - - The resolver reads them on the HiSparse copy stream, where a compute-stream - temporary can already be recycled (CUDA graphs do), scrambling rows' requests. - """ - topk = torch.arange(20, dtype=torch.int32).view(10, 2) - cache = SimpleNamespace( - source_block_table=torch.zeros((2, 1), dtype=torch.int32), - swap_in=MagicMock(return_value=topk), - ) - index_group = object.__new__(HiSparseMLAIndexGroup) - index_group.caches = [cache] - index_group.physical_topk_indices = torch.empty((11, 2), dtype=torch.int32) - index_group.row_indices = torch.arange(10, dtype=torch.int32) - index_group.request_ids = torch.empty_like(index_group.row_indices) - metadata = SimpleNamespace( - query_start_loc=torch.tensor([0, 8, 9], dtype=torch.int32), - req_id_per_token=torch.tensor([0] * 8 + [1, 0], dtype=torch.int32), - block_size=64, - ) - - index_group.convert_logical_to_physical_topk( - 0, topk, metadata, block_stride_rows=None, return_valid_counts=False - ) - - request_ids = cache.swap_in.call_args.args[0] - assert request_ids.data_ptr() == index_group.request_ids.data_ptr() - torch.testing.assert_close( - request_ids, torch.tensor([0] * 8 + [1, -1], dtype=torch.int32) - ) - - def test_flashinfer_hisparse_decode_runs_batched_attention(): device = torch.device("cpu") num_tokens = 6 diff --git a/tests/v1/e2e/general/test_hisparse.py b/tests/v1/e2e/general/test_hisparse.py index 573b424ea7..2409b5cb69 100644 --- a/tests/v1/e2e/general/test_hisparse.py +++ b/tests/v1/e2e/general/test_hisparse.py @@ -95,6 +95,8 @@ def test_hisparse_spill_and_prefix_restore( A regression omitted attention metadata before FULL graph replay, so decode completed normally while its newly written KV rows were never copied to host. + MTP layers write after the target forward; their rows used to be mirrored + before the drafter wrote them, so the drafter later read stale host rows. """ capability = current_platform.get_device_capability() if capability is None or capability.major < 9: @@ -171,49 +173,70 @@ def test_hisparse_spill_and_prefix_restore( model_runner.cudagraph_manager, "run_fullgraph", record_fullgraph ) + # The MTP layer writes its rows after the target forward, so it is + # mirrored at the start of the next step instead of in finish_forward. + draft_layers = set(worker._draft_layers) + assert draft_layers original_finish_forward = worker.finish_forward - verified_full_replay_rows = 0 + original_finish_previous_step = worker._finish_previous_step + verified_rows = {"target": 0, "draft": 0} copied_nonzero_kv = False + draft_check_pending = False + + def check_host_rows(layer_indices: set[int], kind: str) -> None: + nonlocal copied_nonzero_kv + # Synchronize only the test's inspection, not the worker itself. + with gpu_sync_allowed(): + torch.accelerator.synchronize() + assert worker._row_mirrors + for layer_index in layer_indices: + cache = worker.cache_handles[layer_index] + source_index = cache.runtime.resident_source_index + for mirror in worker._row_mirrors: + source_row = mirror.source_starts[source_index] + destination_row = mirror.destination_start + num_rows = mirror.num_rows + source_block, source_offset = divmod( + source_row, worker.kernel_block_size + ) + with gpu_sync_allowed(): + gpu_rows = worker.resident_caches[layer_index][ + source_block, source_offset : source_offset + num_rows + ].cpu() + host_rows = worker.host_caches[layer_index][ + destination_row : destination_row + num_rows + ] + assert torch.equal(host_rows, gpu_rows) + copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item()) + verified_rows[kind] += num_rows def finish_forward(): - nonlocal full_replay_pending, verified_full_replay_rows - nonlocal copied_nonzero_kv + nonlocal full_replay_pending, draft_check_pending try: original_finish_forward() if not full_replay_pending: return - # Synchronize only the test's inspection, not finish_forward itself. - with gpu_sync_allowed(): - torch.accelerator.synchronize() - assert worker._row_mirrors - for layer_index, cache in enumerate(worker.cache_handles): - source_index = cache.runtime.resident_source_index - for mirror in worker._row_mirrors: - source_row = mirror.source_starts[source_index] - destination_row = mirror.destination_start - num_rows = mirror.num_rows - source_block, source_offset = divmod( - source_row, worker.kernel_block_size - ) - with gpu_sync_allowed(): - gpu_rows = worker.resident_caches[layer_index][ - source_block, source_offset : source_offset + num_rows - ].cpu() - host_rows = worker.host_caches[layer_index][ - destination_row : destination_row + num_rows - ] - assert torch.equal(host_rows, gpu_rows) - copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item()) - verified_full_replay_rows += num_rows + target_layers = set(range(len(worker.cache_handles))) - draft_layers + check_host_rows(target_layers, "target") + draft_check_pending = True finally: full_replay_pending = False + def finish_previous_step(): + nonlocal draft_check_pending + original_finish_previous_step() + if draft_check_pending: + draft_check_pending = False + check_host_rows(draft_layers, "draft") + monkeypatch.setattr(worker, "finish_forward", finish_forward) + monkeypatch.setattr(worker, "_finish_previous_step", finish_previous_step) expected = runner.generate_greedy([target], max_tokens=8) assert full_graph_calls > 0 - assert verified_full_replay_rows > 0 + assert verified_rows["target"] > 0 + assert verified_rows["draft"] > 0 assert copied_nonzero_kv runner.generate_greedy(pressure, max_tokens=8) diff --git a/tests/v1/kv_connector/unit/test_hisparse_connector.py b/tests/v1/kv_connector/unit/test_hisparse_connector.py index 384460fa2c..a2eeb8cd63 100644 --- a/tests/v1/kv_connector/unit/test_hisparse_connector.py +++ b/tests/v1/kv_connector/unit/test_hisparse_connector.py @@ -1,15 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextlib +import ctypes from types import SimpleNamespace from unittest.mock import MagicMock import pytest import torch +from vllm.config.compilation import CUDAGraphMode from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( ExampleConnector, ) +from vllm.distributed.kv_transfer.kv_connector.v1.hisparse import ( + worker as worker_module, +) from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import ( HiSparseConnector, HiSparseConnectorMetadata, @@ -19,8 +25,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import ( HiSparseConnectorWorker, ) from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector +from vllm.v1.hisparse import runtime as runtime_module from vllm.v1.hisparse.coordinator import get_hisparse_coordinator from vllm.v1.hisparse.runtime import HiSparseCacheHandle +from vllm.v1.hisparse.types import ( + SparseKVOffloadCommand, + SparseKVPageTransfer, + SparseKVRowMirror, +) from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector @@ -83,6 +95,7 @@ def test_full_graph_step_prepares_host_mirror_outside_model(): worker._group_leaders = (("layer", handle),) worker._per_layer_mirrored = set() worker._submitted_mirror_layers = set() + worker._draft_layers = () worker.is_host_writer = True worker._enqueue_row_dma = MagicMock() worker.start_step = MagicMock( @@ -116,6 +129,195 @@ def test_full_graph_step_prepares_host_mirror_outside_model(): runtime.invalidate_written_slots.assert_called_once() +class _FakeEvent: + log: list[tuple[str, object]] = [] + + def __init__(self, *args, **kwargs) -> None: + pass + + def record(self, stream=None) -> None: + self.log.append(("event", self)) + + def query(self) -> bool: + return True + + def synchronize(self) -> None: + pass + + +@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.FULL, CUDAGraphMode.NONE]) +def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode): + """Draft-layer host rows must hold the drafter's writes before page release. + + The drafter writes its layers' rows after the target forward. Mirroring + them at the end of the target forward (the only mirror a FULL graph gets) + leaves pre-draft rows on the host, and the page is still reported clean. + """ + block_size, width = 4, 8 + resident_block, host_block, transfer_id = 2, 1, 7 + log: list[tuple[str, object]] = [] + _FakeEvent.log = log + + def swap_blocks_batch(src, dst, sizes): + for src_ptr, dst_ptr, size in zip(src.tolist(), dst.tolist(), sizes.tolist()): + ctypes.memmove(dst_ptr, src_ptr, size) + log.append(("dma", dst_ptr)) + + monkeypatch.setattr(torch, "Event", _FakeEvent) + monkeypatch.setattr(torch, "Stream", lambda *args, **kwargs: MagicMock()) + monkeypatch.setattr(torch.cuda, "Stream", lambda *args, **kwargs: MagicMock()) + monkeypatch.setattr(torch.cuda, "stream", lambda _: contextlib.nullcontext()) + compute_stream = MagicMock() + monkeypatch.setattr(worker_module, "current_stream", lambda: compute_stream) + monkeypatch.setattr(worker_module.ops, "swap_blocks_batch", swap_blocks_batch) + monkeypatch.setattr( + runtime_module, + "get_forward_context", + lambda: SimpleNamespace(cudagraph_runtime_mode=cg_mode), + ) + + slots = torch.arange(block_size) + resident_block * block_size + request_state_indices = torch.full((1,), -1, dtype=torch.int32) + handles = [] + for draft_layer in (False, True): + runtime = SimpleNamespace( + is_group_leader=True, + eager_host_mirror=True, + resident_source_index=0, + host_cache=torch.zeros(4 * block_size, width), + request_state_indices=request_state_indices, + begin_forward=MagicMock(), + invalidate_written_slots=MagicMock(), + ) + handle = HiSparseCacheHandle(runtime) + handle.view = SimpleNamespace( + block_size=block_size, cache=torch.zeros(4, block_size, width) + ) + handle.slot_mapping = slots + handle.mirror_slot_mapping = slots + handle.draft_layer = draft_layer + handles.append(handle) + target, draft = handles + # Rows the draft layer held before this step's drafter ran. + draft.view.cache[resident_block] = -1.0 + + worker = HiSparseConnectorWorker( + SimpleNamespace( + scheduler_config=SimpleNamespace(max_num_batched_tokens=16), + num_lookahead_tokens=3, + ), + MagicMock(), + ) + worker.initialize( + handles, + ["target", "draft"], + MagicMock(), + max_num_reqs=1, + host_num_blocks=4, + device=torch.device("cpu"), + pinned_host_pools=[], + ) + connector = object.__new__(HiSparseConnector) + connector.connector_worker = worker + connector._get_connector_metadata = MagicMock( + return_value=HiSparseConnectorMetadata( + SparseKVOffloadCommand( + [ + SparseKVPageTransfer( + transfer_id=transfer_id, + host_block_id=host_block, + resident_block_ids=(resident_block,), + after_forward=True, + ) + ] + ), + (), + (), + { + "req": ( + SparseKVRowMirror( + source_starts=(resident_block * block_size,), + destination_start=host_block * block_size, + num_rows=block_size, + ), + ) + }, + True, + ) + ) + # A verification step: four query tokens of one request fill the page. + attn_metadata = SimpleNamespace( + num_actual_tokens=block_size, + num_decode_tokens=0, + num_reqs=1, + max_query_len=block_size, + req_id_per_token=torch.zeros(block_size, dtype=torch.int32), + ) + + connector.start_load_kv( + SimpleNamespace(), + request_state_indices=None, + request_ids=["req"], + num_tokens=block_size, + attn_metadata={"target": attn_metadata, "draft": attn_metadata}, + ) + target.view.cache[resident_block] = 1.0 + if cg_mode != CUDAGraphMode.FULL: + target.finish_kv_update() + connector.finish_forward() + enqueued_before_draft = list(worker._enqueued_transfer_ids) + + # The drafter writes the draft layer's rows for the same positions. + if cg_mode != CUDAGraphMode.FULL: + draft.prepare_group_for_batch(attn_metadata) + draft.view.cache[resident_block] = 2.0 + if cg_mode != CUDAGraphMode.FULL: + draft.finish_kv_update() + # The next step's start mirrors the drafter's rows, then hands the page over. + compute_stream.wait_event.reset_mock() + connector._get_connector_metadata.return_value = HiSparseConnectorMetadata( + None, (), (), {}, True + ) + connector.start_load_kv( + SimpleNamespace(), + request_state_indices=None, + request_ids=[], + num_tokens=0, + attn_metadata={}, + ) + ((completion_event, _),) = worker._pending_transfer_events + worker_meta = connector.build_connector_worker_meta() + + host_rows = slice(host_block * block_size, (host_block + 1) * block_size) + torch.testing.assert_close(target.runtime.host_cache[host_rows], torch.ones(4, 8)) + torch.testing.assert_close( + draft.runtime.host_cache[host_rows], torch.full((4, 8), 2.0) + ) + draft_host = draft.runtime.host_cache + draft_host_range = range( + draft_host.data_ptr(), + draft_host.data_ptr() + draft_host.numel() * draft_host.element_size(), + ) + draft_copies = [ + index + for index, (kind, ptr) in enumerate(log) + if kind == "dma" and ptr in draft_host_range + ] + # One draft-layer copy, and the page is not handed back before it. + assert len(draft_copies) == 1 + assert enqueued_before_draft == [] + assert log.index(("event", completion_event)) > draft_copies[0] + # The next forward waits for the draft copy: the scheduler may already + # have handed the blocks it reads and writes to another request. + draft_copy_event = next( + entry[1] for entry in log[draft_copies[0] :] if entry[0] == "event" + ) + assert compute_stream.wait_event.call_args_list[0].args == (draft_copy_event,) + assert worker_meta is not None + assert worker_meta.enqueued_transfer_counts == {transfer_id: 1} + assert worker_meta.completed_transfer_counts == {transfer_id: 1} + + def test_scheduled_prefix_hit_publishes_adopted_copies(): """Copies adopted after scheduling must reach the worker's block table.""" from tests.v1.core.test_prefix_caching import ( diff --git a/tests/v1/worker/test_utils.py b/tests/v1/worker/test_utils.py index 43ac710f35..abcbdb24f2 100644 --- a/tests/v1/worker/test_utils.py +++ b/tests/v1/worker/test_utils.py @@ -44,6 +44,8 @@ def _make_hisparse_worker() -> HiSparseConnectorWorker: worker._row_mirror_num_rows = 0 worker._per_layer_mirrored = set() worker._submitted_mirror_layers = set() + worker._draft_layers = () + worker._draft_mirror_pending = False worker._pending_dma_descriptors = deque() worker._dma_free_descriptors = [] worker.host_write_events = (MagicMock(), MagicMock()) @@ -785,30 +787,6 @@ def test_hisparse_finish_forward_does_not_repeat_per_layer_mirrors(): worker._enqueue_row_dma.assert_not_called() -def test_hisparse_finish_forward_submits_lazy_post_forward_transfer(monkeypatch): - runtime = SimpleNamespace(eager_host_mirror=False) - worker = _make_hisparse_worker() - worker.is_host_writer = True - worker.cache_handles = [SimpleNamespace(runtime=runtime, num_actual_tokens=0)] - transfer = SparseKVPageTransfer(7, 2, (1,), after_forward=True) - worker._post_forward_transfers = [transfer] - worker._forward_ready_event = MagicMock() - worker._enqueue_host_mirror = MagicMock() - worker._submit_transfers = MagicMock() - worker._dma_submitted = False - worker._submitted_mirror_layers = set() - worker._finish_mirror_phase = MagicMock() - worker._release_completed_dma_descriptors = MagicMock() - stream = MagicMock() - monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream) - - worker.finish_forward() - - worker._finish_mirror_phase.assert_called_once_with(worker._forward_ready_event) - worker._submit_transfers.assert_called_once_with([transfer]) - worker.host_write_event.record.assert_called_once_with(stream) - - def test_hisparse_prefill_mirrors_source_groups_and_flushes_partial_group(): slots = torch.tensor([7, 8], dtype=torch.int64) source_indices = [0, 0, 1, 1, 1, 1, 2] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py index b223616401..bf6329b3bf 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/hisparse/worker.py @@ -301,6 +301,10 @@ class HiSparseConnectorWorker: self._submitted_mirror_layers: set[int] = set() self._layer_ready_events = tuple(torch.Event() for _ in cache_handles) self._forward_ready_event = torch.Event() + self._draft_layers = tuple( + index for index, cache in enumerate(cache_handles) if cache.draft_layer + ) + self._draft_mirror_pending = False self._set_row_mirrors(()) self.cache_layer_names = cache_layer_names self._group_leaders = tuple( @@ -344,9 +348,11 @@ class HiSparseConnectorWorker: self._init_dma() if self.is_host_writer: for layer_index, handle in enumerate(cache_handles): - handle.submit_layer_mirror = partial( - self._enqueue_layer_mirror, layer_index - ) + # Draft layers are mirrored once in the next start_step instead. + if not handle.draft_layer: + handle.submit_layer_mirror = partial( + self._enqueue_layer_mirror, layer_index + ) self._initialized = True def set_request_state_indices(self, indices: torch.Tensor) -> None: @@ -397,16 +403,17 @@ class HiSparseConnectorWorker: num_tokens: int = 0, ) -> None: self._stage_row_mirror_mapping(num_tokens) + self._finish_previous_step() previous_host_write_event = self.host_write_event self.host_write_event = self.host_write_events[self._next_host_write_event] self._next_host_write_event ^= 1 current_stream().wait_event(previous_host_write_event) self._release_completed_dma_descriptors() + self._dma_submitted = False mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids) if self._slot_mapping_staging is not None: self._slot_mapping_staging.candidates = mirrors self._set_row_mirrors(mirrors) - self._dma_submitted = False self._clear_forward_mirror_state() for handle in self.cache_handles: handle.all_context_pages_resident = metadata.all_context_pages_resident @@ -849,7 +856,12 @@ class HiSparseConnectorWorker: if num_rows == 0: return if self.is_host_writer: - expected_layers = {index for index, _ in active} + # The drafter has not written its layers' rows yet; the next + # start_step mirrors them after it runs. + self._draft_mirror_pending = bool(self._draft_layers) + expected_layers = {index for index, _ in active}.difference( + self._draft_layers + ) if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers: raise RuntimeError( "HiSparse per-layer DMA did not mirror every active layer: " @@ -884,9 +896,6 @@ class HiSparseConnectorWorker: compute_stream = current_stream() self._forward_ready_event.record() self._finish_mirror_phase(self._forward_ready_event) - transfers = self._post_forward_transfers - self._post_forward_transfers = [] - self._submit_transfers(transfers) if self.is_host_writer: if self._dma_submitted: compute_stream.wait_event(self.host_write_event) @@ -895,6 +904,24 @@ class HiSparseConnectorWorker: self.host_write_event.record(compute_stream) self._release_completed_dma_descriptors() + def _finish_previous_step(self) -> None: + """Mirror the previous step's draft-layer rows, then hand its pages over. + + Runs before this step's work, so after the previous step's drafter. + Post-forward transfers are ordered behind the draft mirror, so a page is + never reported clean, and its resident block never released, before + its draft-layer rows reach the host. Both land in the previous step's + host write event, which this step waits on before its forward and its + host copies, since the scheduler may already have reused the + finished requests' blocks they read and write. + """ + if self._draft_mirror_pending: + self._draft_mirror_pending = False + self._enqueue_row_dma(self._draft_layers) + transfers = self._post_forward_transfers + self._post_forward_transfers = [] + self._submit_transfers(transfers) + def take_completed_host_copies(self) -> list[int]: """Drain host copies this worker has enqueued for this step.""" completed = self._completed_host_copy_dst_ids diff --git a/vllm/model_executor/layers/attention_layer_base.py b/vllm/model_executor/layers/attention_layer_base.py index cee57fdd04..d5332a47a8 100644 --- a/vllm/model_executor/layers/attention_layer_base.py +++ b/vllm/model_executor/layers/attention_layer_base.py @@ -21,6 +21,8 @@ class AttentionLayerBase(ABC): impl: "AttentionImpl" supports_dcp: bool = True + # Set by the speculator on its draft model's layers. + is_draft_layer: bool = False def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: """Bind the allocated KV cache tensor to this layer. diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index 21d578efd0..5a18332e41 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -39,8 +39,6 @@ class SparseMLAIndexGroup: logical_topk_indices: torch.Tensor physical_topk_indices: torch.Tensor valid_topk_counts: torch.Tensor - row_indices: torch.Tensor - request_ids: torch.Tensor side_stream: torch.Stream logical_topk_ready: torch.Event physical_topk_ready: torch.Event @@ -252,19 +250,13 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup): ) source_block_table = cache.source_block_table assert source_block_table is not None - # CUDA-graph padding rows past the batch's tokens map to request 0; - # mark them -1 so residency resolution skips them. - request_ids = self.request_ids[:num_tokens] - request_ids.copy_(req_id_per_token) - request_ids.masked_fill_( - self.row_indices[:num_tokens] >= attn_metadata.query_start_loc[-1], -1 - ) return cache.swap_in( - request_ids, + req_id_per_token, block_table=source_block_table, logical_topk_indices=logical_topk_indices, block_size=attn_metadata.block_size, return_valid_counts=return_valid_counts, + num_valid_rows=attn_metadata.query_start_loc[-1:], ) def stage_prefill_rows( @@ -384,16 +376,6 @@ class SparseMLAIndexGroupBuilder: dtype=torch.int32, device=self.logical_topk_indices.device, ), - row_indices=torch.arange( - workspace_rows, - dtype=torch.int32, - device=self.logical_topk_indices.device, - ), - request_ids=torch.empty( - workspace_rows, - dtype=torch.int32, - device=self.logical_topk_indices.device, - ), side_stream=_create_side_stream(self.logical_topk_indices.device), logical_topk_ready=_create_event(), physical_topk_ready=_create_event(), diff --git a/vllm/v1/hisparse/binding.py b/vllm/v1/hisparse/binding.py index 88c8976b56..d021d8b7be 100644 --- a/vllm/v1/hisparse/binding.py +++ b/vllm/v1/hisparse/binding.py @@ -284,6 +284,7 @@ def bind_hisparse_kv_caches( assert source_cache.untyped_storage().data_ptr() == ( host_pool.registered.untyped_storage().data_ptr() ) + cache_handle.draft_layer = forward_context[layer_name].is_draft_layer cache_handle.runtime.shared_host_region = host_pool.shared_region cache_handle.runtime.bind_source_cache( source_cache, diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index 2d9c6cc83f..fc415fa2ec 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -662,6 +662,10 @@ class HiSparseIndexGroup: self.lru_init = torch.arange(region_stride, dtype=torch.int16, device=device) self.lru_slots = self.lru_init.repeat(max_num_reqs, 1).contiguous() self.shared_topk = _create_shared_topk_state(device, max_swap_rows, top_k) + # Request ids the resolver reads, built on the copy stream: a + # compute-stream write would race with its logical_topk_ready wait. + self.row_indices = torch.arange(max_swap_rows, dtype=torch.int32, device=device) + self.request_ids = torch.empty_like(self.row_indices) self.copy_stream = ( copy_stream if copy_stream is not None else _create_copy_stream(device) ) @@ -997,6 +1001,7 @@ class HiSparseRuntime: shared_rows: slice, attention_indices_out: torch.Tensor | None, valid_counts_out: torch.Tensor | None, + num_valid_rows: torch.Tensor, ) -> None: group = self.index_group compute_stream = current_stream() @@ -1005,9 +1010,17 @@ class HiSparseRuntime: else: group.copy_stream.wait_stream(compute_stream) with group.copy_stream: + # CUDA-graph padding rows past the batch's tokens map to request 0; + # mark them -1 so residency resolution skips them. + num_tokens = logical_topk_indices.shape[0] + request_ids = group.request_ids[:num_tokens] + request_ids.copy_(req_id_per_token) + request_ids.masked_fill_( + group.row_indices[:num_tokens] >= num_valid_rows, -1 + ) self._resolve_residency( resident=resident, - req_id_per_token=req_id_per_token, + req_id_per_token=request_ids, block_table=block_table, topk_indices=logical_topk_indices, block_size=block_size, @@ -1051,6 +1064,7 @@ class HiSparseRuntime: block_table: torch.Tensor, logical_topk_indices: torch.Tensor, block_size: int, + num_valid_rows: torch.Tensor, return_valid_counts: bool = False, attention_indices_out: torch.Tensor | None = None, valid_counts_out: torch.Tensor | None = None, @@ -1074,6 +1088,7 @@ class HiSparseRuntime: shared_rows=shared_rows, attention_indices_out=attention_indices_out, valid_counts_out=valid_counts_out, + num_valid_rows=num_valid_rows, ) if not self._swap_staged: @@ -1110,6 +1125,8 @@ class HiSparseCacheHandle: self.mirror_staging_cache: torch.Tensor | None = None self.mirror_staging_slots: torch.Tensor | None = None self.submit_layer_mirror: Callable[[], None] | None = None + # Speculator layers write their rows after the target forward. + self.draft_layer = False self.index_group_caches: list[HiSparseCacheHandle] = [self] def prepare_group_for_batch(self, attn_metadata: Any | None) -> None: @@ -1203,6 +1220,7 @@ class HiSparseCacheHandle: logical_topk_indices: torch.Tensor, *, block_size: int, + num_valid_rows: torch.Tensor, return_valid_counts: bool = False, attention_indices_out: torch.Tensor | None = None, valid_counts_out: torch.Tensor | None = None, @@ -1216,6 +1234,7 @@ class HiSparseCacheHandle: return_valid_counts=return_valid_counts, attention_indices_out=attention_indices_out, valid_counts_out=valid_counts_out, + num_valid_rows=num_valid_rows, ) diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 20a3b29eab..0728ddb87c 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -224,13 +224,13 @@ class DraftModelSpeculator(BaseSpeculator): self.model = self.load_draft_model(target_model, target_attn_layer_names) self._validate_local_argmax_reduction() - all_attn_layers = set[str]( - get_layers_from_vllm_config( - self.vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - ).keys() + all_attn_layers = get_layers_from_vllm_config( + self.vllm_config, + AttentionLayerBase, # type: ignore[type-abstract] ) - self.draft_attn_layer_names = all_attn_layers - target_attn_layer_names + self.draft_attn_layer_names = set(all_attn_layers) - target_attn_layer_names + for layer_name in self.draft_attn_layer_names: + all_attn_layers[layer_name].is_draft_layer = True target_supports_mm = self.vllm_config.model_config.supports_multimodal_inputs draft_supports_mm = supports_multimodal_embeddings(self.model) From 01549796563f2e8bd0d6b08243cfb954a7b42ec8 Mon Sep 17 00:00:00 2001 From: kliuae <17350011+kliuae@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:58:04 +0800 Subject: [PATCH 245/248] [ROCm][Perf] Kimi-K3 enable prefill checkpoints on ROCm (#58344) Signed-off-by: kliuae --- .../models/kimi_k3/test_amd_kda_checkpoint.py | 521 ++++++++++++++++++ tests/models/kimi_k3/test_amd_kda_chunk.py | 62 ++- vllm/models/kimi_k3/amd/kda.py | 62 ++- vllm/models/kimi_k3/amd/kda_metadata.py | 56 ++ vllm/models/kimi_k3/amd/ops/kda_checkpoint.py | 41 ++ vllm/models/kimi_k3/amd/ops/kda_chunk.py | 12 +- vllm/models/kimi_k3/amd/ops/kda_prefill.py | 8 +- 7 files changed, 748 insertions(+), 14 deletions(-) create mode 100644 tests/models/kimi_k3/test_amd_kda_checkpoint.py create mode 100644 vllm/models/kimi_k3/amd/ops/kda_checkpoint.py diff --git a/tests/models/kimi_k3/test_amd_kda_checkpoint.py b/tests/models/kimi_k3/test_amd_kda_checkpoint.py new file mode 100644 index 0000000000..0a622bfa04 --- /dev/null +++ b/tests/models/kimi_k3/test_amd_kda_checkpoint.py @@ -0,0 +1,521 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The ROCm KDA prefill checkpoint: where it is taken and what it stores. + +Two things have to line up for a checkpoint to be safe: the metadata must name +a position the chunk kernel will actually write, and the conv window saved +there must be the one a resumed prefill would have left. +""" + +import pytest +import torch + +from tests.v1.attention.utils import ( + BatchSpec, + create_common_attn_metadata, + create_vllm_config, +) +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointExporter, + MambaPrefillCheckpointMetadata, +) +from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention +from vllm.model_executor.layers.mamba.ops.causal_conv1d import causal_conv1d_fn +from vllm.models.kimi_k3.amd.kda import KimiK3DeltaAttention +from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDAMetadataBuilder +from vllm.models.kimi_k3.amd.ops.kda_checkpoint import ( + KimiK3ROCmKDAPrefillCheckpointExporter, +) +from vllm.models.kimi_k3.amd.ops.kda_chunk import ( + KDA_CHECKPOINT_ALIGNMENT, + is_fused_kda_chunk_supported, +) +from vllm.platforms import current_platform +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID +from vllm.v1.kv_cache_interface import MambaSpec + +pytestmark = pytest.mark.skipif( + not ( + current_platform.is_rocm() + and torch.cuda.is_available() + and is_fused_kda_chunk_supported() + ), + reason="The ROCm KDA prefill checkpoint is exported by the fused chunk " + "kernel, which needs a gfx950 build that includes it", +) + +# A mamba block size the chunk walk lands on. The real K3 value is 768; any +# multiple of the kernel's 64-token chunk behaves the same and keeps the +# fixtures small. +BLOCK_SIZE = 64 + + +class _StubSpeculativeConfig: + method = "dspark" + parallel_drafting = False + + def __init__(self, num_speculative_tokens: int) -> None: + self.num_speculative_tokens = num_speculative_tokens + + def use_eagle_block_drop(self) -> bool: + return False + + +def _builder( + num_prefill_checkpoint_blocks: int = 1, + mamba_cache_mode: str = "align", + block_size: int = BLOCK_SIZE, + prefix_match_unit: int | None = None, + num_spec: int = 0, +) -> KimiK3ROCmKDAMetadataBuilder: + vllm_config = create_vllm_config( + model_name="Qwen/Qwen3.5-0.8B", block_size=block_size + ) + vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode + vllm_config.cache_config.prefix_match_unit = prefix_match_unit + if num_spec: + vllm_config.speculative_config = _StubSpeculativeConfig(num_spec) + return KimiK3ROCmKDAMetadataBuilder( + kv_cache_spec=MambaSpec( + block_size=block_size, + shapes=((16, 64),), + dtypes=(torch.float16,), + mamba_cache_mode=mamba_cache_mode, + num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks, + prefill_checkpoint_alignment=( + KDA_CHECKPOINT_ALIGNMENT if num_prefill_checkpoint_blocks else None + ), + num_speculative_blocks=num_spec, + ), + layer_names=["layer.0"], + vllm_config=vllm_config, + device=torch.device("cuda"), + ) + + +def _build(batch: BatchSpec, block_size: int = BLOCK_SIZE, **kw): + common = create_common_attn_metadata( + batch, block_size, torch.device("cuda"), arange_block_indices=True + ) + return _builder(block_size=block_size, **kw).build(0, common) + + +def _i32(values: list[int]) -> torch.Tensor: + return torch.tensor(values, dtype=torch.int32, device="cuda") + + +def test_checkpoint_metadata_targets_last_aligned_boundary() -> None: + # Slot p holds the state after (p + 1) * block_size tokens, so a 200-token + # prompt checkpoints at 192 into slot 2 and a 128-token one at 64 into slot + # 0. The block table is an arange, so request 1 owns blocks 4..7. + md = _build(BatchSpec(seq_lens=[200, 128], query_lens=[200, 128])) + + assert md.checkpoint is not None + torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 64])) + torch.testing.assert_close(md.checkpoint.state_indices, _i32([2, 4])) + + +def test_checkpoint_offset_is_relative_to_query_start() -> None: + # 256 tokens computed, 144 more this step: the boundary at 384 lands 128 + # tokens into the chunk, and cdiv(400, 64) - 2 = 5 is its slot. + md = _build(BatchSpec(seq_lens=[400, 64], query_lens=[144, 64])) + + assert md.checkpoint is not None + torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([128, 0])) + torch.testing.assert_close(md.checkpoint.state_indices, _i32([5, NULL_BLOCK_ID])) + + +def test_checkpoint_metadata_avoids_live_state_row() -> None: + # Writing the row the walk itself owns would race it. + md = _build(BatchSpec(seq_lens=[200, 130], query_lens=[200, 130])) + + assert md.checkpoint is not None + # `mamba_get_block_table_tensor` hands the walk block (seq_len - 1) // bs. + live = torch.tensor([(200 - 1) // BLOCK_SIZE, (130 - 1) // BLOCK_SIZE]) + table = torch.arange(2 * 4).view(2, 4) + live_rows = table[torch.arange(2), live].cuda() + assert not bool((md.checkpoint.state_indices == live_rows).any()) + + +def test_checkpoint_metadata_skips_chunk_without_boundary() -> None: + # 128 computed, 40 more: the only boundary at 128 is the state this chunk + # started from, so there is nothing new to snapshot. + assert _build(BatchSpec(seq_lens=[168], query_lens=[40])).checkpoint is None + + +def test_checkpoint_metadata_skips_null_reservation() -> None: + # Only the chunk that finishes a prompt gets a block reserved; every + # earlier one leaves the shared null block, which nothing hashes and + # writing it would corrupt every other request's scratch row. + batch = BatchSpec(seq_lens=[200], query_lens=[200]) + common = create_common_attn_metadata( + batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True + ) + # cdiv(200, 64) - 2 = 2 is the checkpoint slot. + common.block_table_tensor[0, 2] = NULL_BLOCK_ID + + md = _builder().build(0, common) + + assert md.checkpoint is not None + torch.testing.assert_close(md.checkpoint.state_indices, _i32([NULL_BLOCK_ID])) + + +def test_checkpoint_metadata_absent_without_reserved_block() -> None: + md = _build( + BatchSpec(seq_lens=[200], query_lens=[200]), + num_prefill_checkpoint_blocks=0, + ) + assert getattr(md, "checkpoint", None) is None + + +def test_spec_checkpoint_metadata_gathers_non_spec_rows_by_index() -> None: + # gdn_attn selects the non-spec rows by mask and reclassifies their + # one-token decodes as prefills, so the chunk kernel is handed rows 1, 3 + # and 4 here while a num_decodes: slice would hand it rows 0, 1 and 2. + num_spec = 2 + batch = BatchSpec( + seq_lens=[300, 200, 400, 512, 128], + query_lens=[num_spec + 1, 200, num_spec + 1, 1, 128], + ) + common = create_common_attn_metadata( + batch, BLOCK_SIZE, torch.device("cuda"), arange_block_indices=True + ) + # Align mode reads 1 + num_speculative_blocks columns, so widen the table. + rows, cols = common.block_table_tensor.shape + width = cols + num_spec + common.block_table_tensor = torch.arange( + rows * width, dtype=torch.int32, device="cuda" + ).view(rows, width) + + md = _builder(num_spec=num_spec).build( + 0, + common, + num_accepted_tokens=torch.ones(rows, dtype=torch.int32, device="cuda"), + num_decode_draft_tokens_cpu=torch.tensor( + [num_spec, -1, num_spec, -1, -1], dtype=torch.int32 + ), + ) + + assert md.spec_sequence_masks is not None + assert md.num_decodes == 0 and md.num_prefills == 3 + assert md.checkpoint is not None + # Row 3 resumes mid-block, so its chunk holds no reusable boundary. + torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192, 0, 64])) + # Request r owns blocks r*width.., so slot cdiv(200, 64) - 2 = 2 for + # request 1 and cdiv(128, 64) - 2 = 0 for request 4. + torch.testing.assert_close( + md.checkpoint.state_indices, _i32([width + 2, NULL_BLOCK_ID, 4 * width]) + ) + + +def test_checkpoint_metadata_absent_outside_align_mode() -> None: + md = _build( + BatchSpec(seq_lens=[200], query_lens=[200]), + mamba_cache_mode="none", + ) + assert getattr(md, "checkpoint", None) is None + + +# --------------------------------------------------------------------------- +# Conv half +# --------------------------------------------------------------------------- + +DIM = 384 +WIDTH = 4 +STATE_LEN = WIDTH - 1 + + +def store_conv_checkpoints(x, conv_state, cu, offsets, rows, state_len) -> None: + KimiK3ROCmKDAPrefillCheckpointExporter(state_len=state_len).export( + MambaPrefillCheckpointMetadata(offsets, rows), + raw_qkv=x, + conv_state=conv_state, + cu_seqlens=cu, + ) + + +def _conv_fixture( + seqlens: list[int], slots: int = 16, seed: int = 0, num_spec: int = 0 +): + # kda_state_shape widens the conv row to conv_kernel_size - 1 + num_spec, + # but the window every reader starts from stays the leading STATE_LEN. + torch.manual_seed(seed) + total = sum(seqlens) + x = torch.randn(total, DIM, device="cuda", dtype=torch.bfloat16) + cu = torch.tensor( + [0] + torch.tensor(seqlens).cumsum(0).tolist(), + device="cuda", + dtype=torch.int32, + ) + conv_state = torch.full( + (slots, DIM, STATE_LEN + num_spec), + float("nan"), + device="cuda", + dtype=torch.bfloat16, + ) + return x, cu, conv_state + + +def test_conv_checkpoint_saves_window_before_offset() -> None: + seqlens = [200, 130] + x, cu, conv_state = _conv_fixture(seqlens) + offsets = _i32([192, 128]) + rows = _i32([5, 9]) + + store_conv_checkpoints(x, conv_state, cu, offsets, rows, STATE_LEN) + + for seq, (offset, row) in enumerate(zip([192, 128], [5, 9])): + end = int(cu[seq]) + offset + expected = x[end - STATE_LEN : end].transpose(0, 1) + torch.testing.assert_close(conv_state[row], expected) + + +def test_conv_checkpoint_skips_opted_out_sequences() -> None: + x, cu, conv_state = _conv_fixture([200, 200]) + before = conv_state.clone() + + store_conv_checkpoints( + x, conv_state, cu, _i32([0, 192]), _i32([5, NULL_BLOCK_ID]), STATE_LEN + ) + + assert torch.equal(conv_state.isnan(), before.isnan()) + + +def test_exporter_stores_conv_window_through_shared_interface() -> None: + x, cu, conv_state = _conv_fixture([200, 200]) + exporter = KimiK3ROCmKDAPrefillCheckpointExporter(state_len=STATE_LEN) + assert isinstance(exporter, MambaPrefillCheckpointExporter) + checkpoint = MambaPrefillCheckpointMetadata( + _i32([192, 0]), _i32([5, NULL_BLOCK_ID]) + ) + + exporter.export(checkpoint, raw_qkv=x, conv_state=conv_state, cu_seqlens=cu) + + torch.testing.assert_close(conv_state[5], x[192 - STATE_LEN : 192].transpose(0, 1)) + untouched = torch.ones(16, dtype=torch.bool, device="cuda") + untouched[5] = False + assert bool(conv_state[untouched].isnan().all()) + + +@pytest.mark.parametrize("num_spec", [0, 2, 8]) +def test_conv_checkpoint_matches_prefill_truncated_at_offset(num_spec: int) -> None: + # causal_conv1d_fn's own state write is the reference: a prefill that + # stopped at the offset leaves exactly the row a later prefix-cache hit + # reads back. Speculative decoding widens the row without moving that + # window, so a checkpoint sized off the row would sit num_spec tokens in + # the past. + seqlens = [200, 130] + offsets = [192, 64] + x, cu, conv_state = _conv_fixture(seqlens, seed=3, num_spec=num_spec) + weight = torch.randn(DIM, WIDTH, device="cuda", dtype=torch.bfloat16) * 0.3 + bias = torch.randn(DIM, device="cuda", dtype=torch.bfloat16) + rows = [5, 9] + + store_conv_checkpoints(x, conv_state, cu, _i32(offsets), _i32(rows), STATE_LEN) + + # A prefill of just the prefix, into its own cache. Row 0 is the null block + # the conv kernel skips by design, so the reference rows start at 1. + prefix = torch.cat( + [x[int(cu[n]) : int(cu[n]) + off] for n, off in enumerate(offsets)] + ) + prefix_cu = torch.tensor( + [0] + torch.tensor(offsets).cumsum(0).tolist(), + device="cuda", + dtype=torch.int32, + ) + reference = torch.zeros_like(conv_state) + causal_conv1d_fn( + prefix.transpose(0, 1), + weight, + bias, + activation="silu", + conv_states=reference, + has_initial_state=torch.zeros(2, dtype=torch.bool, device="cuda"), + cache_indices=_i32([1, 2]), + query_start_loc=prefix_cu, + ) + + # causal_conv1d_fn writes only the leading STATE_LEN columns. + window = slice(None), slice(None, STATE_LEN) + torch.testing.assert_close(conv_state[rows[0]][window], reference[1][window]) + torch.testing.assert_close(conv_state[rows[1]][window], reference[2][window]) + + +def test_conv_checkpoint_leaves_spec_tail_of_row_alone() -> None: + # The draft-token columns belong to causal_conv1d_update. Filling the whole + # row would hand every reader, which indexes from column 0, a window + # shifted num_spec tokens into the past. + num_spec = 4 + x, cu, conv_state = _conv_fixture([200], num_spec=num_spec, seed=11) + + store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([5]), STATE_LEN) + + torch.testing.assert_close( + conv_state[5, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1) + ) + assert bool(conv_state[5, :, STATE_LEN:].isnan().all()), ( + "the checkpoint wrote into the speculative-decode tail of the row" + ) + + +def test_conv_checkpoint_honours_transposed_cache_view() -> None: + # The SD conv layout reaches the layer transposed, not contiguous. + x, cu, _ = _conv_fixture([200]) + raw = torch.full( + (16, STATE_LEN, DIM), float("nan"), device="cuda", dtype=torch.bfloat16 + ) + conv_state = raw.transpose(-1, -2) + assert not conv_state.is_contiguous() + + store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([7]), STATE_LEN) + + torch.testing.assert_close(conv_state[7], x[192 - STATE_LEN : 192].transpose(0, 1)) + untouched = torch.ones(16, dtype=torch.bool, device="cuda") + untouched[7] = False + assert bool(raw[untouched].isnan().all()) + + +def test_checkpoint_metadata_indexes_prefill_rows_after_decodes() -> None: + # The chunk kernel receives the prefill tail alone, so the metadata has to + # be indexed the same way or it snapshots the wrong request's boundary. + md = _build(BatchSpec(seq_lens=[64, 200], query_lens=[1, 200])) + + assert md.num_decodes == 1 and md.num_prefills == 1 + assert md.checkpoint is not None + torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([192])) + # Request 1 owns blocks 4..7, so its slot-2 state block is 6. + torch.testing.assert_close(md.checkpoint.state_indices, _i32([6])) + + +def test_conv_checkpoint_addresses_row_past_int32_limit() -> None: + # state_stride_0 is the whole mamba page, padded up to the attention page, + # so row * stride passes 2**31 a few thousand blocks into a large cache and + # an int32 wrap stores outside the tensor. The margin on the plain prefill + # path is 1.6x, which is luck: a bigger cache, a higher utilization or the + # wider row speculative decoding allocates all close it. + stride_row = DIM * (STATE_LEN + 8) + row = 2**31 // stride_row + 1 + need = (row + 1) * stride_row * 2 + free, _ = torch.accelerator.get_memory_info() + if free < need + (4 << 30): + pytest.skip(f"needs {need >> 30} GiB of free VRAM to reach the wrap") + + x, cu, _ = _conv_fixture([200]) + conv_state = torch.zeros( + row + 1, DIM, STATE_LEN + 8, device="cuda", dtype=torch.bfloat16 + ) + assert conv_state.stride(0) * row > 2**31, "fixture does not reach the wrap" + + store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([row]), STATE_LEN) + + torch.testing.assert_close( + conv_state[row, :, :STATE_LEN], x[192 - STATE_LEN : 192].transpose(0, 1) + ) + assert not bool(conv_state[:row].any()), "the store wrote outside its row" + + +def test_conv_checkpoint_reads_row_strided_projection() -> None: + # mixed_qkv is a band of the fused QKVGFAB projection, so its row stride is + # the whole projection width, not DIM. + torch.manual_seed(5) + total, pad = 200, 177 + projection = torch.randn(total, DIM + pad, device="cuda", dtype=torch.bfloat16) + x = projection[:, :DIM] + assert x.stride(0) == DIM + pad + + cu = _i32([0, total]) + conv_state = torch.full( + (16, DIM, STATE_LEN), float("nan"), device="cuda", dtype=torch.bfloat16 + ) + store_conv_checkpoints(x, conv_state, cu, _i32([192]), _i32([4]), STATE_LEN) + + torch.testing.assert_close(conv_state[4], x[192 - STATE_LEN : 192].transpose(0, 1)) + + +def test_finer_prefix_match_unit_moves_checkpoint_to_hash_grid() -> None: + # A partial prefix hit matches at prefix_match_unit, so a state captured + # only at block boundaries is too coarse to resume from. Block 256 with + # unit 128 moves a 700-token prompt's checkpoint from 512 to 640. + md = _build( + BatchSpec(seq_lens=[700], query_lens=[700]), + block_size=256, + prefix_match_unit=128, + ) + + assert md.checkpoint is not None + torch.testing.assert_close(md.checkpoint.checkpoint_offsets, _i32([640])) + # Slot is still block-indexed: cdiv(700, 256) - 2 = 1, request 0 owns 0..2. + torch.testing.assert_close(md.checkpoint.state_indices, _i32([1])) + + +def test_checkpoint_metadata_skips_unaligned_offset() -> None: + # The scheduler declines a boundary off the published alignment and + # reserves nothing, so the builder has to reach the same verdict or it + # writes a block nobody hashed. + # Block 192, unit 96: a 700-token prompt's last hash boundary is 672, which + # is 32 tokens off the chunk grid. + md = _build( + BatchSpec(seq_lens=[700], query_lens=[700]), + block_size=192, + prefix_match_unit=96, + ) + + assert md.checkpoint is None + + +# --------------------------------------------------------------------------- +# Opting in +# --------------------------------------------------------------------------- + + +def _spec_for(monkeypatch, **attrs) -> MambaSpec: + """Run the layer's `get_kv_cache_spec` over a stubbed base spec.""" + base = MambaSpec(block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)) + monkeypatch.setattr( + GatedDeltaNetAttention, "get_kv_cache_spec", lambda self, cfg: base + ) + layer = KimiK3DeltaAttention.__new__(KimiK3DeltaAttention) + for name, value in dict( + use_prefill_checkpoint=True, + use_fused_chunk=True, + num_spec=0, + use_safe_gate=True, + ).items(): + setattr(layer, name, attrs.get(name, value)) + vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B") + vllm_config.cache_config.mamba_cache_mode = "align" + return layer.get_kv_cache_spec(vllm_config) + + +def test_opt_in_publishes_chunk_walk_alignment(monkeypatch) -> None: + # The scheduler learns the kernel's granularity only from the spec, so + # reserving a block without publishing the alignment would hash a state + # that is never written. + spec = _spec_for(monkeypatch) + + assert spec.num_prefill_checkpoint_blocks == 1 + assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT + + +@pytest.mark.parametrize( + "attr,value", + [ + ("use_prefill_checkpoint", False), + ("use_fused_chunk", False), + ("use_safe_gate", False), + ], +) +def test_paths_that_cannot_export_stay_opted_out(monkeypatch, attr, value) -> None: + spec = _spec_for(monkeypatch, **{attr: value}) + + assert spec.num_prefill_checkpoint_blocks == 0 + assert spec.prefill_checkpoint_alignment is None + + +def test_speculative_decoding_does_not_opt_out(monkeypatch) -> None: + # num_spec reaches the kernels as a wider conv row and an interleaved + # batch, both of which the layer handles. + spec = _spec_for(monkeypatch, num_spec=8) + + assert spec.num_prefill_checkpoint_blocks == 1 + assert spec.prefill_checkpoint_alignment == KDA_CHECKPOINT_ALIGNMENT diff --git a/tests/models/kimi_k3/test_amd_kda_chunk.py b/tests/models/kimi_k3/test_amd_kda_chunk.py index 3e99018edc..c53e9bacdb 100644 --- a/tests/models/kimi_k3/test_amd_kda_chunk.py +++ b/tests/models/kimi_k3/test_amd_kda_chunk.py @@ -13,17 +13,17 @@ import torch from vllm.platforms import current_platform -def _on_gfx950() -> bool: - if not current_platform.is_rocm(): +def _fused_chunk_available() -> bool: + if not (current_platform.is_rocm() and torch.cuda.is_available()): return False - from vllm.platforms.rocm import on_gfx950 + from vllm.models.kimi_k3.amd.ops.kda_chunk import is_fused_kda_chunk_supported - return on_gfx950() + return is_fused_kda_chunk_supported() pytestmark = pytest.mark.skipif( - not _on_gfx950(), - reason="The fused KDA chunk kernel is only built for gfx950", + not _fused_chunk_available(), + reason="The fused KDA chunk kernel needs a gfx950 build that includes it", ) HEAD_DIM = 128 @@ -701,6 +701,56 @@ def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None: torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3) +@pytest.mark.parametrize("use_fused", [False, True]) +def test_state_indices_accept_a_block_table_column(use_fused: bool) -> None: + # The layer passes block_table[:, 0], which is contiguous only while the + # table has one column. Speculative decoding reserves a state page per + # draft token, so the table widens and that column becomes strided. + _requires_kernel() + seqlens = [512, 320, 64] + inp = _inputs(seqlens, seed=43) + n = len(seqlens) + rows = [3 * i + 5 for i in range(n)] + cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12) + ref_cache = cache.clone() + + table = torch.zeros(n, 9, dtype=torch.int32, device="cuda") + table[:, 0] = idx + column = table[:, 0] + assert not column.is_contiguous() + + o_ref, _ = _run_paged(inp, ref_cache, idx, warm, use_fused=use_fused) + o_got, _ = _run_paged(inp, cache, column, warm, use_fused=use_fused) + + torch.testing.assert_close(o_got, o_ref) + torch.testing.assert_close(cache, ref_cache) + + +@pytest.mark.parametrize("use_fused", [False, True]) +def test_padded_zero_length_sequence_is_inert(use_fused: bool) -> None: + # A batch carrying speculative decodes pads its non-spec group with + # zero-length sequences whose cache row still belongs to a request, so + # consuming no tokens has to mean writing nothing and perturbing nobody. + _requires_kernel() + live = [200, 1, 128] + # _inputs draws h0 last, so the padded batch's live rows are the same draw + # as the unpadded one's and the two runs compare elementwise. + padded, unpadded = _inputs([*live, 0], seed=41), _inputs(live, seed=41) + rows = [3 * i + 5 for i in range(4)] + + cache, idx, warm = _paged_cache(padded, [True] * 4, rows, slots=24) + before = cache.clone() + _run_paged(padded, cache, idx, warm, use_fused=use_fused) + + ref_cache, ref_idx, ref_warm = _paged_cache(unpadded, [True] * 3, rows[:3], 24) + _run_paged(unpadded, ref_cache, ref_idx, ref_warm, use_fused=use_fused) + + assert torch.equal(cache[rows[3]], before[rows[3]]), ( + "the padded sequence's cache row was written" + ) + torch.testing.assert_close(cache[ref_idx.long()], ref_cache[ref_idx.long()]) + + @pytest.mark.parametrize("groups", [2, 4]) def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None: """At G>1 the read and the write are different blocks of one launch.""" diff --git a/vllm/models/kimi_k3/amd/kda.py b/vllm/models/kimi_k3/amd/kda.py index d98d16521f..e2dea74b6b 100644 --- a/vllm/models/kimi_k3/amd/kda.py +++ b/vllm/models/kimi_k3/amd/kda.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import replace + import torch from einops import rearrange from torch import nn @@ -38,7 +40,11 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader from vllm.model_executor.utils import set_weight_attrs from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend +from vllm.models.kimi_k3.amd.ops.kda_checkpoint import ( + KimiK3ROCmKDAPrefillCheckpointExporter, +) from vllm.models.kimi_k3.amd.ops.kda_chunk import ( + KDA_CHECKPOINT_ALIGNMENT, is_fused_kda_chunk_supported, ) from vllm.models.kimi_k3.amd.ops.kda_decode import ( @@ -55,6 +61,7 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig from vllm.v1.attention.backend import AttentionBackend from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata +from vllm.v1.kv_cache_interface import MambaSpec logger = init_logger(__name__) @@ -79,6 +86,23 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): num_spec=self.num_spec, ) + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec: + spec = super().get_kv_cache_spec(vllm_config) + assert isinstance(spec, MambaSpec) + # Only the fused path exports the checkpoint snapshots + can_checkpoint = ( + self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate + ) + if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align": + logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.") + return replace( + spec, + num_prefill_checkpoint_blocks=int(can_checkpoint), + prefill_checkpoint_alignment=( + KDA_CHECKPOINT_ALIGNMENT if can_checkpoint else None + ), + ) + def __init__( self, config: KimiLinearConfig, @@ -226,6 +250,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): "Kimi-K3 KDA prefill backend: %s", "fused" if self.use_fused_chunk else "triton", ) + # TODO: Remove the extra config arg once tested + self.use_prefill_checkpoint = ( + additional_config.get("kda_prefill_checkpoint", True) + if isinstance(additional_config, dict) + else True + ) + self._checkpoint_exporter = KimiK3ROCmKDAPrefillCheckpointExporter( + state_len=self.conv_size - 1 + ) self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid") decode_norm_weight = None @@ -511,10 +544,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): assert non_spec_state_indices_tensor is not None assert has_initial_state is not None - # A mixed non-spec batch is decode-first: the decodes are - # length-1 sequences, and the chunk kernel returns NaN for those. - # Send them to the recurrent kernel and give the chunk kernel the - # prefill tail only. + # Decode-first, so peel the length-1 decodes to the recurrent + # kernel. A spec batch interleaves its non-spec rows, so the + # peel cannot apply and the chunk kernel takes them. core_attn_out_decode = None split_non_spec = spec_sequence_masks is None and m.num_decodes > 0 # Without a spec split the non-spec tokens are exactly @@ -522,7 +554,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): # can write their slice straight into it. With one, the tokens # are interleaved and have to be scattered by index afterwards. direct = spec_sequence_masks is None - use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None if split_non_spec: assert non_spec_query_start_loc is not None nd_tok = m.num_decode_tokens @@ -559,6 +590,22 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): prefill_has_initial_state = has_initial_state nd_tok = 0 + checkpoint = getattr(m, "checkpoint", None) + checkpoint_state = None + checkpoint_offsets = None + checkpoint_state_indices = None + if checkpoint is not None: + assert prefill_query_start_loc is not None + checkpoint_state = recurrent_state + checkpoint_offsets = checkpoint.checkpoint_offsets + checkpoint_state_indices = checkpoint.state_indices + self._checkpoint_exporter.export( + checkpoint, + raw_qkv=mixed_qkv_ns[nd_tok:], + conv_state=conv_state, + cu_seqlens=prefill_query_start_loc, + ) + ( core_attn_out_non_spec, _, @@ -578,8 +625,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention): cu_seqlens=prefill_query_start_loc, chunk_indices=m.chunk_indices, chunk_offsets=m.chunk_offsets, - use_fused_chunk=use_fused_chunk, + use_fused_chunk=self.use_fused_chunk, out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None, + checkpoint_state=checkpoint_state, + checkpoint_offsets=checkpoint_offsets, + checkpoint_state_indices=checkpoint_state_indices, ) # chunk_kda_prefill updates `recurrent_state` in place, so # there is no final state to write back here. diff --git a/vllm/models/kimi_k3/amd/kda_metadata.py b/vllm/models/kimi_k3/amd/kda_metadata.py index 9681aca1d8..dcc03d2814 100644 --- a/vllm/models/kimi_k3/amd/kda_metadata.py +++ b/vllm/models/kimi_k3/amd/kda_metadata.py @@ -7,16 +7,23 @@ The request classification and cudagraph staging intentionally mirror differently on device rather than on the host. """ +from dataclasses import replace + import torch from vllm.logger import init_logger +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, +) from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE from vllm.triton_utils import tl, triton from vllm.utils.math_utils import next_power_of_2 from vllm.v1.attention.backends.gdn_attn import ( GDNAttentionBackend, + GDNAttentionMetadata, GDNAttentionMetadataBuilder, ) +from vllm.v1.attention.backends.utils import CommonAttentionMetadata logger = init_logger(__name__) @@ -86,6 +93,55 @@ def prepare_chunk_metadata_device( class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder): + def build( # type: ignore[override] + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + num_accepted_tokens: torch.Tensor | None = None, + num_decode_draft_tokens_cpu: torch.Tensor | None = None, + fast_build: bool = False, + ) -> GDNAttentionMetadata: + attn_metadata = super().build( + common_prefix_len, + common_attn_metadata, + num_accepted_tokens, + num_decode_draft_tokens_cpu, + fast_build, + ) + if attn_metadata.checkpoint is None: + return attn_metadata + # The shared builder keys checkpoint rows on every request; the ROCm + # prefill path indexes them by prefill_query_start_loc, so rebuild with + # those rows. + return replace( + attn_metadata, + checkpoint=self._build_checkpoint_metadata( + common_attn_metadata, attn_metadata, num_decode_draft_tokens_cpu + ), + ) + + def _build_checkpoint_metadata( + self, + m: CommonAttentionMetadata, + attn_metadata: GDNAttentionMetadata, + num_decode_draft_tokens_cpu: torch.Tensor | None, + ) -> MambaPrefillCheckpointMetadata | None: + if attn_metadata.num_prefills == 0: + return None + # request_rows must line up with prefill_query_start_loc, either the + # prefill tail of a decode-first batch, or every non-spec row + # including padding + num_decodes = attn_metadata.num_decodes + request_rows = list( + range(num_decodes, num_decodes + attn_metadata.num_prefills) + ) + if attn_metadata.spec_sequence_masks is not None: + assert num_decode_draft_tokens_cpu is not None + request_rows = ( + (num_decode_draft_tokens_cpu < 0).nonzero().flatten().tolist() + ) + return self.checkpoint_builder.build(m, request_rows) + def _build_chunk_metadata( self, prefill_query_start_loc: torch.Tensor, diff --git a/vllm/models/kimi_k3/amd/ops/kda_checkpoint.py b/vllm/models/kimi_k3/amd/ops/kda_checkpoint.py new file mode 100644 index 0000000000..5560adc626 --- /dev/null +++ b/vllm/models/kimi_k3/amd/ops/kda_checkpoint.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Conv half of the ROCm Kimi-K3 prefill checkpoint.""" + +from dataclasses import dataclass + +import torch + +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, +) +from vllm.model_executor.layers.mamba.kda_checkpoint import ( + FlashKDAPrefillCheckpointExporter, +) + + +@dataclass(frozen=True) +class KimiK3ROCmKDAPrefillCheckpointExporter(FlashKDAPrefillCheckpointExporter): + """Conv window only; `fused_kda_chunk` writes the recurrent half itself. + + `state_len` is `conv_kernel_size - 1`, not the spec-widened conv row. + """ + + def export( # type: ignore[override] + self, + checkpoint: MambaPrefillCheckpointMetadata, + *, + raw_qkv: torch.Tensor, + conv_state: torch.Tensor, + cu_seqlens: torch.Tensor, + ) -> None: + assert self.state_len is not None + # An empty recurrent checkpoint masks the kernel's recurrent half off. + super().export( + checkpoint, + raw_qkv=raw_qkv, + conv_state=conv_state, + recurrent_checkpoint=conv_state[:, :0], + recurrent_state=conv_state, + cu_seqlens=cu_seqlens, + ) diff --git a/vllm/models/kimi_k3/amd/ops/kda_chunk.py b/vllm/models/kimi_k3/amd/ops/kda_chunk.py index 556373c084..035a49f852 100644 --- a/vllm/models/kimi_k3/amd/ops/kda_chunk.py +++ b/vllm/models/kimi_k3/amd/ops/kda_chunk.py @@ -16,6 +16,7 @@ from vllm.third_party.flash_linear_attention.ops.index import ( prepare_chunk_indices, prepare_chunk_offsets, ) +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID CHUNK_SIZE = 64 HEAD_DIM = 128 @@ -28,6 +29,15 @@ _DEEP_LEN = 12 _MAX_GROUPS = 32 +def _hip_checkpoint_state_indices(state_indices: torch.Tensor) -> torch.Tensor: + """NULL_BLOCK_ID marks no checkpoint; the HIP kernel skips negative rows.""" + return ( + torch.where(state_indices != NULL_BLOCK_ID, state_indices, -1) + .to(torch.int32) + .contiguous() + ) + + @cache def is_fused_kda_chunk_supported() -> bool: from vllm.platforms.rocm import on_gfx950 @@ -295,7 +305,7 @@ def fused_kda_chunk( None if checkpoint_offsets is None else checkpoint_offsets.to(torch.int32), None if checkpoint_state_indices is None - else checkpoint_state_indices.to(torch.int32).contiguous(), + else _hip_checkpoint_state_indices(checkpoint_state_indices), state_cache, None if state_indices is None else state_indices.to(torch.int32).contiguous(), has_initial_state, diff --git a/vllm/models/kimi_k3/amd/ops/kda_prefill.py b/vllm/models/kimi_k3/amd/ops/kda_prefill.py index 56cb3e1482..5ba2fd6379 100644 --- a/vllm/models/kimi_k3/amd/ops/kda_prefill.py +++ b/vllm/models/kimi_k3/amd/ops/kda_prefill.py @@ -155,7 +155,13 @@ def chunk_kda_prefill( aqk=ws["aqk"], decay=ws["decay"], # v is dead by this point, so it doubles as the output buffer. - out=out if out is not None else v, + out=out + if out is not None + else ( + v + if v.is_contiguous() + else torch.empty_like(v, memory_format=torch.contiguous_format) + ), scale=scale, cu_seqlens=cu_seqlens, initial_state=initial_state, From b558f160a2c0abcb5902acc3c91a14c38a4af173 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Fri, 2 Oct 2026 15:29:49 +0800 Subject: [PATCH 246/248] [Bugfix] Fix minimax-m3 multimodal processor compatability with Transformers v5.18 (#59613) Signed-off-by: Isotr0py --- vllm/transformers_utils/processors/minimax_m3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm/transformers_utils/processors/minimax_m3.py b/vllm/transformers_utils/processors/minimax_m3.py index 237b1b5f1e..9d922f8e4c 100644 --- a/vllm/transformers_utils/processors/minimax_m3.py +++ b/vllm/transformers_utils/processors/minimax_m3.py @@ -375,7 +375,6 @@ class MiniMaxM3VLVideoProcessor(BaseVideoProcessor): def _preprocess( self, videos: list[torch.Tensor], - do_convert_rgb: bool, do_resize: bool, size: SizeDict, resample: "PILImageResampling | InterpolationMode | int | None", From 9d2a6f52b264c20b5816c5a094b191bfbcadd74b Mon Sep 17 00:00:00 2001 From: Alec <35311602+alec-flowers@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:14:37 -0700 Subject: [PATCH 247/248] [Rust Frontend] Expose gRPC port in Python vllm serve (#59659) Co-authored-by: Bugen Zhao Co-authored-by: Claude Opus 5.5 Signed-off-by: Alec Flowers Signed-off-by: Bugen Zhao --- rust/README.md | 21 +++++ rust/src/cmd/src/cli.rs | 31 +++++-- rust/src/cmd/src/cli/tests.rs | 77 ++++++++++++++-- .../examples/external_engine_openai_qwen.rs | 2 +- rust/src/server/src/config.rs | 7 +- rust/src/server/src/lib.rs | 41 +-------- .../test_api_server_process_manager.py | 53 +++++++++++ tests/entrypoints/launchers/test_cli_args.py | 89 +++++++++++++++++-- vllm/entrypoints/cli/serve.py | 9 ++ vllm/entrypoints/launchers/cli_args.py | 45 +++++++++- vllm/v1/utils.py | 9 +- 11 files changed, 320 insertions(+), 64 deletions(-) diff --git a/rust/README.md b/rust/README.md index 140b027b02..42d6c3e16a 100644 --- a/rust/README.md +++ b/rust/README.md @@ -43,6 +43,27 @@ For example: VLLM_USE_RUST_FRONTEND=1 vllm serve Qwen/Qwen3-0.6B ``` +To also expose the Rust frontend's gRPC inference and control services, add +`--grpc-port`. The HTTP listener remains on `--port`: + +```bash +VLLM_USE_RUST_FRONTEND=1 vllm serve Qwen/Qwen3-0.6B \ + --host 127.0.0.1 --port 8000 --grpc-port 50051 +``` + +The gRPC listener binds on `--host` like HTTP, or on `127.0.0.1` when HTTP uses +`--uds`. When HTTP uses TCP, its port and the gRPC port must differ unless +requesting an automatically assigned port with `0`. + +This also works with Python-supervised hybrid data parallelism. It requires a +frontend process and cannot be combined with `--headless` or +`--data-parallel-multi-port-external-lb`. + +The separate `vllm serve MODEL --grpc --port 50051` command launches the Python +SMG `VllmEngine` gRPC server in place of HTTP. The Rust listener exposes the +`vllm.Inference` and `vllm.Control` services alongside HTTP. These are different +gRPC APIs; `--grpc` and `--grpc-port` are mutually exclusive. + ### RL weight synchronization With `VLLM_SERVER_DEV_MODE=1`, the Rust frontend supports the HTTP weight-transfer diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 1fab1c93d1..1269e0b8af 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -268,11 +268,6 @@ pub struct SharedRuntimeArgs { #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)] #[serde(default)] pub max_logprobs: Option, - /// TCP port for the gRPC Inference service. When not set, no gRPC server is - /// started. - #[arg(long)] - #[serde(default)] - pub grpc_port: Option, /// Maximum time to wait for active requests to drain during shutdown. #[arg(long, default_value_t = 0)] #[serde(default)] @@ -505,6 +500,7 @@ impl SharedRuntimeArgs { fn into_bootstrapped_config( self, listen_fd: i32, + grpc_listen_fd: Option, input_address: String, output_address: String, coordinator_address: Option, @@ -539,6 +535,7 @@ impl SharedRuntimeArgs { generation_config: self.generation_config, served_model_name: self.served_model_name, listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd }, + grpc_listener_mode: grpc_listen_fd.map(|fd| HttpListenerMode::InheritedFd { fd }), tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, tool_strict_level: self.tool_strict_level, @@ -555,7 +552,6 @@ impl SharedRuntimeArgs { tls, api_keys: self.api_key, disable_log_stats: self.disable_log_stats, - grpc_port: self.grpc_port, shutdown_timeout, // The engine is launched and supervised by another process. manages_engine: false, @@ -569,6 +565,7 @@ impl SharedRuntimeArgs { fn into_managed_config( self, listener_mode: HttpListenerMode, + grpc_listener_mode: Option, handshake_address: String, advertised_host: String, engine_count: usize, @@ -600,6 +597,7 @@ impl SharedRuntimeArgs { generation_config: self.generation_config, served_model_name: self.served_model_name, listener_mode, + grpc_listener_mode, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, tool_strict_level: self.tool_strict_level, @@ -616,7 +614,6 @@ impl SharedRuntimeArgs { tls, api_keys: self.api_key, disable_log_stats: self.disable_log_stats, - grpc_port: self.grpc_port, shutdown_timeout, manages_engine, keep_alive_timeout, @@ -696,6 +693,10 @@ pub struct FrontendArgs { /// supervisor. #[arg(long)] pub listen_fd: i32, + /// Inherited gRPC listening socket file descriptor passed by the Python + /// supervisor. When not set, no gRPC server is started. + #[arg(long)] + pub grpc_listen_fd: Option, /// Frontend input ROUTER socket address that the Python engines will /// connect to. #[arg(long)] @@ -731,6 +732,7 @@ impl FrontendArgs { let data_parallel_size = self.data_parallel_size.unwrap_or(self.engine_count); self.runtime.into_bootstrapped_config( self.listen_fd, + self.grpc_listen_fd, self.input_address, self.output_address, self.coordinator_address, @@ -760,6 +762,10 @@ pub struct ServeArgs { /// Unix domain socket path. If set, host and port arguments are ignored. #[arg(long)] pub uds: Option, + /// TCP port for the gRPC Inference and Control services. When not set, no + /// gRPC server is started. + #[arg(long)] + pub grpc_port: Option, /// Flag to print debug information about CLI argument parsing and exit. #[educe(Debug(ignore))] @@ -789,9 +795,20 @@ impl ServeArgs { port: self.port, }, }; + // gRPC follows the HTTP TCP host. With a Unix socket it defaults to IPv4 + // loopback rather than all interfaces, so the side-car is never + // accidentally network-exposed. + let grpc_listener_mode = self.grpc_port.map(|port| HttpListenerMode::BindTcp { + host: match self.uds { + Some(_) => "127.0.0.1".to_string(), + None => self.host.clone(), + }, + port, + }); self.runtime.clone().into_managed_config( listener_mode, + grpc_listener_mode, handshake_address, self.managed_engine.handshake_host.clone(), self.managed_engine.data_parallel_size, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 0802d2062a..944febc36c 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -174,6 +174,7 @@ fn serve_args_forward_python_flags_with_separator() { host: "127.0.0.1", port: 8000, uds: None, + grpc_port: None, runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", revision: None, @@ -188,7 +189,6 @@ fn serve_args_forward_python_flags_with_separator() { renderer: Auto, language_model_only: false, max_logprobs: None, - grpc_port: None, shutdown_timeout: 0, http_timeout_keep_alive: None, chat_template: None, @@ -1007,6 +1007,7 @@ fn frontend_args_accept_json() { command: Frontend( FrontendArgs { listen_fd: 3, + grpc_listen_fd: None, input_address: "ipc:///tmp/input.sock", output_address: "ipc:///tmp/output.sock", coordinator_address: Some( @@ -1029,7 +1030,6 @@ fn frontend_args_accept_json() { renderer: Auto, language_model_only: false, max_logprobs: None, - grpc_port: None, shutdown_timeout: 0, http_timeout_keep_alive: None, chat_template: None, @@ -1687,6 +1687,7 @@ fn serve_args_accept_handshake_aliases() { host: "127.0.0.1", port: 8000, uds: None, + grpc_port: None, runtime: SharedRuntimeArgs { model: "Qwen/Qwen3-0.6B", revision: None, @@ -1701,7 +1702,6 @@ fn serve_args_accept_handshake_aliases() { renderer: Auto, language_model_only: false, max_logprobs: None, - grpc_port: None, shutdown_timeout: 0, http_timeout_keep_alive: None, chat_template: None, @@ -1854,6 +1854,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { host: "127.0.0.1", port: 8000, }, + grpc_listener_mode: None, tool_call_parser: Auto, reasoning_parser: Auto, tool_strict_level: Auto, @@ -1887,7 +1888,6 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { tls: None, api_keys: [], disable_log_stats: false, - grpc_port: None, shutdown_timeout: 0ns, manages_engine: true, keep_alive_timeout: 5s, @@ -1949,6 +1949,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { host: "127.0.0.1", port: 8000, }, + grpc_listener_mode: None, tool_call_parser: Auto, reasoning_parser: Auto, tool_strict_level: Auto, @@ -1982,7 +1983,6 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { tls: None, api_keys: [], disable_log_stats: false, - grpc_port: None, shutdown_timeout: 0ns, manages_engine: true, keep_alive_timeout: 5s, @@ -2088,6 +2088,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present listener_mode: InheritedFd { fd: 3, }, + grpc_listener_mode: None, tool_call_parser: None, reasoning_parser: None, tool_strict_level: Auto, @@ -2121,7 +2122,6 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present tls: None, api_keys: [], disable_log_stats: false, - grpc_port: None, shutdown_timeout: 0ns, manages_engine: false, keep_alive_timeout: 5s, @@ -2139,6 +2139,8 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() { "Qwen/Qwen3-0.6B", "--uds", "/tmp/vllm.sock", + "--grpc-port", + "50051", ]) .unwrap(); @@ -2153,6 +2155,69 @@ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() { path: "/tmp/vllm.sock".to_string(), } ); + assert_eq!( + config.grpc_listener_mode, + Some(HttpListenerMode::BindTcp { + host: "127.0.0.1".to_string(), + port: 50051, + }) + ); +} + +#[test] +fn serve_frontend_config_binds_grpc_on_http_host() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "serve", + "Qwen/Qwen3-0.6B", + "--host", + "0.0.0.0", + "--grpc-port", + "50051", + ]) + .unwrap(); + + let Command::Serve(args) = cli.command else { + panic!("expected serve args"); + }; + let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string()); + + assert_eq!( + config.grpc_listener_mode, + Some(HttpListenerMode::BindTcp { + host: "0.0.0.0".to_string(), + port: 50051, + }) + ); +} + +#[test] +fn frontend_config_adopts_inherited_grpc_listener() { + let cli = Cli::try_parse_from([ + "vllm-rs", + "frontend", + "--listen-fd", + "3", + "--grpc-listen-fd", + "4", + "--input-address", + "ipc:///tmp/input.sock", + "--output-address", + "ipc:///tmp/output.sock", + "--args-json", + r#"{"model_tag":"Qwen/Qwen3-0.6B","grpc_port":50051}"#, + ]) + .unwrap(); + + let Command::Frontend(args) = cli.command else { + panic!("expected frontend args"); + }; + let config = args.into_config(); + + assert_eq!( + config.grpc_listener_mode, + Some(HttpListenerMode::InheritedFd { fd: 4 }) + ); } #[test] diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index cb5fed4437..43192ab78f 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -69,6 +69,7 @@ async fn main() -> Result<()> { host: "127.0.0.1".to_string(), port, }, + grpc_listener_mode: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, tool_strict_level: ToolStrictLevel::Auto, @@ -85,7 +86,6 @@ async fn main() -> Result<()> { tls: None, api_keys: Vec::new(), disable_log_stats: false, - grpc_port: None, shutdown_timeout: Duration::ZERO, manages_engine: false, keep_alive_timeout: Duration::from_secs(5), diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index b307f8b3ac..8e51f1440b 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -23,7 +23,7 @@ use vllm_text::backend::hf::HfOverrides; /// when keep-alive is disabled (`0`). pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5); -/// How the HTTP server obtains its listening socket. +/// How the HTTP or gRPC server obtains its listening socket. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum HttpListenerMode { /// Bind a fresh TCP listener on the given host/port. @@ -229,6 +229,8 @@ pub struct Config { pub served_model_name: Vec, /// HTTP listener setup. pub listener_mode: HttpListenerMode, + /// gRPC listener setup. When `None`, no gRPC server is started. + pub grpc_listener_mode: Option, /// Tool-call parser selection. pub tool_call_parser: ParserSelection, /// Reasoning parser selection. @@ -272,9 +274,6 @@ pub struct Config { /// engine-reported scheduler stats and request lifecycle events are not /// exported. pub disable_log_stats: bool, - /// TCP port for the gRPC Inference service. When `None`, no gRPC server is - /// started. - pub grpc_port: Option, /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown. pub shutdown_timeout: Duration, /// Whether the caller manages the engine process and shuts it down when diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 6a36491fd9..f54f8ce3e4 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -37,7 +37,6 @@ use hyper_util::rt::{TokioIo, TokioTimer}; use hyper_util::server::graceful::GracefulShutdown; use hyper_util::service::TowerToHyperService; pub use render::{RenderConfig, serve_render}; -use tokio::net::TcpListener; use tokio::time::{Instant, sleep_until}; use tokio_util::sync::CancellationToken; use tonic::transport::Server as TonicServer; @@ -74,17 +73,6 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve } } -/// Choose the gRPC listener host. It follows the HTTP TCP host when there is -/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback -/// rather than all interfaces, so the side-car is never accidentally -/// network-exposed. -fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str { - match listener_mode { - HttpListenerMode::BindTcp { host, .. } => host.as_str(), - HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1", - } -} - /// Build the shared application state for one configured model and one engine /// client, including any static LoRA adapters from `--lora-modules`. async fn build_state(config: &Config) -> Result> { @@ -229,13 +217,11 @@ where // Optionally bind the gRPC Inference server on a separate port. Bind // synchronously here so bind errors (port in use, permission denied, ...) // surface before serving rather than being deferred until shutdown. - let grpc_setup = if let Some(grpc_port) = config.grpc_port { - let grpc_host = grpc_bind_host(&config.listener_mode); - let grpc_listener = TcpListener::bind((grpc_host, grpc_port)) + let grpc_setup = if let Some(grpc_listener_mode) = &config.grpc_listener_mode { + let grpc_listener = Listener::bind(grpc_listener_mode) .await - .with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?; - let addr = grpc_listener.local_addr()?; - let grpc_listener = Listener::Tcp(grpc_listener); + .with_context(|| format!("failed to bind gRPC listener for {grpc_listener_mode:?}"))?; + let addr = grpc_listener.local_addr_display()?; // gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2. let grpc_tls = config .tls @@ -479,23 +465,4 @@ mod tests { served_names ); } - - #[test] - fn grpc_bind_host_follows_http_tcp_host() { - let mode = HttpListenerMode::BindTcp { - host: "0.0.0.0".to_string(), - port: 8000, - }; - assert_eq!(grpc_bind_host(&mode), "0.0.0.0"); - } - - #[test] - fn grpc_bind_host_defaults_to_loopback_without_tcp_host() { - let unix = HttpListenerMode::BindUnix { - path: "/tmp/vllm.sock".to_string(), - }; - let inherited = HttpListenerMode::InheritedFd { fd: 3 }; - assert_eq!(grpc_bind_host(&unix), "127.0.0.1"); - assert_eq!(grpc_bind_host(&inherited), "127.0.0.1"); - } } diff --git a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py index 10858b6da4..6c579df838 100644 --- a/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py +++ b/tests/entrypoints/launchers/api_server/test_api_server_process_manager.py @@ -536,3 +536,56 @@ def test_rust_frontend_uses_config_model_as_model_tag(monkeypatch, caplog, tmp_p sock.close() assert '"model_tag": "org/model"' in caplog.text + + +def test_rust_frontend_inherits_grpc_listener(monkeypatch): + """The Python-bound gRPC socket is passed to Rust like the HTTP one.""" + import subprocess as subprocess_mod + + from vllm.entrypoints.launchers.cli_args import make_arg_parser + from vllm.utils.argparse_utils import FlexibleArgumentParser + from vllm.v1.utils import RustFrontendProcessManager + + args = make_arg_parser(FlexibleArgumentParser()).parse_args( + ["--model", "org/model", "--grpc-port", "50051"] + ) + + class _FakeProc: + pid = 4321 + returncode = 0 + + def poll(self): + return 0 + + def wait(self, timeout=None): + return 0 + + popen_calls = [] + + def fake_popen(cmd, **kwargs): + popen_calls.append((cmd, kwargs)) + return _FakeProc() + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + grpc_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + monkeypatch.setattr(subprocess_mod, "Popen", fake_popen) + RustFrontendProcessManager( + binary_path="/nonexistent/vllm-rs", + sock=sock, + args=args, + input_address="ipc:///tmp/in", + output_address="ipc:///tmp/out", + engine_start_index=0, + engine_count=1, + data_parallel_size=1, + grpc_sock=grpc_sock, + ) + [(cmd, kwargs)] = popen_calls + grpc_fd = grpc_sock.fileno() + flag_index = cmd.index("--grpc-listen-fd") + assert cmd[flag_index + 1] == str(grpc_fd) + assert list(kwargs["pass_fds"]) == [sock.fileno(), grpc_fd] + finally: + sock.close() + grpc_sock.close() diff --git a/tests/entrypoints/launchers/test_cli_args.py b/tests/entrypoints/launchers/test_cli_args.py index 97c1aa5fc7..6b9d496822 100644 --- a/tests/entrypoints/launchers/test_cli_args.py +++ b/tests/entrypoints/launchers/test_cli_args.py @@ -28,7 +28,7 @@ assert CHATML_JINJA_PATH.exists() def _build_vllm_parsers(): vllm_parser = FlexibleArgumentParser() - subparsers = vllm_parser.add_subparsers() + subparsers = vllm_parser.add_subparsers(dest="subparser") serve_parser = subparsers.add_parser("serve") make_arg_parser(serve_parser) return {"vllm": vllm_parser, "vllm serve": serve_parser} @@ -173,6 +173,75 @@ def test_multiple_valid_inputs(serve_parser): ### Tests for serve argument validation that run prior to loading +@pytest.mark.parametrize( + "port, extra_args", + [ + (50051, []), + (0, ["--port", "0"]), + (50051, ["--port", "50051", "--uds", "/tmp/vllm.sock"]), + ], +) +def test_rust_grpc_port_accepts_frontend_launch( + vllm_parser, monkeypatch, port, extra_args +): + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1") + args = vllm_parser.parse_args( + [ + "serve", + "--grpc-port", + str(port), + "--data-parallel-size", + "8", + "--data-parallel-size-local", + "4", + "--data-parallel-start-rank", + "4", + "--data-parallel-hybrid-lb", + *extra_args, + ] + ) + validate_parsed_serve_args(args) + assert args.grpc_port == port + + +@pytest.mark.parametrize( + "rust_enabled, extra_args, port", + [ + (False, [], 50051), + (False, ["--grpc"], 50051), + (True, ["--grpc"], 50051), + (True, ["--headless"], 50051), + (True, ["--api-server-count", "0"], 50051), + (True, ["--api-server-count", "-1"], 50051), + (True, ["--data-parallel-multi-port-external-lb"], 50051), + (True, ["--port", "50051"], 50051), + (True, [], -1), + (True, [], 65536), + ], +) +def test_rust_grpc_port_rejects_incompatible_launch( + vllm_parser, monkeypatch, rust_enabled, extra_args, port +): + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1" if rust_enabled else "0") + args = vllm_parser.parse_args(["serve", "--grpc-port", str(port), *extra_args]) + error = ( + "--grpc and --grpc-port are mutually exclusive" + if "--grpc" in extra_args + else "--grpc-port" + ) + with pytest.raises(ValueError, match=error): + validate_parsed_serve_args(args) + + +def test_rust_grpc_port_requires_serve_subcommand(monkeypatch): + """Direct Python render/API entrypoints must reject a Rust-only listener.""" + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1") + parser = make_arg_parser(FlexibleArgumentParser()) + args = parser.parse_args(["--grpc-port", "50051"]) + with pytest.raises(ValueError, match="--grpc-port requires"): + validate_parsed_serve_args(args) + + def test_enable_auto_choice_passes_without_tool_call_parser(serve_parser): """Ensure validation fails if tool choice is enabled with no call parser.""" # If we enable-auto-tool-choice, explode with no tool-call-parser @@ -265,13 +334,21 @@ def launch_render_parser(): return _build_launch_render_parser() -def test_launch_render_validates_serve_args(launch_render_parser): +@pytest.mark.parametrize( + "cli_args, error", + [ + (["--enable-auto-tool-choice"], TypeError), + (["--grpc-port", "50051"], ValueError), + ], +) +def test_launch_render_validates_serve_args( + launch_render_parser, monkeypatch, cli_args, error +): """`vllm launch render` reuses the serve parser, so it gets the serve checks""" - args = launch_render_parser.parse_args( - args=["launch", "render", "--enable-auto-tool-choice"] - ) + monkeypatch.setenv("VLLM_USE_RUST_FRONTEND", "1") + args = launch_render_parser.parse_args(args=["launch", "render", *cli_args]) assert args.subparser == "launch" - with pytest.raises(TypeError): + with pytest.raises(error): validate_parsed_serve_args(args) diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index 1f6ca2b843..4c9410488b 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -17,6 +17,7 @@ from vllm.entrypoints.launchers.cli_args import ( validate_parsed_serve_args, ) from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor +from vllm.entrypoints.launchers.launcher import create_server_socket from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager @@ -295,6 +296,13 @@ def run_multi_api_server(args: argparse.Namespace): signal.signal(signal.SIGINT, signal_handler) listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1) + # `--grpc-port` is only accepted for the Rust frontend, which inherits this + # listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4 + # loopback when HTTP uses a Unix socket. + grpc_sock = None + if args.grpc_port is not None: + grpc_host = "127.0.0.1" if args.uds else (args.host or "") + grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False) engine_args = vllm.AsyncEngineArgs.from_cli_args(args) propagate_flash_late_interaction(args, engine_args) @@ -355,6 +363,7 @@ def run_multi_api_server(args: argparse.Namespace): api_server_manager = RustFrontendProcessManager( binary_path=rust_frontend_path, sock=sock, + grpc_sock=grpc_sock, args=args, input_address=addresses.inputs[0], output_address=addresses.outputs[0], diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index a800e1270d..4c1f121372 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -284,6 +284,11 @@ class FrontendArgs(BaseFrontendArgs): """Host name.""" port: int = 8000 """Port number.""" + grpc_port: int | None = None + """Enable the Rust frontend's additional gRPC Inference and Control services + on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on + `--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined + with the Python gRPC server's `--grpc` flag.""" data_parallel_supervisor_port: int = 9256 """HTTP port for aggregated health endpoints in multi-port external LB mode.""" @@ -451,8 +456,10 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: "--grpc", action="store_true", default=False, - help="Launch a gRPC server instead of the HTTP OpenAI-compatible " - "server. Requires: pip install vllm[grpc].", + help="Launch the Python gRPC (SMG VllmEngine) server on --port instead " + "of the HTTP OpenAI-compatible server. Requires: pip install vllm[grpc]. " + "Cannot be combined with --grpc-port, which enables the Rust " + "frontend's gRPC services.", ) parser = FrontendArgs.add_cli_args(parser) parser = AsyncEngineArgs.add_cli_args(parser) @@ -460,6 +467,38 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: return parser +def validate_grpc_port_arg(args: argparse.Namespace) -> None: + """Validate the Rust frontend's optional gRPC listener.""" + if getattr(args, "grpc_port", None) is None: + return + + if args.grpc: + raise ValueError( + "--grpc and --grpc-port are mutually exclusive. Use --grpc " + "--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 " + "with --grpc-port for the Rust frontend." + ) + if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve": + raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve") + if args.headless or ( + args.api_server_count is not None and args.api_server_count <= 0 + ): + raise ValueError( + "--grpc-port requires a Rust frontend; remove --headless " + "and non-positive --api-server-count" + ) + if args.data_parallel_multi_port_external_lb: + raise ValueError( + "--grpc-port is incompatible with " + "--data-parallel-multi-port-external-lb: " + "its frontends would share the same gRPC port" + ) + if not 0 <= args.grpc_port <= 65535: + raise ValueError("--grpc-port must be between 0 and 65535") + if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds: + raise ValueError("--grpc-port must differ from --port when HTTP uses TCP") + + def validate_parsed_serve_args(args: argparse.Namespace): """Quick checks for model serve args that raise prior to loading.""" # `vllm launch ` builds its parser with make_arg_parser too (see @@ -478,6 +517,8 @@ def validate_parsed_serve_args(args: argparse.Namespace): "the checkpoint's output format." ) + validate_grpc_port_arg(args) + # Enable auto tool needs a tool call parser to be valid if args.enable_auto_tool_choice and not args.tool_call_parser: raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser") diff --git a/vllm/v1/utils.py b/vllm/v1/utils.py index f3bd97784e..ac877263a5 100644 --- a/vllm/v1/utils.py +++ b/vllm/v1/utils.py @@ -354,6 +354,7 @@ class RustFrontendProcessManager: engine_count: int, data_parallel_size: int, stats_update_address: str | None = None, + grpc_sock: Any | None = None, ): import os import subprocess @@ -377,6 +378,12 @@ class RustFrontendProcessManager: "--data-parallel-size", str(data_parallel_size), ] + pass_fds = [fd] + if grpc_sock is not None: + grpc_fd = grpc_sock.fileno() + os.set_inheritable(grpc_fd, True) + cmd.extend(["--grpc-listen-fd", str(grpc_fd)]) + pass_fds.append(grpc_fd) if stats_update_address is not None: cmd.extend(["--coordinator-address", stats_update_address]) from vllm.entrypoints.serve.utils.api_utils import jsonify_non_default_args @@ -418,7 +425,7 @@ class RustFrontendProcessManager: redacted_json = json.dumps(redact_sensitive_args(args_dict), sort_keys=True) logger.info("Launching Rust frontend: %s", " ".join(cmd[:-1] + [redacted_json])) - self._proc = subprocess.Popen(cmd, pass_fds=(fd,)) + self._proc = subprocess.Popen(cmd, pass_fds=pass_fds) # Create a process wrapper with a sentinel fd for monitoring self.processes: list[_SubprocessWrapper] = [ From 58b3298457dde7b4554b3b4e20b238c0ac2c3a65 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Fri, 2 Oct 2026 16:27:11 +0800 Subject: [PATCH 248/248] [Misc] Add Transformers version upper bound in requirements (#59614) Signed-off-by: Isotr0py Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- requirements/common.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/common.txt b/requirements/common.txt index 8e483cc712..b6b33efa92 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -7,7 +7,7 @@ requests >= 2.33.0 tqdm blake3 py-cpuinfo -transformers >= 5.16.1 +transformers >= 5.16.1, < 5.18.0 huggingface_hub >= 1.31.0 # Required for the `huggingface_hub.utils.httpx` re-export tokenizers >= 0.22.0 # Required for fast incremental detokenization. safetensors >= 0.6.2 # MXFP4/MXFP6 dtype support (F8_E8M0, F4) added in 0.6.0: https://github.com/huggingface/safetensors/pull/611