From e55cc59e52791fe7620f6808e409edcf68740b6b Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 7 Jul 2026 16:27:20 +0800 Subject: [PATCH] [Rust Frontend][CI] Unblock more end-to-end test cases (#47735) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao --- .buildkite/test-amd.yaml | 14 ++++- .buildkite/test_areas/rust_frontend.yaml | 23 +++++--- rust/src/cmd/src/cli/unsupported.rs | 8 ++- .../src/routes/inference/generate/types.rs | 6 +- .../src/routes/openai/chat_completions.rs | 2 +- .../routes/openai/chat_completions/convert.rs | 2 +- .../routes/openai/chat_completions/types.rs | 9 +-- .../src/routes/openai/completions/types.rs | 5 +- .../server/src/routes/openai/utils/types.rs | 8 +-- rust/src/server/src/routes/tests.rs | 57 ++++++++++++++++--- rust/src/server/src/routes/tokenize/types.rs | 3 +- rust/src/text/src/lower/token_ids.rs | 2 +- tests/entrypoints/openai/test_uds.py | 3 +- .../serve/instrumentator/test_basic.py | 3 +- 14 files changed, 105 insertions(+), 40 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 9c1b2077a7..51161a51c7 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1977,7 +1977,10 @@ steps: - tests/utils.py - tests/benchmarks/test_serve_cli.py - tests/entrypoints/openai/chat_completion/test_chat_completion.py + - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py - tests/entrypoints/openai/completion/test_shutdown.py + - tests/entrypoints/openai/test_return_token_ids.py + - tests/entrypoints/openai/test_uds.py - tests/v1/sample/test_logprobs_e2e.py - vllm/platforms/rocm.py commands: @@ -1985,7 +1988,10 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex" + - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" + - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" + - pytest -v -s entrypoints/openai/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" - label: Rust Frontend Serve Admin Coverage # TBD @@ -2000,16 +2006,20 @@ steps: - vllm/entrypoints/serve/ - vllm/v1/engine/ - tests/utils.py - - tests/entrypoints/serve/disagg/test_serving_tokens.py + - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py + - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py - tests/entrypoints/serve/instrumentator/test_basic.py - tests/entrypoints/serve/instrumentator/test_metrics.py + - tests/entrypoints/serve/tokenize/test_tokenization.py - vllm/platforms/rocm.py commands: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load" - - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" + - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist" + - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info and not test_tokenize_chat" - label: Rust Frontend Core Correctness # TBD timeout_in_minutes: 180 diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index adb27c4a04..6b2aa4cb0b 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -15,24 +15,26 @@ steps: - tests/utils.py - tests/benchmarks/test_serve_cli.py - tests/entrypoints/openai/chat_completion/test_chat_completion.py - # - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py + - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py # - tests/entrypoints/openai/completion/test_prompt_validation.py - tests/entrypoints/openai/completion/test_shutdown.py - # - tests/entrypoints/openai/test_return_token_ids.py - # - tests/entrypoints/openai/test_uds.py + - tests/entrypoints/openai/test_return_token_ids.py + - tests/entrypoints/openai/test_uds.py - tests/v1/sample/test_logprobs_e2e.py commands: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)" - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex" - # - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid" + - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple" # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds" - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly" - # - pytest -v -s entrypoints/openai/test_return_token_ids.py - # - pytest -v -s entrypoints/openai/test_uds.py + # test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and + # finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk. + - pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison" + - pytest -v -s entrypoints/openai/test_uds.py - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server" - label: Rust Frontend Serve/Admin Coverage @@ -45,19 +47,24 @@ steps: - vllm/entrypoints/serve/ - vllm/v1/engine/ - tests/utils.py - # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py + - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py - tests/entrypoints/serve/instrumentator/test_basic.py - tests/entrypoints/serve/instrumentator/test_metrics.py # - tests/entrypoints/serve/dev/test_sleep.py + - tests/entrypoints/serve/tokenize/test_tokenization.py commands: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py + - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py + # server_load can be flaky under the Rust frontend; keep it excluded for now. - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load" + # test_generate_logprobs expects Python-style top_logprobs truncation (dedup sampled + cap at max(k, 1)). - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist" # - pytest -v -s entrypoints/serve/dev/test_sleep.py + # /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op). + - pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info and not test_tokenize_chat" - label: Rust Frontend Core Correctness timeout_in_minutes: 30 diff --git a/rust/src/cmd/src/cli/unsupported.rs b/rust/src/cmd/src/cli/unsupported.rs index 521a188d6a..548001412c 100644 --- a/rust/src/cmd/src/cli/unsupported.rs +++ b/rust/src/cmd/src/cli/unsupported.rs @@ -456,13 +456,17 @@ pub struct ServerUnsupportedArgs { /// Enable the `/tokenizer_info` endpoint. May expose chat /// templates and other tokenizer configuration. + /// + /// Accepted as a no-op: the Rust frontend serves `/tokenize` and + /// `/detokenize`, but does not implement `/tokenizer_info` yet. #[arg( long, visible_alias = "no-enable-tokenizer-info-endpoint", default_missing_value = "true", - num_args = 0..=1 + num_args = 0..=1, + hide = true )] - pub enable_tokenizer_info_endpoint: Option, + pub enable_tokenizer_info_endpoint: Option, /// If set to True, log model outputs (generations). /// Requires `--enable-log-requests`. As with `--enable-log-requests`, diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index d4567c44aa..28855968df 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -29,7 +29,9 @@ pub struct GenerateRequest { impl Normalizable for GenerateRequest {} /// Mirrors the Python vLLM `GenerateResponseChoice` class. -#[serde_with::skip_serializing_none] +/// +/// Do not skip serializing `None` fields here: non-streaming response types +/// should serialize `None` as explicit `null`. #[derive(Debug, Clone, Serialize)] pub(super) struct GenerateResponseChoice { pub index: u32, @@ -58,7 +60,6 @@ pub(super) struct GenerateStreamResponse { } /// Mirrors the Python vLLM `GenerateResponse` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct GenerateResponse { pub request_id: String, @@ -68,7 +69,6 @@ pub(super) struct GenerateResponse { } /// Mirrors the Python vLLM `Logprob` class used in prompt-logprobs payloads. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct GenerateLogprob { pub logprob: f32, diff --git a/rust/src/server/src/routes/openai/chat_completions.rs b/rust/src/server/src/routes/openai/chat_completions.rs index 0429d9ac8b..f0368c5614 100644 --- a/rust/src/server/src/routes/openai/chat_completions.rs +++ b/rust/src/server/src/routes/openai/chat_completions.rs @@ -211,7 +211,7 @@ async fn collect_chat_completion( Some(prefix) => Some(format!("{prefix}{}", message.text())), None => Some(message.text()).filter(|t| !t.is_empty()), }, - tool_calls: Some(tool_calls).filter(|calls| !calls.is_empty()), + tool_calls, reasoning: if include_reasoning { reasoning } else { None }, }, logprobs, diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 26a7627149..a284b2197f 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -830,7 +830,7 @@ mod tests { let message = ChatCompletionMessage { role: AssistantRole, content: Some("answer".to_string()), - tool_calls: None, + tool_calls: Vec::new(), reasoning: Some("inner".to_string()), }; let message_json = serde_json::to_value(message).expect("message serializes"); diff --git a/rust/src/server/src/routes/openai/chat_completions/types.rs b/rust/src/server/src/routes/openai/chat_completions/types.rs index b5681d0abd..f3a4dd2413 100644 --- a/rust/src/server/src/routes/openai/chat_completions/types.rs +++ b/rust/src/server/src/routes/openai/chat_completions/types.rs @@ -331,7 +331,9 @@ impl Normalizable for ChatCompletionRequest { } /// Mirrors the Python vLLM `ChatCompletionResponse` class. -#[serde_with::skip_serializing_none] +/// +/// Do not skip serializing `None` fields here: non-streaming response types +/// should serialize `None` as explicit `null`. #[derive(Debug, Clone, Serialize)] pub(super) struct ChatCompletionResponse { pub id: String, @@ -347,7 +349,6 @@ pub(super) struct ChatCompletionResponse { } /// Mirrors the Python vLLM `ChatCompletionResponseChoice` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct ChatCompletionChoice { pub index: u32, @@ -370,12 +371,12 @@ impl fmt::Display for AssistantRole { } /// Mirrors the Python vLLM response `ChatMessage` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct ChatCompletionMessage { pub role: AssistantRole, pub content: Option, - pub tool_calls: Option>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tool_calls: Vec, pub reasoning: Option, } diff --git a/rust/src/server/src/routes/openai/completions/types.rs b/rust/src/server/src/routes/openai/completions/types.rs index fe637d82de..32542b8b35 100644 --- a/rust/src/server/src/routes/openai/completions/types.rs +++ b/rust/src/server/src/routes/openai/completions/types.rs @@ -196,7 +196,9 @@ impl Normalizable for CompletionRequest { } /// Mirrors the Python vLLM `CompletionResponse` class. -#[serde_with::skip_serializing_none] +/// +/// Do not skip serializing `None` fields here: non-streaming response types +/// should serialize `None` as explicit `null`. #[derive(Debug, Clone, Serialize)] pub(super) struct CompletionResponse { pub id: String, @@ -210,7 +212,6 @@ pub(super) struct CompletionResponse { } /// Mirrors the Python vLLM `CompletionResponseChoice` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub(super) struct CompletionChoice { pub index: u32, diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index 8b079bbcc1..98eb10f937 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -311,7 +311,9 @@ pub enum MessageContent { // ============================================================================ /// Mirrors the Python vLLM `UsageInfo` class. -#[serde_with::skip_serializing_none] +/// +/// Do not skip serializing `None` fields here: non-streaming response types +/// should serialize `None` as explicit `null`. #[derive(Debug, Clone, Serialize)] pub struct Usage { pub prompt_tokens: usize, @@ -402,14 +404,12 @@ pub struct LogProbs { } /// Mirrors the Python vLLM `ChatCompletionLogProbs` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub struct ChatLogProbs { pub content: Option>, } /// Mirrors the Python vLLM `ChatCompletionLogProbsContent` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub struct ChatLogProbsContent { pub token: String, @@ -419,7 +419,6 @@ pub struct ChatLogProbsContent { } /// Mirrors the Python vLLM `ChatCompletionLogProb` class. -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Serialize)] pub struct TopLogProb { pub token: String, @@ -436,7 +435,6 @@ pub struct ErrorResponse { pub error: ErrorDetail, } -#[serde_with::skip_serializing_none] #[derive(Debug, Clone, Deserialize, Serialize)] pub struct ErrorDetail { pub message: String, diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 02348151ea..520d504383 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -2191,6 +2191,28 @@ async fn non_stream_chat_returns_json_response() { assert_eq!(json["usage"]["prompt_tokens"], 22); assert_eq!(json["usage"]["completion_tokens"], 3); assert_eq!(json["usage"]["total_tokens"], 25); + + // Unset optional fields are serialized as explicit `null` on + // non-streaming responses... + let response_object = json.as_object().expect("response object"); + let choice = json["choices"][0].as_object().expect("choice object"); + let message = choice["message"].as_object().expect("message object"); + for (object, key) in [ + (response_object, "system_fingerprint"), + (response_object, "prompt_token_ids"), + (response_object, "kv_transfer_params"), + (choice, "logprobs"), + (choice, "stop_reason"), + (choice, "token_ids"), + (message, "reasoning"), + ] { + assert!( + object.contains_key(key) && object[key].is_null(), + "expected explicit null `{key}`: {json}" + ); + } + // ...except `tool_calls`, which Python pops from the payload when empty. + assert!(!message.contains_key("tool_calls"), "{json}"); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -2956,6 +2978,25 @@ async fn non_stream_completions_return_json_response() { assert_eq!(json["choices"][0]["text"], "hi"); assert_eq!(json["choices"][0]["finish_reason"], "stop"); assert_eq!(json["usage"]["completion_tokens"], 3); + + // Unset optional fields are serialized as explicit `null` on + // non-streaming responses. + let response_object = json.as_object().expect("response object"); + let choice = json["choices"][0].as_object().expect("choice object"); + for (object, key) in [ + (response_object, "system_fingerprint"), + (response_object, "kv_transfer_params"), + (choice, "logprobs"), + (choice, "stop_reason"), + (choice, "prompt_logprobs"), + (choice, "token_ids"), + (choice, "prompt_token_ids"), + ] { + assert!( + object.contains_key(key) && object[key].is_null(), + "expected explicit null `{key}`: {json}" + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -4371,10 +4412,10 @@ async fn include_reasoning_false_suppresses_reasoning_in_non_stream_chat() { let json: serde_json::Value = serde_json::from_str(&text).expect("decode json"); assert_eq!(json["choices"][0]["message"]["content"], "answer"); + // Suppressed fields are serialized as explicit `null` on non-streaming + // responses. assert!( - json["choices"][0]["message"] - .as_object() - .is_some_and(|message| !message.contains_key("reasoning")), + json["choices"][0]["message"]["reasoning"].is_null(), "{text}" ); } @@ -4476,14 +4517,14 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() { let choice = json["choices"][0].as_object().expect("choice object"); assert_eq!(json["choices"][0]["message"]["content"], "answer"); + // Suppressed fields are serialized as explicit `null` on non-streaming + // responses. assert!( - json["choices"][0]["message"] - .as_object() - .is_some_and(|message| !message.contains_key("reasoning")), + json["choices"][0]["message"]["reasoning"].is_null(), "{text}" ); - assert!(!choice.contains_key("logprobs"), "{text}"); - assert!(!choice.contains_key("token_ids"), "{text}"); + assert!(choice["logprobs"].is_null(), "{text}"); + assert!(choice["token_ids"].is_null(), "{text}"); assert!(json["prompt_token_ids"].is_array(), "{text}"); } diff --git a/rust/src/server/src/routes/tokenize/types.rs b/rust/src/server/src/routes/tokenize/types.rs index 987e0e23f3..f8c3e1cee8 100644 --- a/rust/src/server/src/routes/tokenize/types.rs +++ b/rust/src/server/src/routes/tokenize/types.rs @@ -102,12 +102,13 @@ pub struct DetokenizeRequest { pub tokens: Vec, } +/// Do not skip serializing `None` fields here: non-streaming response types +/// should serialize `None` as explicit `null`. #[derive(Debug, Clone, Serialize)] pub struct TokenizeResponse { pub count: usize, pub max_model_len: u32, pub tokens: Vec, - #[serde(skip_serializing_if = "Option::is_none")] pub token_strs: Option>, } diff --git a/rust/src/text/src/lower/token_ids.rs b/rust/src/text/src/lower/token_ids.rs index c237185883..ad37d86490 100644 --- a/rust/src/text/src/lower/token_ids.rs +++ b/rust/src/text/src/lower/token_ids.rs @@ -10,7 +10,7 @@ pub enum TokenIdsError { #[error("allowed_token_ids should not be empty")] EmptyAllowedTokenIds, #[error( - "token_id(s) {token_ids:?} in {parameter} contain out-of-vocab token ids. \ + "token_id(s) {token_ids:?} in {parameter} are out of vocabulary. \ Vocabulary size: {vocab_size}" )] OutOfVocab { diff --git a/tests/entrypoints/openai/test_uds.py b/tests/entrypoints/openai/test_uds.py index c79a4870de..f79e40ee41 100644 --- a/tests/entrypoints/openai/test_uds.py +++ b/tests/entrypoints/openai/test_uds.py @@ -40,4 +40,5 @@ async def test_show_version(server: RemoteOpenAIServer): response = client.get(server.url_for("version")) response.raise_for_status() - assert response.json() == {"version": VLLM_VERSION} + # Tolerate additive fields (e.g. the Rust frontend reports its own version). + assert response.json()["version"] == VLLM_VERSION diff --git a/tests/entrypoints/serve/instrumentator/test_basic.py b/tests/entrypoints/serve/instrumentator/test_basic.py index 1ab963dc18..5b00d2e578 100644 --- a/tests/entrypoints/serve/instrumentator/test_basic.py +++ b/tests/entrypoints/serve/instrumentator/test_basic.py @@ -83,7 +83,8 @@ async def test_show_version(server: RemoteOpenAIServer): response = requests.get(server.url_for("version")) response.raise_for_status() - assert response.json() == {"version": VLLM_VERSION} + # Tolerate additive fields (e.g. the Rust frontend reports its own version). + assert response.json()["version"] == VLLM_VERSION @pytest.mark.asyncio