diff --git a/docs/features/custom_arguments.md b/docs/features/custom_arguments.md index 728a2c8990..4b91c58b28 100644 --- a/docs/features/custom_arguments.md +++ b/docs/features/custom_arguments.md @@ -19,7 +19,7 @@ This allows arguments which are not already part of `SamplingParams` to be passe ## Online Custom Arguments -The vLLM REST API allows custom arguments to be passed to the vLLM server via `vllm_xargs`. The example below integrates custom arguments into a vLLM REST API request: +The OpenAI-compatible REST API and Anthropic-compatible `/v1/messages` endpoint allow custom arguments to be passed to the vLLM server via `vllm_xargs`. The example below integrates custom arguments into a vLLM REST API request: ``` bash curl http://localhost:8000/v1/completions \ diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 1491b986a5..d1f178511d 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -182,6 +182,56 @@ class TestImageContentBlocks: } +# ====================================================================== +# vllm_xargs pass-through +# ====================================================================== + + +class TestVllmXargs: + def test_vllm_xargs_passed_through(self): + request = _make_request( + [{"role": "user", "content": "Hello"}], + vllm_xargs={ + "kv_cache_report_mode": "full", + "existing_extension": 7, + }, + ) + + result = _convert(request) + + assert result.vllm_xargs == { + "kv_cache_report_mode": "full", + "existing_extension": 7, + } + + def test_vllm_xargs_reaches_sampling_params_with_kv_transfer(self): + kv_transfer_params = { + "do_remote_decode": True, + "do_remote_prefill": False, + } + request = _make_request( + [{"role": "user", "content": "Hello"}], + vllm_xargs={ + "kv_cache_report_mode": "full", + "existing_extension": "kept", + }, + kv_transfer_params=kv_transfer_params, + ) + + converted = _convert(request) + sampling_params = converted.to_sampling_params( + max_tokens=converted.max_completion_tokens or 0, + default_sampling_params={}, + ) + + assert converted.kv_transfer_params == kv_transfer_params + assert sampling_params.extra_args == { + "kv_cache_report_mode": "full", + "existing_extension": "kept", + "kv_transfer_params": kv_transfer_params, + } + + # ====================================================================== # tool_result content handling # ====================================================================== diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index 02b9356113..18000c4504 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -160,6 +160,13 @@ class AnthropicMessagesRequest(BaseModel): "ECTransfer parameters used for encoder-cache disaggregated serving." ), ) + vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field( + default=None, + description=( + "Additional request parameters with (list of) string or " + "numeric values, used by custom extensions." + ), + ) chat_template_kwargs: dict[str, Any] | None = Field( default=None, description=( diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 7e18fe1ef3..9d6ad63dda 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -489,6 +489,7 @@ class AnthropicServingMessages(OpenAIServingChat): cache_salt=anthropic_request.cache_salt, kv_transfer_params=anthropic_request.kv_transfer_params, ec_transfer_params=anthropic_request.ec_transfer_params, + vllm_xargs=anthropic_request.vllm_xargs, chat_template_kwargs=anthropic_request.chat_template_kwargs, )