[Perf] Add skip_clone to SamplingParams for internal request handling (#31041)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2025-12-24 14:35:57 -08:00
committed by GitHub
parent 09dc7c690c
commit bc5ef333e0
5 changed files with 25 additions and 2 deletions
+13
View File
@@ -211,6 +211,12 @@ class SamplingParams(
set to an integer k, will use only the last k tokens from the prompt
(i.e., left truncation). If set to `None`, truncation is disabled."""
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
skip_clone: bool = False
"""Internal flag indicating that this SamplingParams instance is safe to
reuse without cloning. When True, clone() will return self without
performing a deep copy. This should only be set when the params object
is guaranteed to be dedicated to a single request and won't be modified
in ways that would affect other uses."""
# The below fields are not supposed to be used as an input.
# They are set in post_init.
@@ -270,6 +276,7 @@ class SamplingParams(
logit_bias: dict[int, float] | dict[str, float] | None = None,
allowed_token_ids: list[int] | None = None,
extra_args: dict[str, Any] | None = None,
skip_clone: bool = False,
) -> "SamplingParams":
if logit_bias is not None:
# Convert token_id to integer
@@ -310,6 +317,7 @@ class SamplingParams(
logit_bias=logit_bias,
allowed_token_ids=allowed_token_ids,
extra_args=extra_args,
skip_clone=skip_clone,
)
def __post_init__(self) -> None:
@@ -540,8 +548,13 @@ class SamplingParams(
data that is expensive to copy. However, if not copied, the processor
needs to support parallel decoding for multiple sequences
See https://github.com/vllm-project/vllm/issues/3087
If skip_clone is True, uses shallow copy instead of deep copy.
"""
if self.skip_clone:
return copy.copy(self)
logit_processor_refs = (
None
if self.logits_processors is None