[Perf] Add skip_clone to SamplingParams for internal request handling (#31041)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -211,6 +211,12 @@ class SamplingParams(
|
||||
set to an integer k, will use only the last k tokens from the prompt
|
||||
(i.e., left truncation). If set to `None`, truncation is disabled."""
|
||||
output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
|
||||
skip_clone: bool = False
|
||||
"""Internal flag indicating that this SamplingParams instance is safe to
|
||||
reuse without cloning. When True, clone() will return self without
|
||||
performing a deep copy. This should only be set when the params object
|
||||
is guaranteed to be dedicated to a single request and won't be modified
|
||||
in ways that would affect other uses."""
|
||||
|
||||
# The below fields are not supposed to be used as an input.
|
||||
# They are set in post_init.
|
||||
@@ -270,6 +276,7 @@ class SamplingParams(
|
||||
logit_bias: dict[int, float] | dict[str, float] | None = None,
|
||||
allowed_token_ids: list[int] | None = None,
|
||||
extra_args: dict[str, Any] | None = None,
|
||||
skip_clone: bool = False,
|
||||
) -> "SamplingParams":
|
||||
if logit_bias is not None:
|
||||
# Convert token_id to integer
|
||||
@@ -310,6 +317,7 @@ class SamplingParams(
|
||||
logit_bias=logit_bias,
|
||||
allowed_token_ids=allowed_token_ids,
|
||||
extra_args=extra_args,
|
||||
skip_clone=skip_clone,
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -540,8 +548,13 @@ class SamplingParams(
|
||||
data that is expensive to copy. However, if not copied, the processor
|
||||
needs to support parallel decoding for multiple sequences
|
||||
See https://github.com/vllm-project/vllm/issues/3087
|
||||
|
||||
If skip_clone is True, uses shallow copy instead of deep copy.
|
||||
"""
|
||||
|
||||
if self.skip_clone:
|
||||
return copy.copy(self)
|
||||
|
||||
logit_processor_refs = (
|
||||
None
|
||||
if self.logits_processors is None
|
||||
|
||||
Reference in New Issue
Block a user