[Frontend] Add --spec-method/--spec-model/--spec-tokens CLI aliases (#42476)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -607,6 +607,9 @@ class EngineArgs:
|
|||||||
reasoning_parser_plugin: str | None = None
|
reasoning_parser_plugin: str | None = None
|
||||||
|
|
||||||
speculative_config: dict[str, Any] | None = None
|
speculative_config: dict[str, Any] | None = None
|
||||||
|
spec_method: str | None = None
|
||||||
|
spec_model: str | None = None
|
||||||
|
spec_tokens: int | None = None
|
||||||
|
|
||||||
show_hidden_metrics_for_version: str | None = (
|
show_hidden_metrics_for_version: str | None = (
|
||||||
ObservabilityConfig.show_hidden_metrics_for_version
|
ObservabilityConfig.show_hidden_metrics_for_version
|
||||||
@@ -1435,6 +1438,12 @@ class EngineArgs:
|
|||||||
vllm_group.add_argument(
|
vllm_group.add_argument(
|
||||||
"--speculative-config", "-sc", **vllm_kwargs["speculative_config"]
|
"--speculative-config", "-sc", **vllm_kwargs["speculative_config"]
|
||||||
)
|
)
|
||||||
|
speculative_kwargs = get_kwargs(SpeculativeConfig)
|
||||||
|
vllm_group.add_argument("--spec-method", **speculative_kwargs["method"])
|
||||||
|
vllm_group.add_argument("--spec-model", **speculative_kwargs["model"])
|
||||||
|
vllm_group.add_argument(
|
||||||
|
"--spec-tokens", **speculative_kwargs["num_speculative_tokens"]
|
||||||
|
)
|
||||||
vllm_group.add_argument(
|
vllm_group.add_argument(
|
||||||
"--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
|
"--kv-transfer-config", **vllm_kwargs["kv_transfer_config"]
|
||||||
)
|
)
|
||||||
@@ -1634,6 +1643,21 @@ class EngineArgs:
|
|||||||
"""Initializes and returns a SpeculativeConfig object based on
|
"""Initializes and returns a SpeculativeConfig object based on
|
||||||
`speculative_config`.
|
`speculative_config`.
|
||||||
"""
|
"""
|
||||||
|
for flag, key, value in (
|
||||||
|
("--spec-method", "method", self.spec_method),
|
||||||
|
("--spec-model", "model", self.spec_model),
|
||||||
|
("--spec-tokens", "num_speculative_tokens", self.spec_tokens),
|
||||||
|
):
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
if self.speculative_config is None:
|
||||||
|
self.speculative_config = {}
|
||||||
|
if key in self.speculative_config:
|
||||||
|
raise ValueError(
|
||||||
|
f"{flag} and --speculative-config['{key}'] are mutually exclusive"
|
||||||
|
)
|
||||||
|
self.speculative_config[key] = value
|
||||||
|
|
||||||
if self.speculative_config is None:
|
if self.speculative_config is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -188,6 +188,10 @@ class LLM(PoolingOfflineMixin):
|
|||||||
dictionary or an AttentionConfig instance. If a dictionary, it will
|
dictionary or an AttentionConfig instance. If a dictionary, it will
|
||||||
be converted to an AttentionConfig. Allows specifying the attention
|
be converted to an AttentionConfig. Allows specifying the attention
|
||||||
backend and other attention-related settings.
|
backend and other attention-related settings.
|
||||||
|
spec_method: Top-level alias for `speculative_config["method"]`.
|
||||||
|
spec_model: Top-level alias for `speculative_config["model"]`.
|
||||||
|
spec_tokens: Top-level alias for
|
||||||
|
`speculative_config["num_speculative_tokens"]`.
|
||||||
**kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
|
**kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
|
||||||
|
|
||||||
Note:
|
Note:
|
||||||
@@ -236,6 +240,9 @@ class LLM(PoolingOfflineMixin):
|
|||||||
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
|
compilation_config: int | dict[str, Any] | CompilationConfig | None = None,
|
||||||
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None,
|
quantization_config: dict[str, Any] | QuantizationConfigArgs | None = None,
|
||||||
logits_processors: list[str | type[LogitsProcessor]] | None = None,
|
logits_processors: list[str | type[LogitsProcessor]] | None = None,
|
||||||
|
spec_method: str | None = None,
|
||||||
|
spec_model: str | None = None,
|
||||||
|
spec_tokens: int | None = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""LLM constructor."""
|
"""LLM constructor."""
|
||||||
@@ -357,6 +364,9 @@ class LLM(PoolingOfflineMixin):
|
|||||||
compilation_config=compilation_config_instance,
|
compilation_config=compilation_config_instance,
|
||||||
quantization_config=quantization_config,
|
quantization_config=quantization_config,
|
||||||
logits_processors=logits_processors,
|
logits_processors=logits_processors,
|
||||||
|
spec_method=spec_method,
|
||||||
|
spec_model=spec_model,
|
||||||
|
spec_tokens=spec_tokens,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user