[Frontend] Advertise --enable-x/--disable-x for boolean CLI flags (#61479)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
48f17b77bb
commit
1c4c319792
@@ -230,7 +230,7 @@ are internal constants (no env vars).
|
||||
|
||||
| Flag (`ParallelConfig` field) | Default | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `--enable-shm-tensor-arena` / `--no-enable-shm-tensor-arena` (`enable_shm_tensor_arena`) | **off** | Opt in to route large CPU tensors through the arena (reserves slots in `/dev/shm`). Off = the out-of-band `_reduce_tensor` path only, identical to stock behavior. |
|
||||
| `--enable-shm-tensor-arena` / `--disable-shm-tensor-arena` (`enable_shm_tensor_arena`) | **off** | Opt in to route large CPU tensors through the arena (reserves slots in `/dev/shm`). Off = the out-of-band `_reduce_tensor` path only, identical to stock behavior. |
|
||||
|
||||
Internal constants in `shm_tensor_arena.py`: **8 slots × 256 MB**; tensors larger
|
||||
than a slot, or smaller than the **8 MB** divert threshold, take the out-of-band
|
||||
@@ -268,7 +268,7 @@ than a slot, or smaller than the **8 MB** divert threshold, take the out-of-band
|
||||
(arena + pinning); vision-encoder compute for the same image is ~80 ms.
|
||||
4. **Outstanding — arena vs. out-of-band on `main`.** Re-run the same seeded A/B
|
||||
with the baseline set to current `main` (i.e. `_reduce_tensor` on, arena off
|
||||
via `--no-enable-shm-tensor-arena`) vs. arena on, to quantify the incremental N→1
|
||||
via `--disable-shm-tensor-arena`) vs. arena on, to quantify the incremental N→1
|
||||
copy + pinning gain at TP=4 (and ideally TP=2/8 to show the copy-count scaling).
|
||||
|
||||
## 5. Limitations and future work
|
||||
|
||||
@@ -52,7 +52,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--port 19534 \
|
||||
--gpu-memory-utilization 0.35 \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--disable-prefix-caching \
|
||||
--max-num-batched-tokens 16384 \
|
||||
--allowed-local-media-path /path/to/media \
|
||||
--ec-transfer-config '{
|
||||
@@ -72,7 +72,7 @@ CUDA_VISIBLE_DEVICES=1 vllm serve Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--enable-mm-embeds \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--disable-prefix-caching \
|
||||
--allowed-local-media-path /path/to/media \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
|
||||
@@ -61,7 +61,7 @@ Normally the first serving engine pays for the pass, and on a large MoE model it
|
||||
dominates what is left of the startup time once the weights come from the
|
||||
daemon.
|
||||
|
||||
Unless FlashInfer autotune is disabled (`--no-enable-flashinfer-autotune`),
|
||||
Unless FlashInfer autotune is disabled (`--disable-flashinfer-autotune`),
|
||||
`vllm preload` pays it at preload time: each daemon runs the autotune pass in
|
||||
place — profiling dummy runs against the shard it already holds — after
|
||||
loading and before binding its socket, then writes the cache:
|
||||
|
||||
@@ -355,4 +355,4 @@ vllm serve LiquidAI/d1-3B
|
||||
- A `choice` whose option names are all single letters is labeled with those
|
||||
letters.
|
||||
- When each request carries one question over a state that changes from
|
||||
request to request, `--no-enable-prefix-caching` lowers latency.
|
||||
request to request, `--disable-prefix-caching` lowers latency.
|
||||
|
||||
@@ -39,7 +39,7 @@ Encoder engines should be launched with the following flags:
|
||||
|
||||
- `--enforce-eager` **(required)** – The current EPD implementation is only compatible with encoder instances running in this mode.
|
||||
|
||||
- `--no-enable-prefix-caching` **(required)** – Encoder instances do not consume KV cache; prefix caching is disabled to avoid conflicts with other features.
|
||||
- `--disable-prefix-caching` **(required)** – Encoder instances do not consume KV cache; prefix caching is disabled to avoid conflicts with other features.
|
||||
|
||||
- `--max-num-batched-tokens=<large value>` **(default: 2048)** – This flag controls the token scheduling budget per decoding step and is irrelevant to encoder-only instances. **Set it to a very high value (effectively unlimited) to bypass scheduler limitations.** The actual token budget is managed by the encoder cache manager.
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--enforce-eager \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--disable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
|
||||
@@ -108,7 +108,7 @@ env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--enforce-eager \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--disable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
|
||||
@@ -83,6 +83,51 @@ def test_with_bool_flag(parser):
|
||||
assert args.enable_feature is True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bool_parser():
|
||||
parser = FlexibleArgumentParser()
|
||||
serve = parser.add_subparsers().add_parser("serve")
|
||||
group = serve.add_argument_group("group")
|
||||
group.add_argument(
|
||||
"--enable-feature", "-ef", action=BooleanOptionalAction, default=None
|
||||
)
|
||||
group.add_argument("--disable-logs", action=BooleanOptionalAction, default=False)
|
||||
group.add_argument("--plain", action=BooleanOptionalAction, default=None)
|
||||
return parser, serve
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("args", "expected"),
|
||||
[
|
||||
([], (None, False, None)),
|
||||
(["--enable-feature", "--disable-logs"], (True, True, None)),
|
||||
(["-ef", "--enable-logs"], (True, False, None)),
|
||||
(["--disable-feature"], (False, False, None)),
|
||||
(["--disable_feature"], (False, False, None)),
|
||||
# Hidden backwards-compatible spellings
|
||||
(["--no-enable-feature", "--no-disable-logs"], (False, False, None)),
|
||||
(["--plain"], (None, False, True)),
|
||||
(["--no-plain"], (None, False, False)),
|
||||
],
|
||||
)
|
||||
def test_enable_disable_flags(bool_parser, args, expected):
|
||||
"""`--enable-x`/`--disable-x` are inverses, and `--no-*` still works."""
|
||||
parser, _ = bool_parser
|
||||
ns = parser.parse_args(["serve", *args])
|
||||
assert (ns.enable_feature, ns.disable_logs, ns.plain) == expected
|
||||
|
||||
|
||||
def test_enable_disable_flags_help(bool_parser):
|
||||
"""Help advertises `--enable-x`/`--disable-x` but not `--no-enable-x`."""
|
||||
_, serve = bool_parser
|
||||
usage = serve.format_usage()
|
||||
assert "[--enable-feature | -ef | --disable-feature]" in usage
|
||||
assert "[--disable-logs | --enable-logs]" in usage
|
||||
assert "[--plain | --no-plain]" in usage
|
||||
assert "--no-enable-feature" not in usage
|
||||
assert "--no-disable-logs" not in usage
|
||||
|
||||
|
||||
def test_invalid_choice(parser):
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["--image_input_type", "invalid_choice"])
|
||||
|
||||
@@ -67,7 +67,7 @@ class PoolerConfig:
|
||||
"""
|
||||
Whether the engine-side late-interaction scorer may use the fused
|
||||
flash-maxsim Triton kernel. Disabled automatically when the API server
|
||||
is started with `--no-enable-flash-late-interaction`; the reference
|
||||
is started with `--disable-flash-late-interaction`; the reference
|
||||
scorer is used instead.
|
||||
"""
|
||||
|
||||
|
||||
+5
-5
@@ -2177,10 +2177,10 @@ class VllmConfig:
|
||||
self._verify_aux_output_compatibility()
|
||||
|
||||
# Hybrid KV cache manager (HMA) runtime rules:
|
||||
# - Explicit enable (--no-disable-kv-cache-manager): error if runtime
|
||||
# - Explicit enable (--enable-hybrid-kv-cache-manager): error if runtime
|
||||
# disables it
|
||||
# - No preference: auto-disable for unsupported features or connector configs
|
||||
# - Explicit disable (--disable-kv-cache-manager): always respect it
|
||||
# - Explicit disable (--disable-hybrid-kv-cache-manager): always respect it
|
||||
need_disable_hybrid_kv_cache_manager = False
|
||||
# logger should only print warning message for hybrid models. As we
|
||||
# can't know whether the model is hybrid or not now, so we don't log
|
||||
@@ -2241,7 +2241,7 @@ class VllmConfig:
|
||||
raise ValueError(
|
||||
"Hybrid KV cache manager was explicitly enabled but is not "
|
||||
"supported in this configuration. Consider omitting the "
|
||||
"--no-disable-hybrid-kv-cache-manager flag to let vLLM decide"
|
||||
"--enable-hybrid-kv-cache-manager flag to let vLLM decide"
|
||||
" automatically."
|
||||
)
|
||||
|
||||
@@ -3410,7 +3410,7 @@ class VllmConfig:
|
||||
if self.cache_config.mamba_cache_mode != "none":
|
||||
raise ValueError(
|
||||
"FlashInfer ReplaySSM speculative decoding does not support "
|
||||
"prefix caching; use --no-enable-prefix-caching"
|
||||
"prefix caching; use --disable-prefix-caching"
|
||||
)
|
||||
query_len = 1 + self.num_speculative_tokens
|
||||
if self.cache_config.replayssm_buffer_len < query_len:
|
||||
@@ -3429,7 +3429,7 @@ class VllmConfig:
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
raise ValueError(
|
||||
"FlashInfer ReplaySSM does not support prefix caching yet; "
|
||||
"use --no-enable-prefix-caching"
|
||||
"use --disable-prefix-caching"
|
||||
)
|
||||
elif self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError(
|
||||
|
||||
@@ -2531,7 +2531,7 @@ class NixlBaseConnectorWorker:
|
||||
"is not supported for Mamba hybrid models. "
|
||||
f"Local: {self._physical_blocks_per_logical_kv_block}, "
|
||||
f"Remote: {remote_physical_per_logical}. "
|
||||
"Disable prefix caching with --no-enable-prefix-caching."
|
||||
"Disable prefix caching with --disable-prefix-caching."
|
||||
)
|
||||
|
||||
if block_size_ratio != 1:
|
||||
|
||||
@@ -557,7 +557,7 @@ def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||
|
||||
|
||||
def propagate_flash_late_interaction(args, engine_args) -> None:
|
||||
"""Propagate `--no-enable-flash-late-interaction` into the engine config.
|
||||
"""Propagate `--disable-flash-late-interaction` into the engine config.
|
||||
|
||||
The frontend flag alone only disables the API-server scoring path;
|
||||
mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the
|
||||
|
||||
@@ -11,6 +11,7 @@ from argparse import (
|
||||
ArgumentDefaultsHelpFormatter,
|
||||
ArgumentParser,
|
||||
ArgumentTypeError,
|
||||
BooleanOptionalAction,
|
||||
Namespace,
|
||||
RawDescriptionHelpFormatter,
|
||||
_ArgumentGroup,
|
||||
@@ -116,6 +117,42 @@ class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpForma
|
||||
super().add_arguments(actions)
|
||||
|
||||
|
||||
class EnableDisableAction(BooleanOptionalAction):
|
||||
"""`BooleanOptionalAction` that pairs `--enable-x` with `--disable-x`.
|
||||
|
||||
`--no-enable-x` and `--no-disable-x` are still accepted for backwards
|
||||
compatibility, but are hidden from help.
|
||||
"""
|
||||
|
||||
_inverse_prefixes = {"--enable-": "--disable-", "--disable-": "--enable-"}
|
||||
|
||||
def __init__(self, option_strings, dest, *args, **kwargs):
|
||||
super().__init__(option_strings, dest, *args, **kwargs)
|
||||
inverses = [
|
||||
inverse + opt.removeprefix(prefix)
|
||||
for opt in option_strings
|
||||
for prefix, inverse in self._inverse_prefixes.items()
|
||||
if opt.startswith(prefix)
|
||||
]
|
||||
negated = {opt for opt in self.option_strings if opt not in option_strings}
|
||||
self.negative_option_strings = negated | set(inverses)
|
||||
self.visible_option_strings = (
|
||||
[*option_strings, *inverses] if inverses else list(self.option_strings)
|
||||
)
|
||||
self.option_strings = [*self.option_strings, *inverses]
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
setattr(namespace, self.dest, option_string not in self.negative_option_strings)
|
||||
|
||||
|
||||
def _register_action(action: Action, deprecated: bool, deprecated_actions: set):
|
||||
# Hidden option strings stay registered with the parser but leave help.
|
||||
if isinstance(action, EnableDisableAction):
|
||||
action.option_strings = action.visible_option_strings
|
||||
if deprecated:
|
||||
deprecated_actions.add(action)
|
||||
|
||||
|
||||
class FlexibleArgumentParser(ArgumentParser):
|
||||
"""ArgumentParser that allows both underscore and dash in names."""
|
||||
|
||||
@@ -139,6 +176,7 @@ class FlexibleArgumentParser(ArgumentParser):
|
||||
# Pop kwarg "add_json_tip" to control whether to add the JSON tip
|
||||
self.add_json_tip = kwargs.pop("add_json_tip", True)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.register("action", BooleanOptionalAction, EnableDisableAction)
|
||||
self._show_serve_task_hint = False
|
||||
|
||||
def error(self, message: str) -> NoReturn:
|
||||
@@ -169,29 +207,28 @@ class FlexibleArgumentParser(ArgumentParser):
|
||||
logger.warning_once("argument '%s' is deprecated", dest)
|
||||
return namespace, args
|
||||
|
||||
def add_argument(self, *args, **kwargs):
|
||||
# Python 3.13+ handles the deprecated kwarg natively
|
||||
deprecated = sys.version_info < (3, 13) and kwargs.pop("deprecated", False)
|
||||
action = super().add_argument(*args, **kwargs)
|
||||
_register_action(action, deprecated, self._deprecated)
|
||||
return action
|
||||
|
||||
class _FlexibleArgumentGroup(_ArgumentGroup):
|
||||
def __init__(self, container, *args, **kwargs):
|
||||
super().__init__(container, *args, **kwargs)
|
||||
self._deprecated: set[Action] = container._deprecated
|
||||
|
||||
def add_argument(self, *args, **kwargs):
|
||||
deprecated = kwargs.pop("deprecated", False)
|
||||
deprecated = sys.version_info < (3, 13) and kwargs.pop("deprecated", False)
|
||||
action = super().add_argument(*args, **kwargs)
|
||||
if deprecated:
|
||||
self._deprecated.add(action)
|
||||
_register_action(action, deprecated, self._deprecated)
|
||||
return action
|
||||
|
||||
class _FlexibleArgumentGroup(_ArgumentGroup):
|
||||
def __init__(self, container, *args, **kwargs):
|
||||
super().__init__(container, *args, **kwargs)
|
||||
self._deprecated: set[Action] = container._deprecated
|
||||
|
||||
def add_argument(self, *args, **kwargs):
|
||||
deprecated = kwargs.pop("deprecated", False)
|
||||
action = super().add_argument(*args, **kwargs)
|
||||
if deprecated:
|
||||
self._deprecated.add(action)
|
||||
return action
|
||||
|
||||
def add_argument_group(self, *args, **kwargs):
|
||||
group = self._FlexibleArgumentGroup(self, *args, **kwargs)
|
||||
self._action_groups.append(group)
|
||||
return group
|
||||
def add_argument_group(self, *args, **kwargs):
|
||||
group = self._FlexibleArgumentGroup(self, *args, **kwargs)
|
||||
self._action_groups.append(group)
|
||||
return group
|
||||
|
||||
def format_help(self):
|
||||
# Only use custom help formatting for bottom level parsers
|
||||
|
||||
Reference in New Issue
Block a user