From 79651d6085885cf2842baf85526ece7afa87c87d Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Wed, 26 Aug 2026 23:58:23 -0700 Subject: [PATCH] [Tools][Recipes] Improve sweep recommendations and short-alias parsing (#53946) Signed-off-by: louie-tsai --- docs/models/hardware_supported_models/cpu.md | 8 +- tools/recipes/README.md | 4 +- tools/recipes/RUNTIME_TUNING.md | 10 +- tools/recipes/SWEEP_TUNING.md | 33 ++- tools/recipes/recipe_json_to_vllm_config.py | 125 ++++++++++- tools/recipes/sweep_generation.py | 29 ++- tools/recipes/sweep_recommendation.py | 217 +++++++++++++++---- 7 files changed, 352 insertions(+), 74 deletions(-) diff --git a/docs/models/hardware_supported_models/cpu.md b/docs/models/hardware_supported_models/cpu.md index 6be3c3bd3d..10ca8caa0e 100644 --- a/docs/models/hardware_supported_models/cpu.md +++ b/docs/models/hardware_supported_models/cpu.md @@ -30,7 +30,7 @@ vllm serve --config config.yaml | Model | Architecture | Supported | Recipe | | ------------------------------------ | ---------------------------------------- | --------- | ------ | -| openai/gpt-oss-20b | GptOssForCausalLM | ✅ | — | +| openai/gpt-oss-20b | GptOssForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/openai/gpt-oss-20b?hardware=xeon6) | | meta-llama/Llama-3.1-8B | LlamaForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/meta-llama/Llama-3.1-8B?hardware=xeon6) | | meta-llama/Llama-3.1-8B-Instruct | LlamaForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/meta-llama/Llama-3.1-8B-Instruct?hardware=xeon6) | | meta-llama/Llama-3.2-1B | LlamaForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/meta-llama/Llama-3.2-1B?hardware=xeon6) | @@ -48,13 +48,13 @@ vllm serve --config config.yaml | TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ | LlamaForCausalLM | ✅ | — | | TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ | LlamaForCausalLM | ✅ | — | | ibm-granite/granite-3.2-2b-instruct | GraniteForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/ibm-granite/granite-3.2-2b-instruct?hardware=xeon6) | -| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ | — | +| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-1.7B?hardware=xeon6) | | Qwen/Qwen3-4B | Qwen3ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-4B?hardware=xeon6) | -| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ | — | +| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-8B?hardware=xeon6) | | Qwen/Qwen3-14B | Qwen3ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-14B?hardware=xeon6) | | Qwen/Qwen3-14B-FP8 | Qwen3ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-14B?hardware=xeon6&variant=fp8) | | Qwen/Qwen3-14B-AWQ | Qwen3ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-14B?hardware=xeon6&variant=awq) | -| Qwen/Qwen3-30B-A3B | Qwen3MoeForCausalLM | ✅ | — | +| Qwen/Qwen3-30B-A3B | Qwen3MoeForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/Qwen3-30B-A3B?hardware=xeon6) | | Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 | Qwen3MoeForCausalLM | ✅ | — | | Qwen/QwQ-32B | Qwen2ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/QwQ-32B?hardware=xeon6) | | Qwen/QwQ-32B-AWQ | Qwen2ForCausalLM | ✅ | [Xeon 6](https://recipes.vllm.ai/Qwen/QwQ-32B?hardware=xeon6&variant=awq) | diff --git a/tools/recipes/README.md b/tools/recipes/README.md index e1ff3e5549..058470167a 100644 --- a/tools/recipes/README.md +++ b/tools/recipes/README.md @@ -87,7 +87,7 @@ capacity objectives. python3 tools/recipes/recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct \ --hardware xeon6 \ - --input-tokens 2048 \ + --input-tokens 128 \ --output-tokens 128 \ --concurrency 32 \ --ttft-sla-ms 3000 \ @@ -111,7 +111,7 @@ python3 tools/recipes/recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct \ --hardware xeon6 \ --detect-hardware \ - --input-tokens 2048 \ + --input-tokens 128 \ --output-tokens 128 \ --concurrency 32 \ --generate-sweep diff --git a/tools/recipes/RUNTIME_TUNING.md b/tools/recipes/RUNTIME_TUNING.md index 43031b7d2a..d3a359446c 100644 --- a/tools/recipes/RUNTIME_TUNING.md +++ b/tools/recipes/RUNTIME_TUNING.md @@ -57,7 +57,7 @@ Example: python3 tools/recipes/recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct \ --hardware xeon6 \ - --input-tokens 2048 \ + --input-tokens 128 \ --output-tokens 128 \ --concurrency 32 \ --ttft-sla-ms 3000 \ @@ -172,13 +172,13 @@ max-num-batched-tokens = max( ) ``` -For example, with 2048 input tokens, 128 output tokens, and concurrency 32: +For example, with 128 input tokens, 128 output tokens, and concurrency 32: ```text decode_budget = 32 prefills_per_step = max(1, 32 / 128) = 1 -prefill_budget = 2048 -candidate = max(2048, 32, 32 + 2048) = 2080 +prefill_budget = 128 +candidate = max(2048, 32, 32 + 128) = 2048 ``` If chunked prefill is explicitly disabled and `max-model-len` is available, the @@ -261,7 +261,7 @@ python3 /recipes/recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct \ --hardware xeon6 \ --detect-hardware \ - --input-tokens 2048 \ + --input-tokens 128 \ --output-tokens 128 \ --concurrency 32 \ --ttft-sla-ms 3000 \ diff --git a/tools/recipes/SWEEP_TUNING.md b/tools/recipes/SWEEP_TUNING.md index 7842c8aff5..e7282e122d 100644 --- a/tools/recipes/SWEEP_TUNING.md +++ b/tools/recipes/SWEEP_TUNING.md @@ -11,7 +11,7 @@ python3 tools/recipes/recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct \ --hardware xeon6 \ --detect-hardware \ - --input-tokens 2048 \ + --input-tokens 128 \ --output-tokens 128 \ --concurrency 32 \ --ttft-sla-ms 3000 \ @@ -30,8 +30,11 @@ sweep/ └── SWEEP.md ``` -The sweep currently varies only `max-num-seqs` and -`max-num-batched-tokens`. +The sweep varies only `max-num-seqs` and `max-num-batched-tokens`. It uses an +eight-point directed design: a batch-budget curve at the initial sequence count +plus lower/higher batch interactions at three-quarters and one-half of that +count. This gives broader coverage than a one-parameter-at-a-time sweep without +the cost of a full Cartesian grid. ## Run and Recommend @@ -54,13 +57,23 @@ sweep/recommended-config.yml sweep/recommendation.json ``` -With TTFT/TPOT objectives, the benchmark uses vLLM `--goodput`, and the -recommender selects highest mean request goodput across repeated runs. Without -latency objectives it selects highest mean output-token throughput. Failed -benchmark configurations are excluded. +With TTFT/TPOT objectives, the benchmark uses vLLM `--goodput`. The recommender +calculates duration-weighted combined compliance across repeated runs and +requires both median P99 latency objectives and the minimum compliance ratio +(default `0.99`). Among eligible configurations it selects the highest mean +output-token throughput. Use `recommend.py --minimum-compliance VALUE` to +change the compliance requirement. -`recommended-config.yml` changes only the two swept scheduler parameters. -`recommendation.json` records the measured evidence. +If no configuration is eligible, `recommend.py` records the highest-goodput +configuration as `best_effort` in `recommendation.json`, does not write a +deployable `recommended-config.yml`, and exits with status 2. Without latency +objectives it selects highest mean output-token throughput. Failed benchmark +configurations are excluded in either mode. + +When an eligible configuration exists, `recommended-config.yml` changes only +the two swept scheduler parameters. `recommendation.json` records mean, median, +and worst-run P99 values, combined compliance, and the measured evidence for +every candidate. Deploy: @@ -83,7 +96,7 @@ python3 /recipes/recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct \ --hardware xeon6 \ --detect-hardware \ - --input-tokens 2048 \ + --input-tokens 128 \ --output-tokens 128 \ --concurrency 32 \ --ttft-sla-ms 3000 \ diff --git a/tools/recipes/recipe_json_to_vllm_config.py b/tools/recipes/recipe_json_to_vllm_config.py index 099b5a03c4..5fd7b7729d 100644 --- a/tools/recipes/recipe_json_to_vllm_config.py +++ b/tools/recipes/recipe_json_to_vllm_config.py @@ -59,10 +59,36 @@ except ImportError as exc: DEFAULT_API_BASE = "https://recipes.vllm.ai" -SHORT_ALIASES = { - "-tp": "tensor-parallel-size", +VALUE_SHORT_ALIASES = { + "-q": "quantization", "-pp": "pipeline-parallel-size", + "-n": "nnodes", + "-r": "node-rank", + "-tp": "tensor-parallel-size", + "-dcp": "decode-context-parallel-size", + "-pcp": "prefill-context-parallel-size", "-dp": "data-parallel-size", + "-dpn": "data-parallel-rank", + "-dpr": "data-parallel-start-rank", + "-dpl": "data-parallel-size-local", + "-dpa": "data-parallel-address", + "-dpp": "data-parallel-rpc-port", + "-dpb": "data-parallel-backend", + "-asc": "api-server-count", +} + +FLAG_SHORT_ALIASES = { + "-dph": "data-parallel-hybrid-lb", + "-dpe": "data-parallel-external-lb", + "-dpm": "data-parallel-multi-port-external-lb", + "-ep": "enable-expert-parallel", +} + +DICT_SHORT_ALIASES = { + "-sc": "speculative-config", + "-dc": "diffusion-config", + "-cc": "compilation-config", + "-ac": "attention-config", } @@ -464,6 +490,11 @@ def discover_recipe_source( def coerce(value: str) -> Any: """Convert CLI string values to useful YAML scalar/object types.""" + # Recipes may preserve Python-style booleans in dotted config aliases, + # for example: -cc.pass_config.fuse_rope_kvcache=True. + if value in ("True", "False"): + return value == "True" + try: return json.loads(value) except (json.JSONDecodeError, TypeError): @@ -473,8 +504,19 @@ def coerce(value: str) -> Any: def is_option_token(token: str) -> bool: if token.startswith("--"): return True - if token in SHORT_ALIASES or token == "-O": + + short_name = token.split("=", 1)[0] + if ( + short_name in VALUE_SHORT_ALIASES + or short_name in FLAG_SHORT_ALIASES + or short_name in DICT_SHORT_ALIASES + or token == "-O" + ): return True + + if any(token.startswith(f"{alias}.") for alias in DICT_SHORT_ALIASES): + return True + return token.startswith("-O") and len(token) > 2 @@ -554,22 +596,85 @@ def argv_to_config(argv: list[Any]) -> dict[str, Any]: i += 2 continue - # Selected common short aliases. - if token in SHORT_ALIASES: - if i + 1 >= len(argv): - raise ValueError(f"{token} is missing its value") + # Dotted dictionary aliases accepted by FlexibleArgumentParser: + # -cc.mode=3 + # -cc.mode 3 + # -cc.pass_config.foo=True + # -cc.custom_ops+ -quant_fp8 + # The same syntax applies to -sc, -dc, and -ac. + dotted_alias = next( + (alias for alias in DICT_SHORT_ALIASES if token.startswith(f"{alias}.")), + None, + ) + if dotted_alias is not None: + dotted = token[len(dotted_alias) + 1 :] + raw_key, separator, raw_value = dotted.partition("=") + + append = raw_key.endswith("+") + if append: + raw_key = raw_key[:-1] + + if not raw_key: + raise ValueError(f"{token} must contain a key after {dotted_alias}.") + + if not separator: + if i + 1 >= len(argv): + raise ValueError(f"{token} is missing its value") + # Consume the next token unconditionally. Dotted list values may + # themselves begin with "-", for example "-quant_fp8". + raw_value = argv[i + 1] + i += 2 + else: + i += 1 + + value: Any = raw_value.split(",") if append else coerce(raw_value) + merge_value( config, - [SHORT_ALIASES[token]], - coerce(argv[i + 1]), + [DICT_SHORT_ALIASES[dotted_alias], *raw_key.split(".")], + value, ) + continue + + # Support -alias=value for value-bearing and whole-dict aliases. + if "=" in token: + short_name, raw_value = token.split("=", 1) + if short_name in VALUE_SHORT_ALIASES: + merge_value( + config, + [VALUE_SHORT_ALIASES[short_name]], + coerce(raw_value), + ) + i += 1 + continue + if short_name in DICT_SHORT_ALIASES: + merge_value( + config, + [DICT_SHORT_ALIASES[short_name]], + coerce(raw_value), + ) + i += 1 + continue + + # Boolean short aliases are flags and do not consume a value. + if token in FLAG_SHORT_ALIASES: + merge_value(config, [FLAG_SHORT_ALIASES[token]], True) + i += 1 + continue + + # Scalar and whole-dictionary short aliases consume one value. + if token in VALUE_SHORT_ALIASES or token in DICT_SHORT_ALIASES: + if i + 1 >= len(argv): + raise ValueError(f"{token} is missing its value") + key = VALUE_SHORT_ALIASES.get(token) or DICT_SHORT_ALIASES[token] + merge_value(config, [key], coerce(argv[i + 1])) i += 2 continue if not token.startswith("--"): raise ValueError( f"Unexpected positional/short argument {token!r}. " - "The converter expects Recipes to emit long-form vLLM serve options." + "The converter accepts vLLM serve long options and known short aliases." ) # --key=value diff --git a/tools/recipes/sweep_generation.py b/tools/recipes/sweep_generation.py index 5758711e5e..33a5244a72 100644 --- a/tools/recipes/sweep_generation.py +++ b/tools/recipes/sweep_generation.py @@ -43,7 +43,13 @@ def validate_sweep_workload(workload: WorkloadHints) -> None: def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]: - """Build a small directed sweep around the single initial suggestion.""" + """Build a diverse, bounded sweep around the initial suggestion. + + The initial five-point sweep changed only one parameter at a time and + skipped the useful middle sequence count. Eight directed points cover the + batch-budget curve at full concurrency plus interactions at 3/4 and 1/2 of + the initial scheduler concurrency without paying for a full Cartesian grid. + """ initial_seqs = _positive_int(config, "max-num-seqs") initial_batch = _positive_int(config, "max-num-batched-tokens") @@ -58,10 +64,15 @@ def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]: minimum_batch = max(minimum_batch, max_model_len) lower_seqs = max(1, (initial_seqs + 1) // 2) + middle_seqs = max(lower_seqs, (3 * initial_seqs + 3) // 4) lower_batch = max( minimum_batch, _strict_lower_power_of_two(initial_batch), ) + smaller_batch = max( + minimum_batch, + _strict_lower_power_of_two(lower_batch), + ) higher_batch = max( minimum_batch, _strict_upper_power_of_two(initial_batch), @@ -90,10 +101,13 @@ def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]: # Keep the exact initial suggestion as the measured baseline. Additional # values exist only in the optional sweep package. add("initial", initial_seqs, initial_batch) - add("lower_scheduler_concurrency", lower_seqs, initial_batch) + add("smaller_batch_budget", initial_seqs, smaller_batch) add("lower_batch_budget", initial_seqs, lower_batch) - add("latency_focused", lower_seqs, lower_batch) add("higher_batch_budget", initial_seqs, higher_batch) + add("middle_seqs_lower_batch", middle_seqs, lower_batch) + add("middle_seqs_higher_batch", middle_seqs, higher_batch) + add("lower_seqs_lower_batch", lower_seqs, lower_batch) + add("lower_seqs_higher_batch", lower_seqs, higher_batch) return candidates @@ -286,9 +300,12 @@ recommended-config.yml recommendation.json ``` -With TTFT/TPOT objectives it selects highest mean request goodput across -repeated runs. Without latency objectives it selects highest mean output-token -throughput. Configurations with failed requests are excluded. +With TTFT/TPOT objectives it requires duration-weighted combined compliance of +at least 99% plus median P99 TTFT/TPOT compliance, then selects highest mean +output-token throughput. If no candidate qualifies, it records a best-effort +candidate but does not write `recommended-config.yml`. Without latency +objectives it selects highest mean output-token throughput. Configurations with +failed requests are excluded. `recommended-config.yml` copies the initial configuration and changes only `max-num-seqs` and `max-num-batched-tokens`. diff --git a/tools/recipes/sweep_recommendation.py b/tools/recipes/sweep_recommendation.py index a49b02849a..70efaad625 100755 --- a/tools/recipes/sweep_recommendation.py +++ b/tools/recipes/sweep_recommendation.py @@ -10,7 +10,7 @@ import argparse import json from collections import defaultdict from pathlib import Path -from statistics import mean +from statistics import mean, median from typing import Any try: @@ -23,6 +23,7 @@ DEFAULT_CONFIG_PATH: str | None = None DEFAULT_ENV_PATH: str | None = None DEFAULT_TTFT_SLA_MS: float | None = None DEFAULT_TPOT_SLA_MS: float | None = None +DEFAULT_MINIMUM_COMPLIANCE: float = 0.99 def _number(value: object) -> float | None: @@ -37,6 +38,16 @@ def _mean(rows: list[dict[str, Any]], key: str) -> float | None: return mean(numbers) if numbers else None +def _percentile_summary( + rows: list[dict[str, Any]], key: str +) -> tuple[float | None, float | None, float | None]: + values = [_number(row.get(key)) for row in rows] + numbers = [value for value in values if value is not None] + if not numbers: + return None, None, None + return mean(numbers), median(numbers), max(numbers) + + def _positive_int(row: dict[str, Any], key: str) -> int | None: value = row.get(key) if isinstance(value, bool) or not isinstance(value, int) or value <= 0: @@ -69,6 +80,9 @@ def _aggregate_candidates( rows: list[dict[str, Any]], *, use_goodput: bool, + ttft_sla_ms: float | None = None, + tpot_sla_ms: float | None = None, + minimum_compliance: float = DEFAULT_MINIMUM_COMPLIANCE, ) -> list[dict[str, Any]]: grouped: dict[tuple[int, int], list[dict[str, Any]]] = defaultdict(list) @@ -83,9 +97,28 @@ def _aggregate_candidates( for (seqs, batch), runs in sorted(grouped.items()): failed_requests = sum(int(_number(run.get("failed")) or 0) for run in runs) output_throughput = _mean(runs, "output_throughput") + request_throughput = _mean(runs, "request_throughput") request_goodput = _mean(runs, "request_goodput") - p99_ttft_ms = _mean(runs, "p99_ttft_ms") - p99_tpot_ms = _mean(runs, "p99_tpot_ms") + mean_p99_ttft_ms, median_p99_ttft_ms, worst_p99_ttft_ms = _percentile_summary( + runs, "p99_ttft_ms" + ) + mean_p99_tpot_ms, median_p99_tpot_ms, worst_p99_tpot_ms = _percentile_summary( + runs, "p99_tpot_ms" + ) + + completed_requests = sum( + int(_number(run.get("completed")) or 0) for run in runs + ) + estimated_compliant_requests = sum( + (_number(run.get("request_goodput")) or 0.0) + * (_number(run.get("duration")) or 0.0) + for run in runs + ) + combined_compliance_ratio = ( + estimated_compliant_requests / completed_requests + if completed_requests > 0 + else None + ) valid = failed_requests == 0 and output_throughput is not None reason = None @@ -96,6 +129,25 @@ def _aggregate_candidates( valid = False reason = "request_goodput is missing; rerun sweep with SLO goodput enabled" + p99_sla_eligible = valid + if ttft_sla_ms is not None: + p99_sla_eligible = bool( + p99_sla_eligible + and median_p99_ttft_ms is not None + and median_p99_ttft_ms <= ttft_sla_ms + ) + if tpot_sla_ms is not None: + p99_sla_eligible = bool( + p99_sla_eligible + and median_p99_tpot_ms is not None + and median_p99_tpot_ms <= tpot_sla_ms + ) + compliance_eligible = bool( + valid + and combined_compliance_ratio is not None + and combined_compliance_ratio >= minimum_compliance + ) + candidates.append( { "max_num_seqs": seqs, @@ -103,9 +155,26 @@ def _aggregate_candidates( "run_count": len(runs), "failed_requests": failed_requests, "mean_request_goodput": request_goodput, + "mean_request_throughput": request_throughput, "mean_output_throughput": output_throughput, - "mean_p99_ttft_ms": p99_ttft_ms, - "mean_p99_tpot_ms": p99_tpot_ms, + "mean_p99_ttft_ms": mean_p99_ttft_ms, + "median_p99_ttft_ms": median_p99_ttft_ms, + "worst_p99_ttft_ms": worst_p99_ttft_ms, + "mean_p99_tpot_ms": mean_p99_tpot_ms, + "median_p99_tpot_ms": median_p99_tpot_ms, + "worst_p99_tpot_ms": worst_p99_tpot_ms, + "estimated_compliant_requests": estimated_compliant_requests, + "completed_requests": completed_requests, + "combined_compliance_ratio": combined_compliance_ratio, + "combined_compliance_percent": ( + combined_compliance_ratio * 100 + if combined_compliance_ratio is not None + else None + ), + "benchmark_valid": valid, + "p99_sla_eligible": p99_sla_eligible, + "compliance_eligible": compliance_eligible, + "sla_eligible": p99_sla_eligible and compliance_eligible, "valid": valid, "invalid_reason": reason, "summary_files": sorted({str(run["_summary_path"]) for run in runs}), @@ -119,7 +188,7 @@ def _select_candidate( candidates: list[dict[str, Any]], *, use_goodput: bool, -) -> tuple[dict[str, Any], str]: +) -> tuple[dict[str, Any] | None, dict[str, Any], str]: valid = [candidate for candidate in candidates if candidate["valid"]] if not valid: raise ValueError( @@ -128,21 +197,37 @@ def _select_candidate( ) if use_goodput: - objective = "highest_mean_request_goodput" - winner = max( + objective = "p99_constrained_throughput" + best_effort = max( valid, key=lambda candidate: ( candidate["mean_request_goodput"], + candidate["combined_compliance_ratio"] or 0.0, candidate["mean_output_throughput"], -candidate["max_num_batched_tokens"], -candidate["max_num_seqs"], ), ) - if winner["mean_request_goodput"] <= 0: + if best_effort["mean_request_goodput"] <= 0: raise ValueError( "No sweep configuration produced non-zero request goodput for " "the supplied TTFT/TPOT objectives." ) + eligible = [candidate for candidate in valid if candidate["sla_eligible"]] + winner = ( + max( + eligible, + key=lambda candidate: ( + candidate["mean_output_throughput"], + candidate["combined_compliance_ratio"], + candidate["mean_request_goodput"], + -candidate["max_num_batched_tokens"], + -candidate["max_num_seqs"], + ), + ) + if eligible + else None + ) else: objective = "highest_mean_output_throughput" winner = max( @@ -153,8 +238,9 @@ def _select_candidate( -candidate["max_num_seqs"], ), ) + best_effort = winner - return winner, objective + return winner, best_effort, objective def _load_config(path: Path) -> dict[str, Any]: @@ -219,6 +305,15 @@ def parse_args() -> argparse.Namespace: default=DEFAULT_TPOT_SLA_MS, help="TPOT objective used to generate request goodput.", ) + parser.add_argument( + "--minimum-compliance", + type=float, + default=DEFAULT_MINIMUM_COMPLIANCE, + help=( + "Minimum duration-weighted fraction of completed requests that must " + "meet all supplied objectives (default: 0.99)." + ), + ) parser.add_argument( "--output-config", default="recommended-config.yml", @@ -234,6 +329,8 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() + if not 0.0 < args.minimum_compliance <= 1.0: + raise ValueError("--minimum-compliance must be greater than 0 and at most 1.") script_dir = Path(__file__).resolve().parent config_path = _resolve(script_dir, args.config) @@ -258,23 +355,64 @@ def main() -> int: raise ValueError(f"No summary.json sweep results found under {results_dir}.") use_goodput = args.ttft_sla_ms is not None or args.tpot_sla_ms is not None - candidates = _aggregate_candidates(rows, use_goodput=use_goodput) - winner, objective = _select_candidate(candidates, use_goodput=use_goodput) - - initial_config = _load_config(config_path) - recommended_config = dict(initial_config) - recommended_config["max-num-seqs"] = winner["max_num_seqs"] - recommended_config["max-num-batched-tokens"] = winner["max_num_batched_tokens"] - - _write_config( - output_config, - source_path=config_path, - config=recommended_config, - objective=objective, + candidates = _aggregate_candidates( + rows, + use_goodput=use_goodput, + ttft_sla_ms=args.ttft_sla_ms, + tpot_sla_ms=args.tpot_sla_ms, + minimum_compliance=args.minimum_compliance, + ) + winner, best_effort, objective = _select_candidate( + candidates, use_goodput=use_goodput ) + initial_config = _load_config(config_path) + if winner is not None: + recommended_config = dict(initial_config) + recommended_config["max-num-seqs"] = winner["max_num_seqs"] + recommended_config["max-num-batched-tokens"] = winner["max_num_batched_tokens"] + _write_config( + output_config, + source_path=config_path, + config=recommended_config, + objective=objective, + ) + elif output_config.exists(): + output_config.unlink() + + measured = None + recommended = None + if winner is not None: + recommended = { + "max_num_seqs": winner["max_num_seqs"], + "max_num_batched_tokens": winner["max_num_batched_tokens"], + } + measured = { + key: winner[key] + for key in ( + "run_count", + "mean_request_throughput", + "mean_request_goodput", + "mean_output_throughput", + "combined_compliance_ratio", + "combined_compliance_percent", + "estimated_compliant_requests", + "completed_requests", + "mean_p99_ttft_ms", + "median_p99_ttft_ms", + "worst_p99_ttft_ms", + "mean_p99_tpot_ms", + "median_p99_tpot_ms", + "worst_p99_tpot_ms", + ) + } + recommendation = { + "status": ( + "sla_feasible" if winner is not None else "no_sla_feasible_configuration" + ), "selection_objective": objective, + "minimum_compliance_ratio": args.minimum_compliance, "slo": { "ttft_ms": args.ttft_sla_ms, "tpot_ms": args.tpot_sla_ms, @@ -283,16 +421,14 @@ def main() -> int: "max_num_seqs": initial_config.get("max-num-seqs"), "max_num_batched_tokens": initial_config.get("max-num-batched-tokens"), }, - "recommended": { - "max_num_seqs": winner["max_num_seqs"], - "max_num_batched_tokens": winner["max_num_batched_tokens"], - }, - "measured": { - "run_count": winner["run_count"], - "mean_request_goodput": winner["mean_request_goodput"], - "mean_output_throughput": winner["mean_output_throughput"], - "mean_p99_ttft_ms": winner["mean_p99_ttft_ms"], - "mean_p99_tpot_ms": winner["mean_p99_tpot_ms"], + "recommended": recommended, + "measured": measured, + "best_effort": { + "max_num_seqs": best_effort["max_num_seqs"], + "max_num_batched_tokens": best_effort["max_num_batched_tokens"], + "mean_request_goodput": best_effort["mean_request_goodput"], + "combined_compliance_ratio": best_effort["combined_compliance_ratio"], + "p99_sla_eligible": best_effort["p99_sla_eligible"], }, "candidates": candidates, } @@ -301,6 +437,12 @@ def main() -> int: encoding="utf-8", ) + if winner is None: + print("No SLA-feasible runtime configuration was found.") + print(f"Wrote diagnostic evidence to {output_json}") + print("No recommended config was written.") + return 2 + print("Recommended runtime configuration") print() print(f" max-num-seqs: {winner['max_num_seqs']}") @@ -309,11 +451,12 @@ def main() -> int: print(f"Selection objective: {objective}") if use_goodput: print(f" mean request goodput: {winner['mean_request_goodput']:.2f} req/s") + print(f" combined compliance: {winner['combined_compliance_percent']:.2f}%") print(f" mean output throughput: {winner['mean_output_throughput']:.2f} tok/s") - if winner["mean_p99_ttft_ms"] is not None: - print(f" mean P99 TTFT: {winner['mean_p99_ttft_ms']:.2f} ms") - if winner["mean_p99_tpot_ms"] is not None: - print(f" mean P99 TPOT: {winner['mean_p99_tpot_ms']:.2f} ms") + if winner["median_p99_ttft_ms"] is not None: + print(f" median P99 TTFT: {winner['median_p99_ttft_ms']:.2f} ms") + if winner["median_p99_tpot_ms"] is not None: + print(f" median P99 TPOT: {winner['median_p99_tpot_ms']:.2f} ms") print() print(f"Wrote {output_config}") print(f"Wrote {output_json}")