From 9b3382685e85d6b7e6ab4adbe91f95a9924622fa Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Tue, 11 Aug 2026 18:42:08 -0700 Subject: [PATCH] [Tools] vLLM Recipes conversion : support different data types variants and strategies (#51878) Signed-off-by: louie-tsai --- tools/recipes/README.md | 74 +++++++++--- tools/recipes/recipe_json_to_vllm_config.py | 118 ++++++++++++++++++-- 2 files changed, 167 insertions(+), 25 deletions(-) diff --git a/tools/recipes/README.md b/tools/recipes/README.md index 746f5e93a4..ac4cf3d298 100644 --- a/tools/recipes/README.md +++ b/tools/recipes/README.md @@ -9,12 +9,31 @@ Converts a hardware-specific vLLM Recipes JSON rendering into: - `config.yml` — native configuration for `vllm serve --config` - `env.sh` — environment variables required by the recipe +The converter uses the Recipes JSON API as the source of truth. It does not reimplement model-variant, hardware, or strategy compatibility rules. + ## Install Dependency ```bash pip install pyyaml ``` +## Recipe Selection Model + +Discovery follows the JSON links published by vLLM Recipes: + +```text +/models.json + -> /{hf_id}.json + -> recommended_command.by_hardware[{hardware}] + -> /{hf_id}/hw/{hardware}.json + -> recommended strategy + -> alternatives[{strategy}] +``` + +The per-hardware JSON is the recommended deployment rendering for that model and hardware. Alternative strategy renderings are referenced through the JSON's `alternatives` map. + +The converter follows those exact links instead of constructing strategy paths locally. This also handles the Recipes API's legacy default-hardware strategy path automatically. + ## Interactive Recipe Discovery If you do not know the recipe JSON URL, run the script without an input: @@ -25,33 +44,39 @@ python3 tools/recipes/recipe_json_to_vllm_config.py The script will: -1. Search the Recipes model index. +1. Search the Recipes model index, including promoted variant model IDs. 2. Ask which model to use. 3. Show the hardware configurations available for that model. -4. Resolve the correct hardware-specific JSON endpoint. -5. Generate `config.yml` and `env.sh`. +4. Show the recommended strategy and generated alternative strategies. +5. Resolve the exact Recipes JSON endpoint. +6. Generate `config.yml` and `env.sh`. Example interaction: ```text No recipe JSON supplied; starting Recipes API discovery. -Model search (for example: llama 3.1): llama 3.1 +Model search (for example: llama 3.1): llama 3.1 fp8 Matching models: - [1] meta-llama/Llama-3.1-8B-Instruct — Llama-3.1-8B-Instruct [Meta] + [1] nvidia/Llama-3.1-8B-Instruct-FP8 — Llama-3.1-8B-Instruct [Meta] Select model: 1 Available hardware: - [1] h100 - [2] h200 - [3] trillium - [4] xeon6 -Select hardware: 4 + [1] arc_pro_b70 + [2] b200 + [3] h100 +Select hardware: 1 + +Available strategies: + [1] single_node_tp (recommended) + [2] multi_node_tp +Select strategy: 1 Resolved recipe: - Model: meta-llama/Llama-3.1-8B-Instruct - Hardware: xeon6 - JSON: https://recipes.vllm.ai/meta-llama/Llama-3.1-8B-Instruct/hw/xeon6.json + Model: nvidia/Llama-3.1-8B-Instruct-FP8 + Hardware: arc_pro_b70 + Strategy: single_node_tp + JSON: https://recipes.vllm.ai/nvidia/Llama-3.1-8B-Instruct-FP8/hw/arc_pro_b70.json ``` ## Non-Interactive Recipe Discovery @@ -64,7 +89,18 @@ python3 tools/recipes/recipe_json_to_vllm_config.py \ --hardware xeon6 ``` -The script verifies that the requested hardware rendering exists before using it. +When `--strategy` is omitted with both `--model` and `--hardware`, the converter uses the Recipes-recommended strategy from the per-hardware JSON. + +To request a generated alternative strategy explicitly: + +```bash +python3 tools/recipes/recipe_json_to_vllm_config.py \ + --model nvidia/Llama-3.1-8B-Instruct-FP8 \ + --hardware arc_pro_b70 \ + --strategy multi_node_tp +``` + +The converter does not synthesize a URL such as `.../strategies/multi_node_tp.json`. It reads the selected hardware JSON and follows the exact `alternatives["multi_node_tp"]` link published by the Recipes API. ## Direct Recipe JSON Input @@ -81,6 +117,8 @@ A local JSON file is also supported: python3 tools/recipes/recipe_json_to_vllm_config.py recipe.json ``` +When a positional JSON source is supplied, do not combine it with `--model`, `--hardware`, or `--strategy`. + ## Start vLLM After generating the files: @@ -109,8 +147,10 @@ python3 tools/recipes/recipe_json_to_vllm_config.py \ --env-out llama31-xeon6-env.sh ``` -## Scope +## Strategy and Deployment Scope -The converter currently targets recipes that resolve to a single `vllm serve` process. +Strategy discovery and config conversion are separate concerns. -Multi-node, disaggregated prefill/decode, or other multi-process deployment recipes require multiple commands and cannot be represented by a single `config.yml`. +The converter can resolve any generated strategy JSON exposed by the Recipes API. It currently emits one `config.yml`, so the selected rendering must contain a single `vllm serve` `argv`. + +Single-process renderings such as `single_node_tp` can be converted directly. Multi-node, disaggregated prefill/decode, or other multi-process renderings expose fields such as `head_argv`, `worker_argv`, `prefill`, or `decode`; the converter intentionally exits instead of generating an incomplete single-process config. diff --git a/tools/recipes/recipe_json_to_vllm_config.py b/tools/recipes/recipe_json_to_vllm_config.py index 3016ae77b9..52c109dcba 100644 --- a/tools/recipes/recipe_json_to_vllm_config.py +++ b/tools/recipes/recipe_json_to_vllm_config.py @@ -16,14 +16,25 @@ Examples: python3 recipe_json_to_vllm_config.py \ https://recipes.vllm.ai/meta-llama/Llama-3.1-8B-Instruct/hw/xeon6.json - # Non-interactive discovery + # Non-interactive discovery (recommended strategy) python3 recipe_json_to_vllm_config.py \ --model meta-llama/Llama-3.1-8B-Instruct --hardware xeon6 + # Promoted variant + explicit strategy + python3 recipe_json_to_vllm_config.py \ + --model nvidia/Llama-3.1-8B-Instruct-FP8 \ + --hardware arc_pro_b70 \ + --strategy single_node_tp + Then: source env.sh vllm serve --config config.yml +Variants with a distinct Hugging Face model ID are promoted by the Recipes API +and selected through --model; this converter does not need a separate --variant +argument. Strategy selection follows the exact `alternatives` JSON links emitted +by the Recipes API instead of synthesizing strategy URLs locally. + The converter intentionally targets a single `vllm serve` process. If the recipe rendering is multi-node, PD-disaggregated, or another multi-process deployment, it exits instead of silently generating an incomplete config. @@ -71,13 +82,25 @@ def parse_args() -> argparse.Namespace: p.add_argument( "--model", help=( - "Model hf_id or search text for recipe discovery. " - "Use with --hardware for non-interactive operation." + "Model hf_id or search text for recipe discovery. Promoted recipe " + "variants are selected by their own hf_id. Use with --hardware for " + "non-interactive operation." ), ) p.add_argument( "--hardware", - help="Hardware ID for recipe discovery, for example xeon6 or h100.", + help=( + "Hardware ID for recipe discovery, for example xeon6, b200, or arc_pro_b70." + ), + ) + p.add_argument( + "--strategy", + help=( + "Serving strategy for recipe discovery, for example single_node_tp. " + "When omitted with --model and --hardware, use the Recipes-recommended " + "strategy. Interactive discovery offers the recommended strategy and " + "all generated alternatives." + ), ) p.add_argument( "--api-base", @@ -238,10 +261,74 @@ def select_hardware( return selected, by_hardware[selected] +def strategy_sources( + api_base: str, hardware_json_url: str, recipe: dict[str, Any] +) -> tuple[str, dict[str, str]]: + recommended = recipe.get("strategy") + if not isinstance(recommended, str) or not recommended: + raise ValueError( + "Hardware recipe JSON does not contain a usable `strategy` field." + ) + + sources = {recommended: hardware_json_url} + raw_alternatives = recipe.get("alternatives") or {} + if not isinstance(raw_alternatives, dict): + raise ValueError( + "Hardware recipe JSON `alternatives` must be an object when present." + ) + + for strategy, path in raw_alternatives.items(): + if isinstance(strategy, str) and strategy and isinstance(path, str) and path: + sources[strategy] = api_url(api_base, path) + + return recommended, sources + + +def select_strategy( + api_base: str, + hardware_json_url: str, + recipe: dict[str, Any], + requested: str | None, + interactive: bool, +) -> tuple[str, str]: + recommended, sources = strategy_sources(api_base, hardware_json_url, recipe) + strategies = [recommended, *sorted(s for s in sources if s != recommended)] + + if requested: + selected = next( + ( + strategy + for strategy in strategies + if strategy.lower() == requested.lower() + ), + None, + ) + if selected is None: + raise ValueError( + f"Strategy {requested!r} is not available for this model/hardware. " + f"Available: {', '.join(strategies)}" + ) + return selected, sources[selected] + + if not interactive: + return recommended, hardware_json_url + + print("\nAvailable strategies:") + + def strategy_label(strategy: str) -> str: + if strategy == recommended: + return f"{strategy} (recommended)" + return strategy + + selected = choose_from_menu(strategies, strategy_label, "Select strategy: ") + return selected, sources[selected] + + def discover_recipe_source( api_base: str, requested_model: str | None, requested_hardware: str | None, + requested_strategy: str | None, ) -> str: print("No recipe JSON supplied; starting Recipes API discovery.") @@ -287,11 +374,24 @@ def discover_recipe_source( ) hardware, path = select_hardware(by_hardware, requested_hardware) - resolved = api_url(api_base, path) + hardware_json_url = api_url(api_base, path) + hardware_data = load_json(hardware_json_url) + if not isinstance(hardware_data, dict): + raise ValueError(f"{hardware_json_url} did not return a JSON object.") + + interactive = requested_model is None or requested_hardware is None + strategy, resolved = select_strategy( + api_base, + hardware_json_url, + hardware_data, + requested_strategy, + interactive, + ) print("\nResolved recipe:") print(f" Model: {model.get('hf_id')}") print(f" Hardware: {hardware}") + print(f" Strategy: {strategy}") print(f" JSON: {resolved}") print() @@ -527,11 +627,13 @@ def main() -> int: try: source = args.source if source is None: - source = discover_recipe_source(args.api_base, args.model, args.hardware) - elif args.model or args.hardware: + source = discover_recipe_source( + args.api_base, args.model, args.hardware, args.strategy + ) + elif args.model or args.hardware or args.strategy: raise ValueError( "Do not combine a positional recipe JSON source with " - "--model/--hardware discovery options." + "--model/--hardware/--strategy discovery options." ) recipe = load_json(source)